diff --git a/scripts/birdnet_analysis.py b/scripts/birdnet_analysis.py index ec805286c..ba02808fa 100644 --- a/scripts/birdnet_analysis.py +++ b/scripts/birdnet_analysis.py @@ -14,7 +14,7 @@ from utils.analysis import load_global_model, run_analysis from utils.helpers import get_settings, get_wav_files, ANALYZING_NOW from utils.classes import ParseFileName -from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \ +from utils.reporting import extract_detection, summary, write_to_file, write_to_db, write_detections_to_db, apprise, bird_weather, heartbeat, \ update_json_file shutdown = False @@ -94,6 +94,7 @@ def process_file(file_name, report_queue): if not report_queue.empty(): log.warning('reporting queue not yet empty') report_queue.join() + # Use batch insert to reduce database transactions and locking report_queue.put((file, detections)) except BaseException as e: stderr = e.stderr.decode('utf-8') if isinstance(e, CalledProcessError) else "" @@ -114,7 +115,7 @@ def handle_reporting_queue(queue): detection.file_name_extr = extract_detection(file, detection) log.info('%s;%s', summary(file, detection), os.path.basename(detection.file_name_extr)) write_to_file(file, detection) - write_to_db(file, detection) + write_detections_to_db(file, detections) # Moved outside loop to avoid duplicate writes apprise(file, detections) bird_weather(file, detections) heartbeat() diff --git a/scripts/common.php b/scripts/common.php index 4d0fc2a47..f76d07860 100644 --- a/scripts/common.php +++ b/scripts/common.php @@ -121,11 +121,12 @@ function get_label($record, $sort_by, $date=null) { } function get_db() { - if (!isset($_db)) { - $_db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); - $_db->busyTimeout(1000); + static $db = null; + if ($db === null) { + $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); + $db->busyTimeout(5000); // Increased from 1000ms to 5000ms } - return $_db; + return $db; } function fetch_species_array($sort_by, $date=null) { @@ -254,7 +255,7 @@ protected function set_db() { } catch (Exception $ex) { $this->create_tables(); } - $this->db->busyTimeout(1000); + $this->db->busyTimeout(5000); } protected function create_tables() { diff --git a/scripts/history.php b/scripts/history.php index 48455bae9..6db21af68 100644 --- a/scripts/history.php +++ b/scripts/history.php @@ -19,7 +19,7 @@ $chart2 = "Combo2-$theDate.png"; $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $statement1 = $db->prepare("SELECT COUNT(*) FROM detections WHERE Date == \"$theDate\""); ensure_db_ok($statement1); diff --git a/scripts/overview.php b/scripts/overview.php index 8026acd6a..811033792 100644 --- a/scripts/overview.php +++ b/scripts/overview.php @@ -13,7 +13,7 @@ $chart = "Combo-$myDate.png"; $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); if(isset($_GET['custom_image'])){ if(isset($config["CUSTOM_IMAGE"])) { diff --git a/scripts/play.php b/scripts/play.php index 1b520796a..7f056f304 100644 --- a/scripts/play.php +++ b/scripts/play.php @@ -12,7 +12,7 @@ $user = get_user(); $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); if(isset($_GET['deletefile'])) { ensure_authenticated('You must be authenticated to delete files.'); @@ -21,7 +21,7 @@ die(); } $db_writable = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READWRITE); - $db->busyTimeout(1000); + $db->busyTimeout(5000); $statement1 = $db_writable->prepare('DELETE FROM detections WHERE File_Name = :file_name LIMIT 1'); ensure_db_ok($statement1); $statement1->bindValue(':file_name', explode("/", $_GET['deletefile'])[2]); diff --git a/scripts/species_tools.php b/scripts/species_tools.php index 3ca70b879..50b223a56 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -33,7 +33,7 @@ /* ---------- DB open (RO unless deleting) ---------- */ $flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY; $db = new SQLite3(__DIR__ . '/birds.db', $flags); -$db->busyTimeout(1000); +$db->busyTimeout(5000); /* Paths / lists */ $base_symlink = $home . '/BirdSongs/Extracted/By_Date'; diff --git a/scripts/todays_detections.php b/scripts/todays_detections.php index 1e58a7824..79fb983a9 100644 --- a/scripts/todays_detections.php +++ b/scripts/todays_detections.php @@ -23,7 +23,7 @@ } $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $summary = get_summary(); $totalcount = $summary['totalcount']; diff --git a/scripts/utils/reporting.py b/scripts/utils/reporting.py index dfa8a6619..5e7bb965b 100644 --- a/scripts/utils/reporting.py +++ b/scripts/utils/reporting.py @@ -93,6 +93,8 @@ def write_to_db(file: ParseFileName, detection: Detection): for attempt_number in range(3): try: con = sqlite3.connect(DB_PATH) + con.execute("PRAGMA journal_mode=WAL;") + con.execute("PRAGMA synchronous=NORMAL;") cur = con.cursor() cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence, @@ -110,6 +112,35 @@ def write_to_db(file: ParseFileName, detection: Detection): sleep(2) +def write_detections_to_db(file: ParseFileName, detections: list): + """Batch insert multiple detections in a single transaction to reduce locking""" + conf = get_settings() + for attempt_number in range(3): + try: + con = sqlite3.connect(DB_PATH) + con.execute("PRAGMA journal_mode=WAL;") + con.execute("PRAGMA synchronous=NORMAL;") + cur = con.cursor() + + # Start transaction + cur.execute("BEGIN TRANSACTION;") + + # Insert all detections + for detection in detections: + cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence, + conf['LATITUDE'], conf['LONGITUDE'], conf['CONFIDENCE'], str(detection.week), conf['SENSITIVITY'], + conf['OVERLAP'], os.path.basename(detection.file_name_extr))) + + # Commit all at once + con.commit() + con.close() + break + except BaseException as e: + log.warning("Database busy: %s", e) + sleep(2) + + def summary(file: ParseFileName, detection: Detection): # Date;Time;Sci_Name;Com_Name;Confidence;Lat;Lon;Cutoff;Week;Sens;Overlap # 2023-03-03;12:48:01;Phleocryptes melanops;Wren-like Rushbird;0.76950216;-1;-1;0.7;9;1.25;0.0 diff --git a/scripts/weekly_report.php b/scripts/weekly_report.php index c31d5ae3d..590ab8717 100644 --- a/scripts/weekly_report.php +++ b/scripts/weekly_report.php @@ -23,7 +23,7 @@ function safe_percentage($count, $prior_count) { } $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $statement1 = $db->prepare('SELECT Sci_Name, Com_Name, COUNT(*) FROM detections WHERE Date BETWEEN "' . date("Y-m-d", $startdate) . '" AND "' . date("Y-m-d", $enddate) . '" GROUP By Sci_Name ORDER BY COUNT(*) DESC'); ensure_db_ok($statement1);