Fix Database Locking Issues

scripts/birdnet_analysis.py

@@ -14,7 +14,7 @@
 from utils.analysis import load_global_model, run_analysis
 from utils.helpers import get_settings, get_wav_files, ANALYZING_NOW
 from utils.classes import ParseFileName
-from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \
+from utils.reporting import extract_detection, summary, write_to_file, write_to_db, write_detections_to_db, apprise, bird_weather, heartbeat, \
     update_json_file
 
 shutdown = False
@@ -94,6 +94,7 @@ def process_file(file_name, report_queue):
         if not report_queue.empty():
             log.warning('reporting queue not yet empty')
         report_queue.join()
+        # Use batch insert to reduce database transactions and locking
         report_queue.put((file, detections))
     except BaseException as e:
         stderr = e.stderr.decode('utf-8') if isinstance(e, CalledProcessError) else ""
@@ -114,7 +115,7 @@ def handle_reporting_queue(queue):
                 detection.file_name_extr = extract_detection(file, detection)
                 log.info('%s;%s', summary(file, detection), os.path.basename(detection.file_name_extr))
                 write_to_file(file, detection)
-                write_to_db(file, detection)
+            write_detections_to_db(file, detections)  # Moved outside loop to avoid duplicate writes
             apprise(file, detections)
             bird_weather(file, detections)
             heartbeat()

scripts/common.php

@@ -121,11 +121,12 @@ function get_label($record, $sort_by, $date=null) {
 }
 
 function get_db() {
-  if (!isset($_db)) {
-    $_db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-    $_db->busyTimeout(1000);
+  static $db = null;
+  if ($db === null) {
+    $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
+    $db->busyTimeout(5000);  // Increased from 1000ms to 5000ms
   }
-  return $_db;
+  return $db;
 }
 
 function fetch_species_array($sort_by, $date=null) {
@@ -254,7 +255,7 @@ protected function set_db() {
     } catch (Exception $ex) {
       $this->create_tables();
     }
-    $this->db->busyTimeout(1000);
+    $this->db->busyTimeout(5000);
   }
 
   protected function create_tables() {

scripts/history.php

@@ -19,7 +19,7 @@
 $chart2 = "Combo2-$theDate.png";
 
 $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 $statement1 = $db->prepare("SELECT COUNT(*) FROM detections WHERE Date == \"$theDate\"");
 ensure_db_ok($statement1);

scripts/overview.php

@@ -13,7 +13,7 @@
 $chart = "Combo-$myDate.png";
 
 $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 if(isset($_GET['custom_image'])){
   if(isset($config["CUSTOM_IMAGE"])) {

scripts/play.php

@@ -12,7 +12,7 @@
 $user = get_user();
 
 $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 if(isset($_GET['deletefile'])) {
   ensure_authenticated('You must be authenticated to delete files.');
@@ -21,7 +21,7 @@
     die();
   }
   $db_writable = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READWRITE);
-  $db->busyTimeout(1000);
+  $db->busyTimeout(5000);
   $statement1 = $db_writable->prepare('DELETE FROM detections WHERE File_Name = :file_name LIMIT 1');
   ensure_db_ok($statement1);
   $statement1->bindValue(':file_name', explode("/", $_GET['deletefile'])[2]);

scripts/species_tools.php

@@ -33,7 +33,7 @@
 /* ---------- DB open (RO unless deleting) ---------- */
 $flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY;
 $db   = new SQLite3(__DIR__ . '/birds.db', $flags);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 /* Paths / lists */
 $base_symlink   = $home . '/BirdSongs/Extracted/By_Date';

scripts/todays_detections.php

@@ -23,7 +23,7 @@
 }
 
 $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 $summary = get_summary();
 $totalcount = $summary['totalcount'];

scripts/utils/reporting.py

@@ -93,6 +93,8 @@ def write_to_db(file: ParseFileName, detection: Detection):
     for attempt_number in range(3):
         try:
             con = sqlite3.connect(DB_PATH)
+            con.execute("PRAGMA journal_mode=WAL;")
+            con.execute("PRAGMA synchronous=NORMAL;")
             cur = con.cursor()
             cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                         (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence,
@@ -110,6 +112,35 @@ def write_to_db(file: ParseFileName, detection: Detection):
             sleep(2)
 
 
+def write_detections_to_db(file: ParseFileName, detections: list):
+    """Batch insert multiple detections in a single transaction to reduce locking"""
+    conf = get_settings()
+    for attempt_number in range(3):
+        try:
+            con = sqlite3.connect(DB_PATH)
+            con.execute("PRAGMA journal_mode=WAL;")
+            con.execute("PRAGMA synchronous=NORMAL;")
+            cur = con.cursor()
+
+            # Start transaction
+            cur.execute("BEGIN TRANSACTION;")
+
+            # Insert all detections
+            for detection in detections:
+                cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                            (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence,
+                             conf['LATITUDE'], conf['LONGITUDE'], conf['CONFIDENCE'], str(detection.week), conf['SENSITIVITY'],
+                             conf['OVERLAP'], os.path.basename(detection.file_name_extr)))
+
+            # Commit all at once
+            con.commit()
+            con.close()
+            break
+        except BaseException as e:
+            log.warning("Database busy: %s", e)
+            sleep(2)
+
+
 def summary(file: ParseFileName, detection: Detection):
     # Date;Time;Sci_Name;Com_Name;Confidence;Lat;Lon;Cutoff;Week;Sens;Overlap
     # 2023-03-03;12:48:01;Phleocryptes melanops;Wren-like Rushbird;0.76950216;-1;-1;0.7;9;1.25;0.0

scripts/weekly_report.php

@@ -23,7 +23,7 @@ function safe_percentage($count, $prior_count) {
 }
 
 $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY);
-$db->busyTimeout(1000);
+$db->busyTimeout(5000);
 
 $statement1 = $db->prepare('SELECT Sci_Name, Com_Name, COUNT(*) FROM detections WHERE Date BETWEEN "' . date("Y-m-d", $startdate) . '" AND "' . date("Y-m-d", $enddate) . '" GROUP By Sci_Name ORDER BY COUNT(*) DESC');
 ensure_db_ok($statement1);