From 9b901b0d96873f17596a763cf95bf8120dc2e8ff Mon Sep 17 00:00:00 2001 From: Frank Kunz Date: Wed, 15 Apr 2026 20:22:45 +0200 Subject: [PATCH 1/4] Fix database locking issue in common.php The get_db() function was creating a new database connection on every call due to incorrect variable scoping. This caused connection leaks that led to 'Database busy. database is locked' errors when reloading pages. Changed from using a global variable (which wasn't working due to missing global keyword) to a static variable that maintains state within the function scope. This ensures only one connection is created per request, preventing connection accumulation and permanent locks. --- scripts/common.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/common.php b/scripts/common.php index 4d0fc2a47..8b2c9d186 100644 --- a/scripts/common.php +++ b/scripts/common.php @@ -121,11 +121,12 @@ function get_label($record, $sort_by, $date=null) { } function get_db() { - if (!isset($_db)) { - $_db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); - $_db->busyTimeout(1000); + static $db = null; + if ($db === null) { + $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); + $db->busyTimeout(1000); } - return $_db; + return $db; } function fetch_species_array($sort_by, $date=null) { From 2d86b93de6b3d5d3d5244ee062453d302bbc14f8 Mon Sep 17 00:00:00 2001 From: Frank Kunz Date: Thu, 16 Apr 2026 21:07:05 +0200 Subject: [PATCH 2/4] Enable WAL mode in Python reporting to allow concurrent reads during writes WAL (Write-Ahead Logging) mode allows SQLite readers to continue reading data even while a write transaction is in progress. This is critical for applications like BirdNET-Pi where the analysis script frequently writes detections while the web interface reads from the database. This change enables WAL mode and sets synchronous=NORMAL for better performance while maintaining data safety. --- scripts/utils/reporting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/utils/reporting.py b/scripts/utils/reporting.py index dfa8a6619..ee8d0b6c9 100644 --- a/scripts/utils/reporting.py +++ b/scripts/utils/reporting.py @@ -93,6 +93,8 @@ def write_to_db(file: ParseFileName, detection: Detection): for attempt_number in range(3): try: con = sqlite3.connect(DB_PATH) + con.execute("PRAGMA journal_mode=WAL;") + con.execute("PRAGMA synchronous=NORMAL;") cur = con.cursor() cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence, From abb03ad035b0d7f1ece6fbd91f99daeee526639b Mon Sep 17 00:00:00 2001 From: Frank Kunz Date: Thu, 16 Apr 2026 21:07:27 +0200 Subject: [PATCH 3/4] Implement batch inserts to reduce database locking Instead of creating a separate transaction for each detection, this change batches all detections from a single audio file into one transaction. This significantly reduces the number of write operations and exclusive locks on the database, allowing the web interface to read more reliably during analysis. The batch function write_detections_to_db() handles multiple detections in a single BEGIN/COMMIT transaction, minimizing lock duration. --- scripts/birdnet_analysis.py | 5 +++-- scripts/utils/reporting.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/birdnet_analysis.py b/scripts/birdnet_analysis.py index ec805286c..ba02808fa 100644 --- a/scripts/birdnet_analysis.py +++ b/scripts/birdnet_analysis.py @@ -14,7 +14,7 @@ from utils.analysis import load_global_model, run_analysis from utils.helpers import get_settings, get_wav_files, ANALYZING_NOW from utils.classes import ParseFileName -from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \ +from utils.reporting import extract_detection, summary, write_to_file, write_to_db, write_detections_to_db, apprise, bird_weather, heartbeat, \ update_json_file shutdown = False @@ -94,6 +94,7 @@ def process_file(file_name, report_queue): if not report_queue.empty(): log.warning('reporting queue not yet empty') report_queue.join() + # Use batch insert to reduce database transactions and locking report_queue.put((file, detections)) except BaseException as e: stderr = e.stderr.decode('utf-8') if isinstance(e, CalledProcessError) else "" @@ -114,7 +115,7 @@ def handle_reporting_queue(queue): detection.file_name_extr = extract_detection(file, detection) log.info('%s;%s', summary(file, detection), os.path.basename(detection.file_name_extr)) write_to_file(file, detection) - write_to_db(file, detection) + write_detections_to_db(file, detections) # Moved outside loop to avoid duplicate writes apprise(file, detections) bird_weather(file, detections) heartbeat() diff --git a/scripts/utils/reporting.py b/scripts/utils/reporting.py index ee8d0b6c9..5e7bb965b 100644 --- a/scripts/utils/reporting.py +++ b/scripts/utils/reporting.py @@ -112,6 +112,35 @@ def write_to_db(file: ParseFileName, detection: Detection): sleep(2) +def write_detections_to_db(file: ParseFileName, detections: list): + """Batch insert multiple detections in a single transaction to reduce locking""" + conf = get_settings() + for attempt_number in range(3): + try: + con = sqlite3.connect(DB_PATH) + con.execute("PRAGMA journal_mode=WAL;") + con.execute("PRAGMA synchronous=NORMAL;") + cur = con.cursor() + + # Start transaction + cur.execute("BEGIN TRANSACTION;") + + # Insert all detections + for detection in detections: + cur.execute("INSERT INTO detections VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (detection.date, detection.time, detection.scientific_name, detection.common_name, detection.confidence, + conf['LATITUDE'], conf['LONGITUDE'], conf['CONFIDENCE'], str(detection.week), conf['SENSITIVITY'], + conf['OVERLAP'], os.path.basename(detection.file_name_extr))) + + # Commit all at once + con.commit() + con.close() + break + except BaseException as e: + log.warning("Database busy: %s", e) + sleep(2) + + def summary(file: ParseFileName, detection: Detection): # Date;Time;Sci_Name;Com_Name;Confidence;Lat;Lon;Cutoff;Week;Sens;Overlap # 2023-03-03;12:48:01;Phleocryptes melanops;Wren-like Rushbird;0.76950216;-1;-1;0.7;9;1.25;0.0 From ded7eedc3e172c1457afa64c42d2e2f7cee1e3a7 Mon Sep 17 00:00:00 2001 From: Frank Kunz Date: Thu, 16 Apr 2026 21:25:32 +0200 Subject: [PATCH 4/4] Increase busy timeout from 1s to 5s in all PHP scripts The 1-second busy timeout was too short for scenarios where the Python analysis script holds write locks while processing multiple detections. Increasing to 5 seconds gives the web interface more time to wait for write operations to complete before showing 'database is busy' errors. This change affects all PHP scripts that access the database: - common.php (get_db and ImageProvider) - history.php - overview.php - play.php - species_tools.php - todays_detections.php - weekly_report.php --- scripts/common.php | 4 ++-- scripts/history.php | 2 +- scripts/overview.php | 2 +- scripts/play.php | 4 ++-- scripts/species_tools.php | 2 +- scripts/todays_detections.php | 2 +- scripts/weekly_report.php | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/common.php b/scripts/common.php index 8b2c9d186..f76d07860 100644 --- a/scripts/common.php +++ b/scripts/common.php @@ -124,7 +124,7 @@ function get_db() { static $db = null; if ($db === null) { $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); - $db->busyTimeout(1000); + $db->busyTimeout(5000); // Increased from 1000ms to 5000ms } return $db; } @@ -255,7 +255,7 @@ protected function set_db() { } catch (Exception $ex) { $this->create_tables(); } - $this->db->busyTimeout(1000); + $this->db->busyTimeout(5000); } protected function create_tables() { diff --git a/scripts/history.php b/scripts/history.php index 48455bae9..6db21af68 100644 --- a/scripts/history.php +++ b/scripts/history.php @@ -19,7 +19,7 @@ $chart2 = "Combo2-$theDate.png"; $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $statement1 = $db->prepare("SELECT COUNT(*) FROM detections WHERE Date == \"$theDate\""); ensure_db_ok($statement1); diff --git a/scripts/overview.php b/scripts/overview.php index 8026acd6a..811033792 100644 --- a/scripts/overview.php +++ b/scripts/overview.php @@ -13,7 +13,7 @@ $chart = "Combo-$myDate.png"; $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); if(isset($_GET['custom_image'])){ if(isset($config["CUSTOM_IMAGE"])) { diff --git a/scripts/play.php b/scripts/play.php index 1b520796a..7f056f304 100644 --- a/scripts/play.php +++ b/scripts/play.php @@ -12,7 +12,7 @@ $user = get_user(); $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); if(isset($_GET['deletefile'])) { ensure_authenticated('You must be authenticated to delete files.'); @@ -21,7 +21,7 @@ die(); } $db_writable = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READWRITE); - $db->busyTimeout(1000); + $db->busyTimeout(5000); $statement1 = $db_writable->prepare('DELETE FROM detections WHERE File_Name = :file_name LIMIT 1'); ensure_db_ok($statement1); $statement1->bindValue(':file_name', explode("/", $_GET['deletefile'])[2]); diff --git a/scripts/species_tools.php b/scripts/species_tools.php index 3ca70b879..50b223a56 100644 --- a/scripts/species_tools.php +++ b/scripts/species_tools.php @@ -33,7 +33,7 @@ /* ---------- DB open (RO unless deleting) ---------- */ $flags = isset($_GET['delete']) ? SQLITE3_OPEN_READWRITE : SQLITE3_OPEN_READONLY; $db = new SQLite3(__DIR__ . '/birds.db', $flags); -$db->busyTimeout(1000); +$db->busyTimeout(5000); /* Paths / lists */ $base_symlink = $home . '/BirdSongs/Extracted/By_Date'; diff --git a/scripts/todays_detections.php b/scripts/todays_detections.php index 1e58a7824..79fb983a9 100644 --- a/scripts/todays_detections.php +++ b/scripts/todays_detections.php @@ -23,7 +23,7 @@ } $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $summary = get_summary(); $totalcount = $summary['totalcount']; diff --git a/scripts/weekly_report.php b/scripts/weekly_report.php index c31d5ae3d..590ab8717 100644 --- a/scripts/weekly_report.php +++ b/scripts/weekly_report.php @@ -23,7 +23,7 @@ function safe_percentage($count, $prior_count) { } $db = new SQLite3('./scripts/birds.db', SQLITE3_OPEN_READONLY); -$db->busyTimeout(1000); +$db->busyTimeout(5000); $statement1 = $db->prepare('SELECT Sci_Name, Com_Name, COUNT(*) FROM detections WHERE Date BETWEEN "' . date("Y-m-d", $startdate) . '" AND "' . date("Y-m-d", $enddate) . '" GROUP By Sci_Name ORDER BY COUNT(*) DESC'); ensure_db_ok($statement1);