diff --git a/bin/maillogsentinel_setup.py b/bin/maillogsentinel_setup.py index 2ae9be4..a36ef35 100644 --- a/bin/maillogsentinel_setup.py +++ b/bin/maillogsentinel_setup.py @@ -751,7 +751,7 @@ def interactive_cli_setup(target_config_path, setup_log_fh): setup_log_fh, ) _setup_print_and_log( - f"You may need to manually perform these steps or re-run with sudo IF you want systemd integration.", + "You may need to manually perform these steps or re-run with sudo IF you want systemd integration.", setup_log_fh, ) _setup_print_and_log( @@ -965,10 +965,14 @@ def interactive_cli_setup(target_config_path, setup_log_fh): f"FATAL ERROR during interactive setup: {e_main_interactive.__class__.__name__}: {e_main_interactive}", setup_log_fh, ) - import traceback + _setup_print_and_log( + f"FATAL ERROR during interactive setup: {e_main_interactive.__class__.__name__}: {e_main_interactive}", + setup_log_fh, + ) + # import traceback # No longer needed - _setup_print_and_log(traceback.format_exc(), setup_log_fh) - return False # Indicate failure + # _setup_print_and_log(traceback.format_exc(), setup_log_fh) # No longer needed + return False return True @@ -983,13 +987,12 @@ def non_interactive_setup(source_config_path: Path, setup_log_fh): Updates global backed_up_items and created_final_paths lists. """ import sys # Ensure sys is imported for stdout - import traceback print("non_interactive_setup CALLED", flush=True) # traceback.print_stack(file=sys.stdout, limit=10) # Removed for debugging print("---", flush=True) - _setup_print_and_log(f"--- MailLogSentinel Non-Interactive Setup ---", setup_log_fh) + _setup_print_and_log("--- MailLogSentinel Non-Interactive Setup ---", setup_log_fh) if os.geteuid() != 0: _setup_print_and_log( @@ -1156,7 +1159,7 @@ def non_interactive_setup(source_config_path: Path, setup_log_fh): ) usermod_cmd = shutil.which("usermod") if not usermod_cmd: - _setup_print_and_log(f"ERROR: 'usermod' not found.", setup_log_fh) + _setup_print_and_log("ERROR: 'usermod' not found.", setup_log_fh) sys.exit(1) # Restored error try: process_result = subprocess.run( diff --git a/lib/maillogsentinel/parser.py b/lib/maillogsentinel/parser.py index 1d4239e..3749de0 100644 --- a/lib/maillogsentinel/parser.py +++ b/lib/maillogsentinel/parser.py @@ -18,7 +18,6 @@ Callable, ) import logging -import sys # Add bin directory to sys.path to allow importing ipinfo # sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "bin")) # Removed for cleaner path management diff --git a/lib/maillogsentinel/sql_exporter.py b/lib/maillogsentinel/sql_exporter.py index 88b6312..f6812f2 100644 --- a/lib/maillogsentinel/sql_exporter.py +++ b/lib/maillogsentinel/sql_exporter.py @@ -8,14 +8,14 @@ import csv import datetime -import hashlib import json import logging -import os from pathlib import Path from typing import Optional, List, Dict, Any import importlib.resources # Added for loading bundled data +import tempfile +import shutil from lib.maillogsentinel.config import AppConfig # Import AppConfig # Constants @@ -608,7 +608,7 @@ def run_sql_export(config: AppConfig, output_log_level: str = "INFO") -> bool: if sql_file_path.exists(): sql_file_path.unlink(missing_ok=True) # cleanup return False - except CSVSchemaError as e: # Already logged by validate_csv_header + except CSVSchemaError: # Already logged by validate_csv_header # Cleanup already handled in validate_csv_header's calling block return False except Exception as e: @@ -663,10 +663,6 @@ def run_sql_export(config: AppConfig, output_log_level: str = "INFO") -> bool: return True -import tempfile -import shutil - - # Test configuration class class DummyTestConfig: def __init__(self, mapping_file_path_str="", base_dir_name=None): diff --git a/lib/maillogsentinel/sql_importer.py b/lib/maillogsentinel/sql_importer.py index 77bd264..d61192a 100644 --- a/lib/maillogsentinel/sql_importer.py +++ b/lib/maillogsentinel/sql_importer.py @@ -10,10 +10,9 @@ import logging import sqlite3 import time -import datetime # For handling imported file records import os # For os.getpid() in FileLock from pathlib import Path -from typing import List, Tuple, Dict, Any, Optional +from typing import Optional import fcntl # For file locking on POSIX systems import importlib.resources # Added for loading bundled data @@ -714,6 +713,3 @@ def __init__(self, base_path: Path): print("\nsql_importer.py direct test finished.") # Note: For a real test, you'd use pytest and mock AppConfig, Path.is_file(), open(), etc. # This direct run is just for very basic flow checking. - -# Required for FileLock to get PID (if not already imported by another module like logging) -import os diff --git a/tests/bin/test_maillogsentinel_setup.py b/tests/bin/test_maillogsentinel_setup.py index b6610bb..7dc467f 100644 --- a/tests/bin/test_maillogsentinel_setup.py +++ b/tests/bin/test_maillogsentinel_setup.py @@ -6,7 +6,6 @@ import tempfile import os import io # For mock_log_fh spec -import locale # For mocking in specific test import re @@ -413,7 +412,6 @@ def test_path_management_creation(self): try: original_backed_up_items = mls_setup.backed_up_items - original_created_final_paths = mls_setup.created_final_paths mls_setup.backed_up_items = [] mls_setup.created_final_paths = [] @@ -433,7 +431,6 @@ def test_path_management_creation(self): finally: os.remove(config_path_str) mls_setup.backed_up_items = original_backed_up_items - mls_setup.created_final_paths = original_created_final_paths def test_path_management_backup_existing(self): """Test backup of workdir and statedir when they already exist.""" @@ -475,7 +472,6 @@ def test_path_management_backup_existing(self): try: original_backed_up_items = mls_setup.backed_up_items - original_created_final_paths = mls_setup.created_final_paths mls_setup.backed_up_items = [] mls_setup.created_final_paths = [] @@ -502,7 +498,6 @@ def test_path_management_backup_existing(self): finally: os.remove(config_path_str) mls_setup.backed_up_items = original_backed_up_items - mls_setup.created_final_paths = original_created_final_paths # User/Group Management Tests def test_user_verification_non_existent(self): @@ -946,7 +941,6 @@ def path_exists_side_effect(*args_passed): config.read_string(VALID_CONFIG_CONTENT) try: - original_created_final_paths = mls_setup.created_final_paths mls_setup.created_final_paths = [] with patch( "configparser.ConfigParser.read", return_value=[config_path_str_val] @@ -1374,7 +1368,7 @@ def test_systemd_control_commands_success(self): "bin.maillogsentinel_setup.subprocess.run" ) as mock_subprocess_run, patch( "bin.maillogsentinel_setup._setup_print_and_log" - ) as mock_setup_print, patch( + ), patch( "bin.maillogsentinel_setup.sys.exit" ) as mock_sys_exit: # noqa: F841 @@ -1405,7 +1399,7 @@ def test_systemd_control_commands_success(self): # Expected calls to systemd-analyze for calendar validation # These come from the VALID_CONFIG_CONTENT and the defaults in non_interactive_setup # Order matters here as they are called before systemctl daemon-reload. - expected_calendar_validation_calls = [ + [ unittest.mock.call( [ "/usr/bin/systemd-analyze", diff --git a/tests/lib/maillogsentinel/test_log_utils.py b/tests/lib/maillogsentinel/test_log_utils.py index 49b5980..a722b23 100644 --- a/tests/lib/maillogsentinel/test_log_utils.py +++ b/tests/lib/maillogsentinel/test_log_utils.py @@ -6,7 +6,6 @@ # Import the function to be tested from lib.maillogsentinel.log_utils import ( _parse_log_line, - MONTHS, ) # MONTHS needed for test_parse_log_line_invalid_month by implication # --- Mocks and Fixtures --- diff --git a/tests/lib/maillogsentinel/test_progress.py b/tests/lib/maillogsentinel/test_progress.py index 3baf40f..45dfae3 100644 --- a/tests/lib/maillogsentinel/test_progress.py +++ b/tests/lib/maillogsentinel/test_progress.py @@ -1,11 +1,10 @@ # tests/lib/maillogsentinel/test_progress.py import unittest -from unittest.mock import patch, MagicMock +from unittest.mock import patch import io import sys from lib.maillogsentinel.progress import ( ProgressTracker, # Import the class - get_terminal_width, GREEN, RED, ORANGE, diff --git a/tests/lib/maillogsentinel/test_report.py b/tests/lib/maillogsentinel/test_report.py index 89bdf1c..cc39d63 100644 --- a/tests/lib/maillogsentinel/test_report.py +++ b/tests/lib/maillogsentinel/test_report.py @@ -299,7 +299,7 @@ def test_send_report_success( assert len(attachments) == 1 assert attachments[0].get_filename() == mock_app_config.csv_filename mock_logger.info.assert_any_call( - f"Report sent from testuser@my.server.com to recipient@example.com" + "Report sent from testuser@my.server.com to recipient@example.com" ) @@ -376,7 +376,7 @@ def test_send_report_sender_override( sent_msg: EmailMessage = mock_smtp_instance.send_message.call_args[0][0] assert sent_msg["From"] == "override@sender.com" mock_logger.info.assert_any_call( - f"Report sent from override@sender.com to recipient@example.com" + "Report sent from override@sender.com to recipient@example.com" ) diff --git a/tests/lib/maillogsentinel/test_sql_exporter.py b/tests/lib/maillogsentinel/test_sql_exporter.py index 617129d..0a8d7ea 100644 --- a/tests/lib/maillogsentinel/test_sql_exporter.py +++ b/tests/lib/maillogsentinel/test_sql_exporter.py @@ -6,6 +6,7 @@ import json from pathlib import Path import datetime +from unittest.mock import patch from lib.maillogsentinel.sql_exporter import ( load_column_mapping, @@ -16,7 +17,6 @@ format_sql_value, generate_insert_statement, run_sql_export, - SQLExportError, CSVSchemaError, ) from lib.maillogsentinel.config import AppConfig # For mocking config @@ -66,9 +66,10 @@ def now(cls, tz=None): @pytest.fixture -def mock_logger(mocker): +def mock_logger(): """Fixture to mock the logger.""" - return mocker.patch("lib.maillogsentinel.sql_exporter.logger") + with patch("lib.maillogsentinel.sql_exporter.logger") as mock_log: + yield mock_log @pytest.fixture @@ -285,7 +286,6 @@ def test_generate_insert_statement(sample_column_mapping_content): # Adjust mapping for this test: "id" is auto-increment, so it shouldn't be in INSERT # "status" is ENUM - expected_sql = "INSERT INTO logs (server, event_time, ip, username, hostname, status) VALUES ('mail.example.com', '2023-01-01 12:00:00', '192.168.1.1', 'testuser', 'client.local', 'OK');" # Note: The order of columns in the output SQL depends on the iteration order of sample_column_mapping_content. # For robust testing, parse the generated SQL or compare sets of (column, value) pairs. # For simplicity here, we rely on dict iteration order (Python 3.7+). @@ -302,7 +302,6 @@ def test_generate_insert_statement(sample_column_mapping_content): # The columns will be: server, event_time, ip, username, hostname, status # Reconstruct the expected string based on the order in sample_column_mapping_content, skipping 'id' - cols_ordered = [k for k in sample_column_mapping_content if k != "id"] # This test is a bit fragile due to string matching. # A more robust test would parse the SQL. @@ -408,91 +407,87 @@ def test_run_sql_export_empty_csv( def test_run_sql_export_basic_flow( - mock_app_config, sample_column_mapping_content, mock_logger, mocker + mock_app_config, sample_column_mapping_content, mock_logger ): - csv_file = mock_app_config.working_dir / mock_app_config.csv_filename - offset_file = mock_app_config.state_dir / "sql_state.offset" - sql_output_dir = mock_app_config.working_dir / "sql" - - if offset_file.exists(): - offset_file.unlink() - - # Prepare CSV data - # Get header from mapping, skipping placeholder for auto-increment ID - csv_headers = [ - info["csv_column_name"] - for _, info in sample_column_mapping_content.items() - if info["csv_column_name"] != "csv_id_placeholder" - ] - - csv_content = ";".join(csv_headers) + "\n" - csv_content += "srv1;2023-01-01 10:00:00;1.1.1.1;user1;host1.com;OK\n" - csv_content += "srv2;2023-01-02 11:00:00;2.2.2.2;user2;host2.net;FAIL\n" - csv_file.write_text(csv_content) - - # Mock datetime.datetime class in the sql_exporter module - mocker.patch( - "lib.maillogsentinel.sql_exporter.datetime.datetime", MockFixedDatetime - ) - - # Set specific times for each run - # First run - MockFixedDatetime.set_now(datetime.datetime(2023, 1, 1, 10, 0, 0)) - assert run_sql_export(mock_app_config) - exported_files1 = list(sql_output_dir.glob("*.sql")) - assert len(exported_files1) == 1 - expected_filename1 = sql_output_dir / "20230101_1000_maillogsentinel_export.sql" - assert expected_filename1 in exported_files1 - sql_content1 = expected_filename1.read_text() - assert ( - "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv1', '2023-01-01 10:00:00', '1.1.1.1', 'user1', 'host1.com', 'OK');" - in sql_content1 - ) - assert ( - "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv2', '2023-01-02 11:00:00', '2.2.2.2', 'user2', 'host2.net', 'FAIL');" - in sql_content1 - ) - assert "BEGIN TRANSACTION;" in sql_content1 - assert "COMMIT;" in sql_content1 - - original_offset = len(csv_content.encode("utf-8")) - assert int(offset_file.read_text()) == original_offset - - # Second run (no new data) - MockFixedDatetime.set_now(datetime.datetime(2023, 1, 1, 10, 1, 0)) # Different time - assert run_sql_export(mock_app_config) - current_sql_files = list(sql_output_dir.glob("*.sql")) - assert len(current_sql_files) == 1 - assert expected_filename1 in current_sql_files - - # Add new data - new_line = "srv3;2023-01-03 12:00:00;3.3.3.3;user3;host3.org;OK\n" - with open(csv_file, "a") as f: - f.write(new_line) - - # Third run - MockFixedDatetime.set_now( - datetime.datetime(2023, 1, 1, 10, 2, 0) - ) # Different time again - assert run_sql_export(mock_app_config) - exported_files3 = list(sql_output_dir.glob("*.sql")) - assert len(exported_files3) == 2 - expected_filename3 = sql_output_dir / "20230101_1002_maillogsentinel_export.sql" - assert expected_filename3 in exported_files3 - sql_content3 = expected_filename3.read_text() - assert ( - "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv3', '2023-01-03 12:00:00', '3.3.3.3', 'user3', 'host3.org', 'OK');" - in sql_content3 - ) - assert "BEGIN TRANSACTION;" in sql_content3 - assert "COMMIT;" in sql_content3 - - assert int(offset_file.read_text()) == original_offset + len( - new_line.encode("utf-8") - ) - mock_logger.info.assert_any_call( - "sql_export: SQL export process finished. Processed: 1 lines. Exported: 1 records." - ) + with patch("lib.maillogsentinel.sql_exporter.datetime.datetime", MockFixedDatetime): + csv_file = mock_app_config.working_dir / mock_app_config.csv_filename + offset_file = mock_app_config.state_dir / "sql_state.offset" + sql_output_dir = mock_app_config.working_dir / "sql" + + if offset_file.exists(): + offset_file.unlink() + + # Prepare CSV data + # Get header from mapping, skipping placeholder for auto-increment ID + csv_headers = [ + info["csv_column_name"] + for _, info in sample_column_mapping_content.items() + if info["csv_column_name"] != "csv_id_placeholder" + ] + + csv_content = ";".join(csv_headers) + "\n" + csv_content += "srv1;2023-01-01 10:00:00;1.1.1.1;user1;host1.com;OK\n" + csv_content += "srv2;2023-01-02 11:00:00;2.2.2.2;user2;host2.net;FAIL\n" + csv_file.write_text(csv_content) + + # Set specific times for each run + # First run + MockFixedDatetime.set_now(datetime.datetime(2023, 1, 1, 10, 0, 0)) + assert run_sql_export(mock_app_config) + exported_files1 = list(sql_output_dir.glob("*.sql")) + assert len(exported_files1) == 1 + expected_filename1 = sql_output_dir / "20230101_1000_maillogsentinel_export.sql" + assert expected_filename1 in exported_files1 + sql_content1 = expected_filename1.read_text() + assert ( + "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv1', '2023-01-01 10:00:00', '1.1.1.1', 'user1', 'host1.com', 'OK');" + in sql_content1 + ) + assert ( + "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv2', '2023-01-02 11:00:00', '2.2.2.2', 'user2', 'host2.net', 'FAIL');" + in sql_content1 + ) + assert "BEGIN TRANSACTION;" in sql_content1 + assert "COMMIT;" in sql_content1 + + original_offset = len(csv_content.encode("utf-8")) + assert int(offset_file.read_text()) == original_offset + + # Second run (no new data) + MockFixedDatetime.set_now(datetime.datetime(2023, 1, 1, 10, 1, 0)) # Different time + assert run_sql_export(mock_app_config) + current_sql_files = list(sql_output_dir.glob("*.sql")) + assert len(current_sql_files) == 1 + assert expected_filename1 in current_sql_files + + # Add new data + new_line = "srv3;2023-01-03 12:00:00;3.3.3.3;user3;host3.org;OK\n" + with open(csv_file, "a") as f: + f.write(new_line) + + # Third run + MockFixedDatetime.set_now( + datetime.datetime(2023, 1, 1, 10, 2, 0) + ) # Different time again + assert run_sql_export(mock_app_config) + exported_files3 = list(sql_output_dir.glob("*.sql")) + assert len(exported_files3) == 2 + expected_filename3 = sql_output_dir / "20230101_1002_maillogsentinel_export.sql" + assert expected_filename3 in exported_files3 + sql_content3 = expected_filename3.read_text() + assert ( + "INSERT INTO test_log_events (server, event_time, ip, username, hostname, status) VALUES ('srv3', '2023-01-03 12:00:00', '3.3.3.3', 'user3', 'host3.org', 'OK');" + in sql_content3 + ) + assert "BEGIN TRANSACTION;" in sql_content3 + assert "COMMIT;" in sql_content3 + + assert int(offset_file.read_text()) == original_offset + len( + new_line.encode("utf-8") + ) + mock_logger.info.assert_any_call( + "sql_export: SQL export process finished. Processed: 1 lines. Exported: 1 records." + ) # (More tests for run_sql_export: mapping file issues, header validation on resume, etc.)