From dd17840fe69c6d8ab2c4e990310012a4eb975d5d Mon Sep 17 00:00:00 2001 From: Farbod Akvan Date: Tue, 14 Jul 2026 10:41:15 +0330 Subject: [PATCH] fix: guarantee complete Gmail scope and label coverage --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- CHANGELOG.md | 14 +++ CITATION.cff | 2 +- README.md | 12 ++ docs/CLI_REFERENCE.md | 19 ++- docs/GETTING_STARTED.md | 20 +++- docs/REPOSITORY_SETUP.md | 2 +- docs/TROUBLESHOOTING.md | 15 +++ pyproject.toml | 2 +- src/firexcore_mailvault/cli.py | 77 ++++++++++++ src/firexcore_mailvault/gmail_audit.py | 112 ++++++++++++++++++ .../protocols/imap/gateway.py | 22 +++- src/firexcore_mailvault/providers/gmail.py | 14 ++- src/firexcore_mailvault/repository.py | 14 +++ src/firexcore_mailvault/sync_engine.py | 42 ++++++- tests/test_gmail_audit.py | 108 +++++++++++++++++ tests/test_providers.py | 17 +++ tests/test_sync_engine.py | 3 + 18 files changed, 483 insertions(+), 14 deletions(-) create mode 100644 src/firexcore_mailvault/gmail_audit.py create mode 100644 tests/test_gmail_audit.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 95457fb..f55589e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -12,7 +12,7 @@ body: id: version attributes: label: MailVault version - placeholder: 2.0.4 + placeholder: 2.0.5 validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index c13ba14..d0e6c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to FireXCore MailVault are documented in this file. The project follows Semantic Versioning. +## 2.0.5 - 2026-07-14 + +### Fixed + +- Corrected Gmail full-scope discovery to use the standard IMAP `SEARCH ALL` criterion. +- Removed the invalid `X-GM-RAW "all"` behavior, which interpreted `all` as a Gmail text search and could silently omit older messages. +- Versioned scan selection keys so existing archives perform a safe full metadata rescan after upgrading instead of reusing the affected checkpoint. + +### Added + +- Added Gmail message-ID fetch support for remote label reconciliation. +- Added the `mailvault audit-labels` command to compare every IMAP-visible Gmail label with locally archived raw EML identities. +- Full-scope Gmail sync now runs the same remote label audit before reporting `complete`; missing remote messages produce `incomplete` and exit code 2. + ## 2.0.4 - 2026-07-13 ### Fixed diff --git a/CITATION.cff b/CITATION.cff index 7b00a00..799064c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,5 +8,5 @@ authors: repository-code: "https://github.com/FireXCore/mailvault" url: "https://github.com/FireXCore/mailvault" license: "Apache-2.0" -version: "2.0.4" +version: "2.0.5" date-released: "2026-07-13" diff --git a/README.md b/README.md index 4944244..b99fd36 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,17 @@ mailvault sync ` The App Password is requested through hidden terminal input. It is not written to configuration, SQLite, JSON, manifests, reports, or logs. +After a full Gmail sync, verify remote label coverage before treating the archive as final: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +Full-scope Gmail sync performs this reconciliation automatically before it can report `complete`. The standalone command can be rerun at any time and writes a JSON report under `reports/`. +

MailVault doctor output

@@ -200,6 +211,7 @@ See [Configuration](docs/CONFIGURATION.md). |---|---| | `mailvault doctor` | Validate TLS, authentication, server capabilities, provider profile, and mailbox discovery. | | `mailvault sync` | Discover metadata and archive complete raw messages with resumable state. | +| `mailvault audit-labels` | Compare every IMAP-visible Gmail label with locally archived raw EML identities. | | `mailvault stats` | Display message, occurrence, MIME-part, blob, and storage counts. | | `mailvault verify` | Recalculate raw-message and blob hashes. | | `mailvault export` | Regenerate portable JSONL and procurement source manifests. | diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 4a1f389..4237340 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -39,7 +39,7 @@ Important options: --query Provider query used with scope=query --include-spam Include Spam/Junk archive roots --include-trash Include Trash archive roots ---mailbox Repeatable generic-IMAP mailbox pattern +--mailbox Repeatable mailbox pattern; Gmail labels are supported --soft-cap Rolling 24-hour soft download cap --hard-cap Rolling 24-hour hard download cap --log-level DEBUG | INFO | WARNING | ERROR @@ -59,6 +59,23 @@ Incorrect: --auth ``` + +## `mailvault audit-labels` + +Performs a read-only Gmail reconciliation. It enumerates every IMAP-visible mailbox or label, retrieves `X-GM-MSGID` values, and compares them with locally archived raw EML identities. + +```text +--account, -a Gmail account present in the archive +--destination, -d Archive root +--host Gmail IMAP hostname; default imap.gmail.com +--port IMAP port; default 993 +--auth app-password | password +--tls-mode implicit | starttls +--timeout Socket timeout in seconds +``` + +The command writes a timestamped JSON report under `reports/` and exits with code `2` when any remote Gmail message lacks raw EML. Full-scope Gmail sync performs the same audit automatically before reporting `complete`. The audit never downloads raw message bodies and does not mutate the mailbox. + ## `mailvault stats` ```text diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 7bf3507..cf1dda3 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -82,6 +82,19 @@ mailvault sync ` The first phase discovers metadata. The second phase fetches pending raw messages. Stopping the process does not invalidate the archive; use the same destination and command to resume. +## Audit Gmail label coverage + +For Gmail, full-scope sync performs remote label reconciliation before reporting `complete`. You can also rerun the audit explicitly before final verification: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +A passing audit proves that every Gmail message visible through those labels has a corresponding archived raw EML identity. + ## Verify the archive ```powershell @@ -100,6 +113,7 @@ mailvault views --destination E:\MailVault 1. Run `doctor`. 2. Run a narrow date or query scope against a new destination when validating a new provider. 3. Inspect `reports`, `logs`, `metadata/messages`, and `manifests`. -4. Run `verify`. -5. Start the full `scope=all` archive. -6. Preserve the destination and resume rather than restarting from an empty directory. +4. Start the full `scope=all` archive. +5. For Gmail, run `audit-labels` and require a passing result. +6. Run `verify`. +7. Preserve the destination and resume rather than restarting from an empty directory. diff --git a/docs/REPOSITORY_SETUP.md b/docs/REPOSITORY_SETUP.md index 0d243c8..a018e8c 100644 --- a/docs/REPOSITORY_SETUP.md +++ b/docs/REPOSITORY_SETUP.md @@ -50,7 +50,7 @@ Enable: ## Releases -Create Git tags in the form `v2.0.4`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. +Create Git tags in the form `v2.0.5`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. PyPI publishing uses a separate trusted-publishing workflow. Configure the GitHub `pypi` environment and PyPI trusted publisher, then set the repository variable `PYPI_PUBLISH_ENABLED=true`. Manual dispatch remains available for controlled validation. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f519cf3..65238e4 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -71,3 +71,18 @@ Do not delete it. Run the same version and command against the same destination. ## Integrity verification fails Do not continue with downstream ingestion. Check missing paths, storage corruption, interrupted external copy operations, antivirus quarantine and filesystem errors. Restore canonical objects from backup, then rerun full verification. + +## Gmail archive created with version 2.0.4 or earlier is incomplete + +Versions through 2.0.4 used `X-GM-RAW "all"` for `scope=all`. Gmail interprets X-GM-RAW arguments using Gmail web-search syntax, so the bare word `all` is a text query rather than a universal match. An affected run can therefore report `complete` after archiving only messages matching that search term. + +Upgrade to 2.0.5 or newer and rerun the same full-scope command against the existing destination. Versioned scan keys force a safe metadata rescan without discarding already archived objects. Then run: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +Do not finalize exports or backups until label coverage passes and `mailvault verify --sample 1` succeeds. diff --git a/pyproject.toml b/pyproject.toml index cf4cdb1..826537c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "firexcore-mailvault" -version = "2.0.4" +version = "2.0.5" description = "Provider-neutral, read-only, evidence-preserving email archival for analytics, eDiscovery, and procurement intelligence." readme = "README.md" requires-python = ">=3.12" diff --git a/src/firexcore_mailvault/cli.py b/src/firexcore_mailvault/cli.py index 1486fa9..64b54ec 100644 --- a/src/firexcore_mailvault/cli.py +++ b/src/firexcore_mailvault/cli.py @@ -15,6 +15,7 @@ from firexcore_mailvault.config import MailVaultConfig, config_from_toml, parse_bytes from firexcore_mailvault.errors import MailVaultError from firexcore_mailvault.exporter import export_jsonl +from firexcore_mailvault.gmail_audit import audit_gmail_labels, write_gmail_label_audit from firexcore_mailvault.lock import RunLock from firexcore_mailvault.logging_setup import configure_logging from firexcore_mailvault.models import ArchiveScope, AuthKind, ProviderKind, SyncSummary, TlsMode @@ -197,6 +198,8 @@ def on_progress(event: str, payload: dict[str, object]) -> None: paths.manifests / "procurement_sources.jsonl" ) _print_sync_summary(summary) + if summary.status == "incomplete": + raise typer.Exit(2) except KeyboardInterrupt as exc: console.print( "[yellow]Sync interrupted safely. Run the same command to resume.[/yellow]" @@ -207,6 +210,80 @@ def on_progress(event: str, payload: dict[str, object]) -> None: raise typer.Exit(1) from exc +@app.command("audit-labels") +def audit_labels_command( + account: Annotated[str, typer.Option("--account", "-a", prompt=True)], + destination: Annotated[Path, typer.Option("--destination", "-d")], + host: Annotated[str, typer.Option("--host")] = "imap.gmail.com", + port: Annotated[int, typer.Option("--port")] = 993, + auth: Annotated[AuthKind, typer.Option("--auth")] = AuthKind.APP_PASSWORD, + tls_mode: Annotated[TlsMode, typer.Option("--tls-mode")] = TlsMode.IMPLICIT, + timeout: Annotated[int, typer.Option("--timeout")] = 90, +) -> None: + """Compare every IMAP-visible Gmail label with locally archived raw EML identities.""" + paths = build_archive_paths(destination) + configure_logging(paths.logs / "mailvault.jsonl", "INFO") + secret = _secret(auth) + + try: + with ( + ArchiveRepository(paths.database) as repository, + ImapGateway( + account, + secret, + host=host, + port=port, + tls_mode=tls_mode, + timeout_seconds=timeout, + client_contact="https://github.com/FireXCore/mailvault/issues", + ) as gateway, + ): + profile = resolve_provider(ProviderKind.GMAIL, gateway.capabilities) + profile.validate_capabilities(gateway.capabilities) + account_id = repository.find_account_id(account) + if account_id is None: + raise typer.BadParameter("Account does not exist in this archive.") + + report = audit_gmail_labels( + gateway, + repository, + account_id=account_id, + account=account, + ) + report_path = write_gmail_label_audit(report, paths.reports) + + table = Table(title="Gmail Label Coverage Audit") + table.add_column("Mailbox") + table.add_column("Remote", justify="right") + table.add_column("Raw", justify="right") + table.add_column("Missing", justify="right") + table.add_column("Result") + for item in report.labels: + table.add_row( + item.mailbox, + f"{item.remote_messages:,}", + f"{item.archived_raw_messages:,}", + f"{item.missing_raw_messages:,}", + "PASS" if item.passed else "FAIL", + ) + console.print(table) + console.print(f"Report: {report_path}") + + if not report.passed: + console.print( + f"[red]Coverage failed:[/red] {report.missing_raw_messages:,} " + "unique Gmail messages do not have archived raw EML." + ) + raise typer.Exit(2) + + console.print("[green]Coverage PASS:[/green] every IMAP-visible label is archived.") + except typer.Exit: + raise + except Exception as exc: + console.print(f"[red]Label audit failed:[/red] {sanitize_text(str(exc))}") + raise typer.Exit(1) from exc + + @app.command("stats") def stats_command( destination: Annotated[Path, typer.Option("--destination", "-d")], diff --git a/src/firexcore_mailvault/gmail_audit.py b/src/firexcore_mailvault/gmail_audit.py new file mode 100644 index 0000000..d31ac47 --- /dev/null +++ b/src/firexcore_mailvault/gmail_audit.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path + +from firexcore_mailvault.atomic import atomic_write_json +from firexcore_mailvault.models import MailboxInfo +from firexcore_mailvault.protocols.imap import ImapGatewayProtocol +from firexcore_mailvault.repository import ArchiveRepository + + +@dataclass(frozen=True, slots=True) +class GmailLabelCoverage: + mailbox: str + flags: tuple[str, ...] + remote_messages: int + archived_raw_messages: int + missing_raw_messages: int + missing_gmail_message_ids: tuple[str, ...] + + @property + def passed(self) -> bool: + return self.missing_raw_messages == 0 + + +@dataclass(frozen=True, slots=True) +class GmailLabelAuditReport: + account: str + generated_at: str + local_raw_message_ids: int + labels: tuple[GmailLabelCoverage, ...] + + @property + def passed(self) -> bool: + return all(item.passed for item in self.labels) + + @property + def missing_raw_messages(self) -> int: + return len( + {gmail_id for label in self.labels for gmail_id in label.missing_gmail_message_ids} + ) + + def as_dict(self) -> dict[str, object]: + return { + "account": self.account, + "generated_at": self.generated_at, + "local_raw_message_ids": self.local_raw_message_ids, + "passed": self.passed, + "missing_raw_messages": self.missing_raw_messages, + "labels": [ + { + **asdict(label), + "passed": label.passed, + } + for label in self.labels + ], + } + + +def audit_gmail_labels( + gateway: ImapGatewayProtocol, + repository: ArchiveRepository, + *, + account_id: int, + account: str, + batch_size: int = 500, +) -> GmailLabelAuditReport: + local_raw_ids = repository.gmail_raw_message_ids(account_id) + coverages: list[GmailLabelCoverage] = [] + + for mailbox in _selectable_mailboxes(gateway.list_mailboxes()): + gateway.select_readonly(mailbox.name) + uids = sorted(set(gateway.search_uids(("imap", "ALL")))) + remote_ids: set[str] = set() + + for index in range(0, len(uids), batch_size): + batch = uids[index : index + batch_size] + remote_ids.update(gateway.fetch_gmail_message_ids(batch).values()) + + missing = tuple(sorted(remote_ids - local_raw_ids)) + coverages.append( + GmailLabelCoverage( + mailbox=mailbox.name, + flags=mailbox.flags, + remote_messages=len(remote_ids), + archived_raw_messages=len(remote_ids & local_raw_ids), + missing_raw_messages=len(missing), + missing_gmail_message_ids=missing, + ) + ) + + return GmailLabelAuditReport( + account=account, + generated_at=datetime.now(UTC).isoformat(), + local_raw_message_ids=len(local_raw_ids), + labels=tuple(coverages), + ) + + +def write_gmail_label_audit(report: GmailLabelAuditReport, reports_dir: Path) -> Path: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + path = reports_dir / f"gmail-label-audit-{timestamp}.json" + atomic_write_json(path, report.as_dict()) + return path + + +def _selectable_mailboxes(mailboxes: list[MailboxInfo]) -> list[MailboxInfo]: + return sorted( + (mailbox for mailbox in mailboxes if mailbox.selectable), + key=lambda item: item.name.casefold(), + ) diff --git a/src/firexcore_mailvault/protocols/imap/gateway.py b/src/firexcore_mailvault/protocols/imap/gateway.py index 1dc3daf..2150e33 100644 --- a/src/firexcore_mailvault/protocols/imap/gateway.py +++ b/src/firexcore_mailvault/protocols/imap/gateway.py @@ -68,6 +68,8 @@ def fetch_metadata( def fetch_raw(self, uid: int) -> bytes: ... + def fetch_gmail_message_ids(self, uids: Sequence[int]) -> dict[int, str]: ... + def reconnect(self) -> None: ... @@ -198,8 +200,12 @@ def select_readonly(self, mailbox: str) -> SelectResult: def search_uids(self, description: object) -> list[int]: client = self._require_client() - if isinstance(description, tuple) and len(description) == 2 and description[0] == "gmail": - return [int(value) for value in client.gmail_search(str(description[1]))] + if isinstance(description, tuple) and len(description) == 2: + strategy, value = description + if strategy == "gmail": + return [int(item) for item in client.gmail_search(str(value))] + if strategy == "imap": + return [int(item) for item in client.search([str(value)])] criteria = cast(Sequence[str | bytes], description) return [int(value) for value in client.search(criteria)] @@ -250,6 +256,18 @@ def fetch_raw(self, uid: int) -> bytes: raise RuntimeError(f"Server returned no raw message bytes for UID {uid}.") return raw + def fetch_gmail_message_ids(self, uids: Sequence[int]) -> dict[int, str]: + if not uids: + return {} + client = self._require_client() + response = client.fetch(list(uids), ["X-GM-MSGID"]) + result: dict[int, str] = {} + for uid, data in response.items(): + value = _find_text(data, b"X-GM-MSGID") + if value is not None: + result[int(uid)] = value + return result + def noop(self) -> None: self._require_client().noop() diff --git a/src/firexcore_mailvault/providers/gmail.py b/src/firexcore_mailvault/providers/gmail.py index 84e5107..567b1fd 100644 --- a/src/firexcore_mailvault/providers/gmail.py +++ b/src/firexcore_mailvault/providers/gmail.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fnmatch from collections.abc import Sequence from firexcore_mailvault.errors import ImapCapabilityError @@ -29,6 +30,14 @@ def choose_mailboxes( include_trash: bool, patterns: Sequence[str], ) -> list[MailboxInfo]: + if patterns: + return [ + mailbox + for mailbox in mailboxes + if mailbox.selectable + and any(fnmatch.fnmatchcase(mailbox.name, pattern) for pattern in patterns) + ] + all_mail = next((item for item in mailboxes if item.has_flag("\\All")), None) if all_mail is None: raise ImapCapabilityError( @@ -56,7 +65,10 @@ def metadata_fetch_items(self, capabilities: ServerCapabilities) -> list[str]: def search_criteria(self, scope: ArchiveScope, query: str | None) -> object: if scope is ArchiveScope.ALL: - return ("gmail", "all") + # Use the standard IMAP ALL search criterion. X-GM-RAW delegates to + # Gmail web-search syntax, where the bare word "all" is a text query + # rather than a universal match. + return ("imap", "ALL") if scope is ArchiveScope.HAS_ATTACHMENTS: return ("gmail", "has:attachment") if not query: diff --git a/src/firexcore_mailvault/repository.py b/src/firexcore_mailvault/repository.py index ae4fd9b..6388fb8 100644 --- a/src/firexcore_mailvault/repository.py +++ b/src/firexcore_mailvault/repository.py @@ -1103,6 +1103,20 @@ def message_document(self, message_id: int) -> dict[str, Any]: ], } + def gmail_raw_message_ids(self, account_id: int) -> set[str]: + rows = self.connection.execute( + """ + SELECT DISTINCT identity.value + FROM message_identities AS identity + JOIN messages AS message ON message.id = identity.message_id + WHERE identity.account_id = ? + AND identity.namespace = 'gmail-x-gm-msgid' + AND message.raw_path IS NOT NULL + """, + (account_id,), + ).fetchall() + return {str(row["value"]) for row in rows} + def stats(self, account_id: int | None = None) -> dict[str, int]: result: dict[str, int] = {} if account_id is None: diff --git a/src/firexcore_mailvault/sync_engine.py b/src/firexcore_mailvault/sync_engine.py index 0bb516b..8cf446d 100644 --- a/src/firexcore_mailvault/sync_engine.py +++ b/src/firexcore_mailvault/sync_engine.py @@ -10,9 +10,10 @@ from firexcore_mailvault.atomic import atomic_write_json from firexcore_mailvault.config import MailVaultConfig from firexcore_mailvault.errors import AuthenticationError, BandwidthLimitReached +from firexcore_mailvault.gmail_audit import audit_gmail_labels, write_gmail_label_audit from firexcore_mailvault.metadata import parse_header_bytes from firexcore_mailvault.mime_parser import parse_message -from firexcore_mailvault.models import ArchivePaths, SyncSummary +from firexcore_mailvault.models import ArchivePaths, ArchiveScope, ProviderKind, SyncSummary from firexcore_mailvault.protocols.imap import ImapGatewayProtocol, effective_batch_size from firexcore_mailvault.providers.base import ProviderProfile from firexcore_mailvault.repository import ArchiveRepository @@ -20,6 +21,7 @@ from firexcore_mailvault.throttling import BandwidthThrottle, ThrottleSettings LOGGER = logging.getLogger(__name__) +_SEARCH_PLAN_VERSION = 2 ProgressCallback = Callable[[str, dict[str, object]], None] T = TypeVar("T") @@ -64,7 +66,7 @@ def run(self) -> SyncSummary: try: self._discover_metadata(context, summary) self._archive_pending(context, summary) - summary.status = "complete" + self._finalize_coverage(context, summary) except BandwidthLimitReached as exc: summary.status = "paused" summary.stop_reason = str(exc) @@ -89,6 +91,38 @@ def run(self) -> SyncSummary: ) return summary + def _finalize_coverage(self, context: SyncContext, summary: SyncSummary) -> None: + if self.profile.kind is not ProviderKind.GMAIL or self.config.scope is not ArchiveScope.ALL: + summary.status = "complete" + return + + report = audit_gmail_labels( + self.gateway, + self.repository, + account_id=context.account_id, + account=self.config.account, + ) + report_path = write_gmail_label_audit(report, self.paths.reports) + self.progress( + "label_audit_complete", + { + "passed": report.passed, + "missing_raw_messages": report.missing_raw_messages, + "report": str(report_path), + }, + ) + + if report.passed: + summary.status = "complete" + return + + summary.status = "incomplete" + summary.stop_reason = ( + "Gmail label coverage failed: " + f"{report.missing_raw_messages} unique remote messages lack raw EML. " + f"See {report_path}." + ) + def _discover_metadata(self, context: SyncContext, summary: SyncSummary) -> None: available = self.gateway.list_mailboxes() selected = self.profile.choose_mailboxes( @@ -200,7 +234,9 @@ def _discover_metadata(self, context: SyncContext, summary: SyncSummary) -> None def _selection_key(self) -> str: query = self.config.query or "" - return f"{self.profile.kind.value}:{self.config.scope.value}:{query}" + return ( + f"v{_SEARCH_PLAN_VERSION}:{self.profile.kind.value}:{self.config.scope.value}:{query}" + ) def _archive_pending(self, context: SyncContext, summary: SyncSummary) -> None: throttle = BandwidthThrottle( diff --git a/tests/test_gmail_audit.py b/tests/test_gmail_audit.py new file mode 100644 index 0000000..82c3318 --- /dev/null +++ b/tests/test_gmail_audit.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from pathlib import Path + +from firexcore_mailvault.config import MailVaultConfig +from firexcore_mailvault.gmail_audit import audit_gmail_labels +from firexcore_mailvault.models import MailboxInfo +from firexcore_mailvault.paths import build_archive_paths +from firexcore_mailvault.protocols.imap.capabilities import parse_capabilities +from firexcore_mailvault.protocols.imap.gateway import SelectResult +from firexcore_mailvault.repository import ArchiveRepository + + +class AuditGateway: + def __init__(self) -> None: + self.capabilities = parse_capabilities([b"IMAP4rev1", b"X-GM-EXT-1"]) + self.selected_mailbox: str | None = None + self._mailbox_uids = { + "[Gmail]/All Mail": [1, 2], + "Suppliers": [2, 3], + } + self._gmail_ids = { + 1: "1001", + 2: "1002", + 3: "1003", + } + + def list_mailboxes(self) -> list[MailboxInfo]: + return [ + MailboxInfo("[Gmail]/All Mail", "/", ("\\All",)), + MailboxInfo("Suppliers", "/", ("\\HasNoChildren",)), + ] + + def select_readonly(self, mailbox: str) -> SelectResult: + self.selected_mailbox = mailbox + return SelectResult(1, 3, None, None, len(self._mailbox_uids[mailbox])) + + def search_uids(self, description: object) -> list[int]: + assert description == ("imap", "ALL") + assert self.selected_mailbox is not None + return self._mailbox_uids[self.selected_mailbox] + + def fetch_metadata(self, uids: list[int], extra_items: list[str]) -> list[object]: + raise AssertionError("metadata fetch is not used by the label audit") + + def fetch_raw(self, uid: int) -> bytes: + raise AssertionError("raw fetch is not used by the label audit") + + def fetch_gmail_message_ids(self, uids: list[int]) -> dict[int, str]: + return {uid: self._gmail_ids[uid] for uid in uids} + + def reconnect(self) -> None: + return None + + +def test_gmail_label_audit_reports_remote_messages_without_raw(tmp_path: Path) -> None: + config = MailVaultConfig( + account="buyer@gmail.com", + destination=tmp_path / "archive", + host="imap.gmail.com", + ) + paths = build_archive_paths(config.destination) + + with ArchiveRepository(paths.database) as repository: + account_id = repository.get_or_create_account(config, config.provider) + now = "2026-01-01T00:00:00+00:00" + with repository.transaction() as connection: + for message_id, gmail_id in ((1, "1001"), (2, "1002")): + connection.execute( + """ + INSERT INTO messages( + id, archive_id, account_id, raw_path, raw_sha256, + raw_size_bytes, created_at, updated_at + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + message_id, + f"archive-{message_id}", + account_id, + f"objects/raw/{gmail_id}", + gmail_id.zfill(64), + 10, + now, + now, + ), + ) + connection.execute( + """ + INSERT INTO message_identities( + account_id, message_id, namespace, value, created_at + ) VALUES(?, ?, 'gmail-x-gm-msgid', ?, ?) + """, + (account_id, message_id, gmail_id, now), + ) + + report = audit_gmail_labels( + AuditGateway(), + repository, + account_id=account_id, + account=config.account, + ) + + assert report.passed is False + assert report.missing_raw_messages == 1 + suppliers = next(item for item in report.labels if item.mailbox == "Suppliers") + assert suppliers.remote_messages == 2 + assert suppliers.archived_raw_messages == 1 + assert suppliers.missing_gmail_message_ids == ("1003",) diff --git a/tests/test_providers.py b/tests/test_providers.py index 570610f..3a93816 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -27,6 +27,7 @@ def test_auto_detects_gmail_and_uses_all_mail() -> None: patterns=(), ) assert [mailbox.name for mailbox in chosen] == ["[Gmail]/All Mail"] + assert profile.search_criteria(ArchiveScope.ALL, None) == ("imap", "ALL") assert profile.search_criteria(ArchiveScope.HAS_ATTACHMENTS, None) == ( "gmail", "has:attachment", @@ -46,3 +47,19 @@ def test_gmail_stable_identity() -> None: profile = GmailImapProfile() assert profile.stable_message_identity(record) == ("gmail-x-gm-msgid", "123") assert profile.stable_thread_identity(record) == ("gmail-x-gm-thrid", "456") + + +def test_gmail_can_select_explicit_label_patterns() -> None: + profile = GmailImapProfile() + chosen = profile.choose_mailboxes( + [ + MailboxInfo("[Gmail]/All Mail", "/", ("\\All",)), + MailboxInfo("NIPHON", "/", ("\\HasNoChildren",)), + MailboxInfo("Mr.Ayobi", "/", ("\\HasNoChildren",)), + ], + include_spam=False, + include_trash=False, + patterns=("NIP*",), + ) + + assert [mailbox.name for mailbox in chosen] == ["NIPHON"] diff --git a/tests/test_sync_engine.py b/tests/test_sync_engine.py index b8cb6c6..7122f20 100644 --- a/tests/test_sync_engine.py +++ b/tests/test_sync_engine.py @@ -56,6 +56,9 @@ def fetch_metadata(self, uids: list[int], extra_items: list[str]) -> list[Metada def fetch_raw(self, uid: int) -> bytes: return self.raw + def fetch_gmail_message_ids(self, uids: list[int]) -> dict[int, str]: + return {uid: str(uid) for uid in uids} + def reconnect(self) -> None: return None