From dd17840fe69c6d8ab2c4e990310012a4eb975d5d Mon Sep 17 00:00:00 2001 From: Farbod Akvan Date: Tue, 14 Jul 2026 10:41:15 +0330 Subject: [PATCH 1/2] fix: guarantee complete Gmail scope and label coverage --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- CHANGELOG.md | 14 +++ CITATION.cff | 2 +- README.md | 12 ++ docs/CLI_REFERENCE.md | 19 ++- docs/GETTING_STARTED.md | 20 +++- docs/REPOSITORY_SETUP.md | 2 +- docs/TROUBLESHOOTING.md | 15 +++ pyproject.toml | 2 +- src/firexcore_mailvault/cli.py | 77 ++++++++++++ src/firexcore_mailvault/gmail_audit.py | 112 ++++++++++++++++++ .../protocols/imap/gateway.py | 22 +++- src/firexcore_mailvault/providers/gmail.py | 14 ++- src/firexcore_mailvault/repository.py | 14 +++ src/firexcore_mailvault/sync_engine.py | 42 ++++++- tests/test_gmail_audit.py | 108 +++++++++++++++++ tests/test_providers.py | 17 +++ tests/test_sync_engine.py | 3 + 18 files changed, 483 insertions(+), 14 deletions(-) create mode 100644 src/firexcore_mailvault/gmail_audit.py create mode 100644 tests/test_gmail_audit.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 95457fb..f55589e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -12,7 +12,7 @@ body: id: version attributes: label: MailVault version - placeholder: 2.0.4 + placeholder: 2.0.5 validations: required: true diff --git a/CHANGELOG.md b/CHANGELOG.md index c13ba14..d0e6c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to FireXCore MailVault are documented in this file. The project follows Semantic Versioning. +## 2.0.5 - 2026-07-14 + +### Fixed + +- Corrected Gmail full-scope discovery to use the standard IMAP `SEARCH ALL` criterion. +- Removed the invalid `X-GM-RAW "all"` behavior, which interpreted `all` as a Gmail text search and could silently omit older messages. +- Versioned scan selection keys so existing archives perform a safe full metadata rescan after upgrading instead of reusing the affected checkpoint. + +### Added + +- Added Gmail message-ID fetch support for remote label reconciliation. +- Added the `mailvault audit-labels` command to compare every IMAP-visible Gmail label with locally archived raw EML identities. +- Full-scope Gmail sync now runs the same remote label audit before reporting `complete`; missing remote messages produce `incomplete` and exit code 2. + ## 2.0.4 - 2026-07-13 ### Fixed diff --git a/CITATION.cff b/CITATION.cff index 7b00a00..799064c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,5 +8,5 @@ authors: repository-code: "https://github.com/FireXCore/mailvault" url: "https://github.com/FireXCore/mailvault" license: "Apache-2.0" -version: "2.0.4" +version: "2.0.5" date-released: "2026-07-13" diff --git a/README.md b/README.md index 4944244..b99fd36 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,17 @@ mailvault sync ` The App Password is requested through hidden terminal input. It is not written to configuration, SQLite, JSON, manifests, reports, or logs. +After a full Gmail sync, verify remote label coverage before treating the archive as final: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +Full-scope Gmail sync performs this reconciliation automatically before it can report `complete`. The standalone command can be rerun at any time and writes a JSON report under `reports/`. +

MailVault doctor output

@@ -200,6 +211,7 @@ See [Configuration](docs/CONFIGURATION.md). |---|---| | `mailvault doctor` | Validate TLS, authentication, server capabilities, provider profile, and mailbox discovery. | | `mailvault sync` | Discover metadata and archive complete raw messages with resumable state. | +| `mailvault audit-labels` | Compare every IMAP-visible Gmail label with locally archived raw EML identities. | | `mailvault stats` | Display message, occurrence, MIME-part, blob, and storage counts. | | `mailvault verify` | Recalculate raw-message and blob hashes. | | `mailvault export` | Regenerate portable JSONL and procurement source manifests. | diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 4a1f389..4237340 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -39,7 +39,7 @@ Important options: --query Provider query used with scope=query --include-spam Include Spam/Junk archive roots --include-trash Include Trash archive roots ---mailbox Repeatable generic-IMAP mailbox pattern +--mailbox Repeatable mailbox pattern; Gmail labels are supported --soft-cap Rolling 24-hour soft download cap --hard-cap Rolling 24-hour hard download cap --log-level DEBUG | INFO | WARNING | ERROR @@ -59,6 +59,23 @@ Incorrect: --auth ``` + +## `mailvault audit-labels` + +Performs a read-only Gmail reconciliation. It enumerates every IMAP-visible mailbox or label, retrieves `X-GM-MSGID` values, and compares them with locally archived raw EML identities. + +```text +--account, -a Gmail account present in the archive +--destination, -d Archive root +--host Gmail IMAP hostname; default imap.gmail.com +--port IMAP port; default 993 +--auth app-password | password +--tls-mode implicit | starttls +--timeout Socket timeout in seconds +``` + +The command writes a timestamped JSON report under `reports/` and exits with code `2` when any remote Gmail message lacks raw EML. Full-scope Gmail sync performs the same audit automatically before reporting `complete`. The audit never downloads raw message bodies and does not mutate the mailbox. + ## `mailvault stats` ```text diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 7bf3507..cf1dda3 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -82,6 +82,19 @@ mailvault sync ` The first phase discovers metadata. The second phase fetches pending raw messages. Stopping the process does not invalidate the archive; use the same destination and command to resume. +## Audit Gmail label coverage + +For Gmail, full-scope sync performs remote label reconciliation before reporting `complete`. You can also rerun the audit explicitly before final verification: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +A passing audit proves that every Gmail message visible through those labels has a corresponding archived raw EML identity. + ## Verify the archive ```powershell @@ -100,6 +113,7 @@ mailvault views --destination E:\MailVault 1. Run `doctor`. 2. Run a narrow date or query scope against a new destination when validating a new provider. 3. Inspect `reports`, `logs`, `metadata/messages`, and `manifests`. -4. Run `verify`. -5. Start the full `scope=all` archive. -6. Preserve the destination and resume rather than restarting from an empty directory. +4. Start the full `scope=all` archive. +5. For Gmail, run `audit-labels` and require a passing result. +6. Run `verify`. +7. Preserve the destination and resume rather than restarting from an empty directory. diff --git a/docs/REPOSITORY_SETUP.md b/docs/REPOSITORY_SETUP.md index 0d243c8..a018e8c 100644 --- a/docs/REPOSITORY_SETUP.md +++ b/docs/REPOSITORY_SETUP.md @@ -50,7 +50,7 @@ Enable: ## Releases -Create Git tags in the form `v2.0.4`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. +Create Git tags in the form `v2.0.5`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. PyPI publishing uses a separate trusted-publishing workflow. Configure the GitHub `pypi` environment and PyPI trusted publisher, then set the repository variable `PYPI_PUBLISH_ENABLED=true`. Manual dispatch remains available for controlled validation. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f519cf3..65238e4 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -71,3 +71,18 @@ Do not delete it. Run the same version and command against the same destination. ## Integrity verification fails Do not continue with downstream ingestion. Check missing paths, storage corruption, interrupted external copy operations, antivirus quarantine and filesystem errors. Restore canonical objects from backup, then rerun full verification. + +## Gmail archive created with version 2.0.4 or earlier is incomplete + +Versions through 2.0.4 used `X-GM-RAW "all"` for `scope=all`. Gmail interprets X-GM-RAW arguments using Gmail web-search syntax, so the bare word `all` is a text query rather than a universal match. An affected run can therefore report `complete` after archiving only messages matching that search term. + +Upgrade to 2.0.5 or newer and rerun the same full-scope command against the existing destination. Versioned scan keys force a safe metadata rescan without discarding already archived objects. Then run: + +```powershell +mailvault audit-labels ` + --account user@gmail.com ` + --host imap.gmail.com ` + --destination E:\MailVault +``` + +Do not finalize exports or backups until label coverage passes and `mailvault verify --sample 1` succeeds. diff --git a/pyproject.toml b/pyproject.toml index cf4cdb1..826537c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "firexcore-mailvault" -version = "2.0.4" +version = "2.0.5" description = "Provider-neutral, read-only, evidence-preserving email archival for analytics, eDiscovery, and procurement intelligence." readme = "README.md" requires-python = ">=3.12" diff --git a/src/firexcore_mailvault/cli.py b/src/firexcore_mailvault/cli.py index 1486fa9..64b54ec 100644 --- a/src/firexcore_mailvault/cli.py +++ b/src/firexcore_mailvault/cli.py @@ -15,6 +15,7 @@ from firexcore_mailvault.config import MailVaultConfig, config_from_toml, parse_bytes from firexcore_mailvault.errors import MailVaultError from firexcore_mailvault.exporter import export_jsonl +from firexcore_mailvault.gmail_audit import audit_gmail_labels, write_gmail_label_audit from firexcore_mailvault.lock import RunLock from firexcore_mailvault.logging_setup import configure_logging from firexcore_mailvault.models import ArchiveScope, AuthKind, ProviderKind, SyncSummary, TlsMode @@ -197,6 +198,8 @@ def on_progress(event: str, payload: dict[str, object]) -> None: paths.manifests / "procurement_sources.jsonl" ) _print_sync_summary(summary) + if summary.status == "incomplete": + raise typer.Exit(2) except KeyboardInterrupt as exc: console.print( "[yellow]Sync interrupted safely. Run the same command to resume.[/yellow]" @@ -207,6 +210,80 @@ def on_progress(event: str, payload: dict[str, object]) -> None: raise typer.Exit(1) from exc +@app.command("audit-labels") +def audit_labels_command( + account: Annotated[str, typer.Option("--account", "-a", prompt=True)], + destination: Annotated[Path, typer.Option("--destination", "-d")], + host: Annotated[str, typer.Option("--host")] = "imap.gmail.com", + port: Annotated[int, typer.Option("--port")] = 993, + auth: Annotated[AuthKind, typer.Option("--auth")] = AuthKind.APP_PASSWORD, + tls_mode: Annotated[TlsMode, typer.Option("--tls-mode")] = TlsMode.IMPLICIT, + timeout: Annotated[int, typer.Option("--timeout")] = 90, +) -> None: + """Compare every IMAP-visible Gmail label with locally archived raw EML identities.""" + paths = build_archive_paths(destination) + configure_logging(paths.logs / "mailvault.jsonl", "INFO") + secret = _secret(auth) + + try: + with ( + ArchiveRepository(paths.database) as repository, + ImapGateway( + account, + secret, + host=host, + port=port, + tls_mode=tls_mode, + timeout_seconds=timeout, + client_contact="https://github.com/FireXCore/mailvault/issues", + ) as gateway, + ): + profile = resolve_provider(ProviderKind.GMAIL, gateway.capabilities) + profile.validate_capabilities(gateway.capabilities) + account_id = repository.find_account_id(account) + if account_id is None: + raise typer.BadParameter("Account does not exist in this archive.") + + report = audit_gmail_labels( + gateway, + repository, + account_id=account_id, + account=account, + ) + report_path = write_gmail_label_audit(report, paths.reports) + + table = Table(title="Gmail Label Coverage Audit") + table.add_column("Mailbox") + table.add_column("Remote", justify="right") + table.add_column("Raw", justify="right") + table.add_column("Missing", justify="right") + table.add_column("Result") + for item in report.labels: + table.add_row( + item.mailbox, + f"{item.remote_messages:,}", + f"{item.archived_raw_messages:,}", + f"{item.missing_raw_messages:,}", + "PASS" if item.passed else "FAIL", + ) + console.print(table) + console.print(f"Report: {report_path}") + + if not report.passed: + console.print( + f"[red]Coverage failed:[/red] {report.missing_raw_messages:,} " + "unique Gmail messages do not have archived raw EML." + ) + raise typer.Exit(2) + + console.print("[green]Coverage PASS:[/green] every IMAP-visible label is archived.") + except typer.Exit: + raise + except Exception as exc: + console.print(f"[red]Label audit failed:[/red] {sanitize_text(str(exc))}") + raise typer.Exit(1) from exc + + @app.command("stats") def stats_command( destination: Annotated[Path, typer.Option("--destination", "-d")], diff --git a/src/firexcore_mailvault/gmail_audit.py b/src/firexcore_mailvault/gmail_audit.py new file mode 100644 index 0000000..d31ac47 --- /dev/null +++ b/src/firexcore_mailvault/gmail_audit.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path + +from firexcore_mailvault.atomic import atomic_write_json +from firexcore_mailvault.models import MailboxInfo +from firexcore_mailvault.protocols.imap import ImapGatewayProtocol +from firexcore_mailvault.repository import ArchiveRepository + + +@dataclass(frozen=True, slots=True) +class GmailLabelCoverage: + mailbox: str + flags: tuple[str, ...] + remote_messages: int + archived_raw_messages: int + missing_raw_messages: int + missing_gmail_message_ids: tuple[str, ...] + + @property + def passed(self) -> bool: + return self.missing_raw_messages == 0 + + +@dataclass(frozen=True, slots=True) +class GmailLabelAuditReport: + account: str + generated_at: str + local_raw_message_ids: int + labels: tuple[GmailLabelCoverage, ...] + + @property + def passed(self) -> bool: + return all(item.passed for item in self.labels) + + @property + def missing_raw_messages(self) -> int: + return len( + {gmail_id for label in self.labels for gmail_id in label.missing_gmail_message_ids} + ) + + def as_dict(self) -> dict[str, object]: + return { + "account": self.account, + "generated_at": self.generated_at, + "local_raw_message_ids": self.local_raw_message_ids, + "passed": self.passed, + "missing_raw_messages": self.missing_raw_messages, + "labels": [ + { + **asdict(label), + "passed": label.passed, + } + for label in self.labels + ], + } + + +def audit_gmail_labels( + gateway: ImapGatewayProtocol, + repository: ArchiveRepository, + *, + account_id: int, + account: str, + batch_size: int = 500, +) -> GmailLabelAuditReport: + local_raw_ids = repository.gmail_raw_message_ids(account_id) + coverages: list[GmailLabelCoverage] = [] + + for mailbox in _selectable_mailboxes(gateway.list_mailboxes()): + gateway.select_readonly(mailbox.name) + uids = sorted(set(gateway.search_uids(("imap", "ALL")))) + remote_ids: set[str] = set() + + for index in range(0, len(uids), batch_size): + batch = uids[index : index + batch_size] + remote_ids.update(gateway.fetch_gmail_message_ids(batch).values()) + + missing = tuple(sorted(remote_ids - local_raw_ids)) + coverages.append( + GmailLabelCoverage( + mailbox=mailbox.name, + flags=mailbox.flags, + remote_messages=len(remote_ids), + archived_raw_messages=len(remote_ids & local_raw_ids), + missing_raw_messages=len(missing), + missing_gmail_message_ids=missing, + ) + ) + + return GmailLabelAuditReport( + account=account, + generated_at=datetime.now(UTC).isoformat(), + local_raw_message_ids=len(local_raw_ids), + labels=tuple(coverages), + ) + + +def write_gmail_label_audit(report: GmailLabelAuditReport, reports_dir: Path) -> Path: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + path = reports_dir / f"gmail-label-audit-{timestamp}.json" + atomic_write_json(path, report.as_dict()) + return path + + +def _selectable_mailboxes(mailboxes: list[MailboxInfo]) -> list[MailboxInfo]: + return sorted( + (mailbox for mailbox in mailboxes if mailbox.selectable), + key=lambda item: item.name.casefold(), + ) diff --git a/src/firexcore_mailvault/protocols/imap/gateway.py b/src/firexcore_mailvault/protocols/imap/gateway.py index 1dc3daf..2150e33 100644 --- a/src/firexcore_mailvault/protocols/imap/gateway.py +++ b/src/firexcore_mailvault/protocols/imap/gateway.py @@ -68,6 +68,8 @@ def fetch_metadata( def fetch_raw(self, uid: int) -> bytes: ... + def fetch_gmail_message_ids(self, uids: Sequence[int]) -> dict[int, str]: ... + def reconnect(self) -> None: ... @@ -198,8 +200,12 @@ def select_readonly(self, mailbox: str) -> SelectResult: def search_uids(self, description: object) -> list[int]: client = self._require_client() - if isinstance(description, tuple) and len(description) == 2 and description[0] == "gmail": - return [int(value) for value in client.gmail_search(str(description[1]))] + if isinstance(description, tuple) and len(description) == 2: + strategy, value = description + if strategy == "gmail": + return [int(item) for item in client.gmail_search(str(value))] + if strategy == "imap": + return [int(item) for item in client.search([str(value)])] criteria = cast(Sequence[str | bytes], description) return [int(value) for value in client.search(criteria)] @@ -250,6 +256,18 @@ def fetch_raw(self, uid: int) -> bytes: raise RuntimeError(f"Server returned no raw message bytes for UID {uid}.") return raw + def fetch_gmail_message_ids(self, uids: Sequence[int]) -> dict[int, str]: + if not uids: + return {} + client = self._require_client() + response = client.fetch(list(uids), ["X-GM-MSGID"]) + result: dict[int, str] = {} + for uid, data in response.items(): + value = _find_text(data, b"X-GM-MSGID") + if value is not None: + result[int(uid)] = value + return result + def noop(self) -> None: self._require_client().noop() diff --git a/src/firexcore_mailvault/providers/gmail.py b/src/firexcore_mailvault/providers/gmail.py index 84e5107..567b1fd 100644 --- a/src/firexcore_mailvault/providers/gmail.py +++ b/src/firexcore_mailvault/providers/gmail.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fnmatch from collections.abc import Sequence from firexcore_mailvault.errors import ImapCapabilityError @@ -29,6 +30,14 @@ def choose_mailboxes( include_trash: bool, patterns: Sequence[str], ) -> list[MailboxInfo]: + if patterns: + return [ + mailbox + for mailbox in mailboxes + if mailbox.selectable + and any(fnmatch.fnmatchcase(mailbox.name, pattern) for pattern in patterns) + ] + all_mail = next((item for item in mailboxes if item.has_flag("\\All")), None) if all_mail is None: raise ImapCapabilityError( @@ -56,7 +65,10 @@ def metadata_fetch_items(self, capabilities: ServerCapabilities) -> list[str]: def search_criteria(self, scope: ArchiveScope, query: str | None) -> object: if scope is ArchiveScope.ALL: - return ("gmail", "all") + # Use the standard IMAP ALL search criterion. X-GM-RAW delegates to + # Gmail web-search syntax, where the bare word "all" is a text query + # rather than a universal match. + return ("imap", "ALL") if scope is ArchiveScope.HAS_ATTACHMENTS: return ("gmail", "has:attachment") if not query: diff --git a/src/firexcore_mailvault/repository.py b/src/firexcore_mailvault/repository.py index ae4fd9b..6388fb8 100644 --- a/src/firexcore_mailvault/repository.py +++ b/src/firexcore_mailvault/repository.py @@ -1103,6 +1103,20 @@ def message_document(self, message_id: int) -> dict[str, Any]: ], } + def gmail_raw_message_ids(self, account_id: int) -> set[str]: + rows = self.connection.execute( + """ + SELECT DISTINCT identity.value + FROM message_identities AS identity + JOIN messages AS message ON message.id = identity.message_id + WHERE identity.account_id = ? + AND identity.namespace = 'gmail-x-gm-msgid' + AND message.raw_path IS NOT NULL + """, + (account_id,), + ).fetchall() + return {str(row["value"]) for row in rows} + def stats(self, account_id: int | None = None) -> dict[str, int]: result: dict[str, int] = {} if account_id is None: diff --git a/src/firexcore_mailvault/sync_engine.py b/src/firexcore_mailvault/sync_engine.py index 0bb516b..8cf446d 100644 --- a/src/firexcore_mailvault/sync_engine.py +++ b/src/firexcore_mailvault/sync_engine.py @@ -10,9 +10,10 @@ from firexcore_mailvault.atomic import atomic_write_json from firexcore_mailvault.config import MailVaultConfig from firexcore_mailvault.errors import AuthenticationError, BandwidthLimitReached +from firexcore_mailvault.gmail_audit import audit_gmail_labels, write_gmail_label_audit from firexcore_mailvault.metadata import parse_header_bytes from firexcore_mailvault.mime_parser import parse_message -from firexcore_mailvault.models import ArchivePaths, SyncSummary +from firexcore_mailvault.models import ArchivePaths, ArchiveScope, ProviderKind, SyncSummary from firexcore_mailvault.protocols.imap import ImapGatewayProtocol, effective_batch_size from firexcore_mailvault.providers.base import ProviderProfile from firexcore_mailvault.repository import ArchiveRepository @@ -20,6 +21,7 @@ from firexcore_mailvault.throttling import BandwidthThrottle, ThrottleSettings LOGGER = logging.getLogger(__name__) +_SEARCH_PLAN_VERSION = 2 ProgressCallback = Callable[[str, dict[str, object]], None] T = TypeVar("T") @@ -64,7 +66,7 @@ def run(self) -> SyncSummary: try: self._discover_metadata(context, summary) self._archive_pending(context, summary) - summary.status = "complete" + self._finalize_coverage(context, summary) except BandwidthLimitReached as exc: summary.status = "paused" summary.stop_reason = str(exc) @@ -89,6 +91,38 @@ def run(self) -> SyncSummary: ) return summary + def _finalize_coverage(self, context: SyncContext, summary: SyncSummary) -> None: + if self.profile.kind is not ProviderKind.GMAIL or self.config.scope is not ArchiveScope.ALL: + summary.status = "complete" + return + + report = audit_gmail_labels( + self.gateway, + self.repository, + account_id=context.account_id, + account=self.config.account, + ) + report_path = write_gmail_label_audit(report, self.paths.reports) + self.progress( + "label_audit_complete", + { + "passed": report.passed, + "missing_raw_messages": report.missing_raw_messages, + "report": str(report_path), + }, + ) + + if report.passed: + summary.status = "complete" + return + + summary.status = "incomplete" + summary.stop_reason = ( + "Gmail label coverage failed: " + f"{report.missing_raw_messages} unique remote messages lack raw EML. " + f"See {report_path}." + ) + def _discover_metadata(self, context: SyncContext, summary: SyncSummary) -> None: available = self.gateway.list_mailboxes() selected = self.profile.choose_mailboxes( @@ -200,7 +234,9 @@ def _discover_metadata(self, context: SyncContext, summary: SyncSummary) -> None def _selection_key(self) -> str: query = self.config.query or "" - return f"{self.profile.kind.value}:{self.config.scope.value}:{query}" + return ( + f"v{_SEARCH_PLAN_VERSION}:{self.profile.kind.value}:{self.config.scope.value}:{query}" + ) def _archive_pending(self, context: SyncContext, summary: SyncSummary) -> None: throttle = BandwidthThrottle( diff --git a/tests/test_gmail_audit.py b/tests/test_gmail_audit.py new file mode 100644 index 0000000..82c3318 --- /dev/null +++ b/tests/test_gmail_audit.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from pathlib import Path + +from firexcore_mailvault.config import MailVaultConfig +from firexcore_mailvault.gmail_audit import audit_gmail_labels +from firexcore_mailvault.models import MailboxInfo +from firexcore_mailvault.paths import build_archive_paths +from firexcore_mailvault.protocols.imap.capabilities import parse_capabilities +from firexcore_mailvault.protocols.imap.gateway import SelectResult +from firexcore_mailvault.repository import ArchiveRepository + + +class AuditGateway: + def __init__(self) -> None: + self.capabilities = parse_capabilities([b"IMAP4rev1", b"X-GM-EXT-1"]) + self.selected_mailbox: str | None = None + self._mailbox_uids = { + "[Gmail]/All Mail": [1, 2], + "Suppliers": [2, 3], + } + self._gmail_ids = { + 1: "1001", + 2: "1002", + 3: "1003", + } + + def list_mailboxes(self) -> list[MailboxInfo]: + return [ + MailboxInfo("[Gmail]/All Mail", "/", ("\\All",)), + MailboxInfo("Suppliers", "/", ("\\HasNoChildren",)), + ] + + def select_readonly(self, mailbox: str) -> SelectResult: + self.selected_mailbox = mailbox + return SelectResult(1, 3, None, None, len(self._mailbox_uids[mailbox])) + + def search_uids(self, description: object) -> list[int]: + assert description == ("imap", "ALL") + assert self.selected_mailbox is not None + return self._mailbox_uids[self.selected_mailbox] + + def fetch_metadata(self, uids: list[int], extra_items: list[str]) -> list[object]: + raise AssertionError("metadata fetch is not used by the label audit") + + def fetch_raw(self, uid: int) -> bytes: + raise AssertionError("raw fetch is not used by the label audit") + + def fetch_gmail_message_ids(self, uids: list[int]) -> dict[int, str]: + return {uid: self._gmail_ids[uid] for uid in uids} + + def reconnect(self) -> None: + return None + + +def test_gmail_label_audit_reports_remote_messages_without_raw(tmp_path: Path) -> None: + config = MailVaultConfig( + account="buyer@gmail.com", + destination=tmp_path / "archive", + host="imap.gmail.com", + ) + paths = build_archive_paths(config.destination) + + with ArchiveRepository(paths.database) as repository: + account_id = repository.get_or_create_account(config, config.provider) + now = "2026-01-01T00:00:00+00:00" + with repository.transaction() as connection: + for message_id, gmail_id in ((1, "1001"), (2, "1002")): + connection.execute( + """ + INSERT INTO messages( + id, archive_id, account_id, raw_path, raw_sha256, + raw_size_bytes, created_at, updated_at + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + message_id, + f"archive-{message_id}", + account_id, + f"objects/raw/{gmail_id}", + gmail_id.zfill(64), + 10, + now, + now, + ), + ) + connection.execute( + """ + INSERT INTO message_identities( + account_id, message_id, namespace, value, created_at + ) VALUES(?, ?, 'gmail-x-gm-msgid', ?, ?) + """, + (account_id, message_id, gmail_id, now), + ) + + report = audit_gmail_labels( + AuditGateway(), + repository, + account_id=account_id, + account=config.account, + ) + + assert report.passed is False + assert report.missing_raw_messages == 1 + suppliers = next(item for item in report.labels if item.mailbox == "Suppliers") + assert suppliers.remote_messages == 2 + assert suppliers.archived_raw_messages == 1 + assert suppliers.missing_gmail_message_ids == ("1003",) diff --git a/tests/test_providers.py b/tests/test_providers.py index 570610f..3a93816 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -27,6 +27,7 @@ def test_auto_detects_gmail_and_uses_all_mail() -> None: patterns=(), ) assert [mailbox.name for mailbox in chosen] == ["[Gmail]/All Mail"] + assert profile.search_criteria(ArchiveScope.ALL, None) == ("imap", "ALL") assert profile.search_criteria(ArchiveScope.HAS_ATTACHMENTS, None) == ( "gmail", "has:attachment", @@ -46,3 +47,19 @@ def test_gmail_stable_identity() -> None: profile = GmailImapProfile() assert profile.stable_message_identity(record) == ("gmail-x-gm-msgid", "123") assert profile.stable_thread_identity(record) == ("gmail-x-gm-thrid", "456") + + +def test_gmail_can_select_explicit_label_patterns() -> None: + profile = GmailImapProfile() + chosen = profile.choose_mailboxes( + [ + MailboxInfo("[Gmail]/All Mail", "/", ("\\All",)), + MailboxInfo("NIPHON", "/", ("\\HasNoChildren",)), + MailboxInfo("Mr.Ayobi", "/", ("\\HasNoChildren",)), + ], + include_spam=False, + include_trash=False, + patterns=("NIP*",), + ) + + assert [mailbox.name for mailbox in chosen] == ["NIPHON"] diff --git a/tests/test_sync_engine.py b/tests/test_sync_engine.py index b8cb6c6..7122f20 100644 --- a/tests/test_sync_engine.py +++ b/tests/test_sync_engine.py @@ -56,6 +56,9 @@ def fetch_metadata(self, uids: list[int], extra_items: list[str]) -> list[Metada def fetch_raw(self, uid: int) -> bytes: return self.raw + def fetch_gmail_message_ids(self, uids: list[int]) -> dict[int, str]: + return {uid: str(uid) for uid in uids} + def reconnect(self) -> None: return None From 363bf7568c6efc06e76a5c84a9f4b5b19bc4697c Mon Sep 17 00:00:00 2001 From: Farbod Akvan Date: Sat, 18 Jul 2026 13:35:19 +0330 Subject: [PATCH 2/2] docs: document resumable Windows-safe view builds --- CHANGELOG.md | 28 ++ README.md | 28 +- docs/ARCHITECTURE.md | 2 +- docs/ARCHIVE_FORMAT.md | 14 +- docs/CLI_REFERENCE.md | 41 ++- docs/DEVELOPMENT.md | 16 ++ docs/GETTING_STARTED.md | 18 ++ docs/OPERATIONS.md | 21 +- docs/README.fa.md | 40 ++- docs/REPOSITORY_SETUP.md | 2 +- docs/RESUMABLE_VIEWS.md | 330 ++++++++++++++++++++++++ docs/SECURITY_MODEL.md | 11 +- docs/TROUBLESHOOTING.md | 48 ++++ docs/assets/views-progress-terminal.png | Bin 0 -> 740558 bytes docs/assets/views-resume-progress.png | Bin 0 -> 983929 bytes docs/releases/v2.0.6.md | 115 +++++++++ 16 files changed, 703 insertions(+), 11 deletions(-) create mode 100644 docs/RESUMABLE_VIEWS.md create mode 100644 docs/assets/views-progress-terminal.png create mode 100644 docs/assets/views-resume-progress.png create mode 100644 docs/releases/v2.0.6.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e6c84..acda5e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to FireXCore MailVault are documented in this file. The project follows Semantic Versioning. +## 2.0.6 - 2026-07-18 + +### Added + +- Added resumable navigation-view builds with durable checkpoints and deterministic source fingerprints. +- Added an exact progress bar with source rows, pointer writes, percentage and estimated time remaining. +- Added `mailvault views --restart` for explicitly discarding an incomplete build. +- Added a completed-view snapshot marker so unchanged archives return `UP TO DATE` without rewriting pointers. + +### Fixed + +- Bounded derived view path segments and pointer filenames with collision-resistant hashes for Windows portability. +- Shortened atomic temporary filenames so destination names are not duplicated into temporary paths. +- Built views in a staging tree and published them transactionally, preserving the previous completed views until the replacement is ready. +- Corrected per-view pointer counters so `by-domain`, `by-thread`, `by-year`, `by-mailbox` and `by-label` totals are reported accurately. + +### Reliability + +- A safe interruption checkpoints only fully written source rows; rerunning the same command resumes from the last durable cursor. +- A changed SQLite source snapshot invalidates an incomplete checkpoint and starts a clean replacement build automatically. +- View builds share the archive process lock with sync so derived outputs cannot be published from a concurrently changing archive. + +### Documentation + +- Added a complete operator and engineering guide for Windows-safe, resumable view builds. +- Added real runtime screenshots covering the release quality gate and live view progress. +- Added exact interruption, resume, restart, publication, state-inspection, and troubleshooting procedures. + ## 2.0.5 - 2026-07-14 ### Fixed diff --git a/README.md b/README.md index b99fd36..a486466 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@

راهنمای فارسی · Getting started · + Resumable views · Architecture · Procurement readiness

@@ -215,11 +216,32 @@ See [Configuration](docs/CONFIGURATION.md). | `mailvault stats` | Display message, occurrence, MIME-part, blob, and storage counts. | | `mailvault verify` | Recalculate raw-message and blob hashes. | | `mailvault export` | Regenerate portable JSONL and procurement source manifests. | -| `mailvault views` | Regenerate disposable navigation views by domain, sender, thread, mailbox, year, and label. | +| `mailvault views` | Build resumable navigation views with exact progress, ETA, transactional publication, and automatic checkpoint recovery. | | `mailvault version` | Print the installed distribution version. | See [CLI reference](docs/CLI_REFERENCE.md). +## Windows-safe, resumable navigation views + +MailVault 2.0.6 hardens the derived navigation-view pipeline for large archives: + +1. **Windows-safe paths** — untrusted labels, sender addresses, thread identifiers, and attachment filenames cannot create unbounded view paths. Directory segments and pointer filenames are bounded and receive deterministic SHA-256 suffixes when required. +2. **Durable resume** — completed source rows are checkpointed into a staging build. After `Ctrl+C`, rerunning the same command resumes from the last durable cursor instead of deleting all completed work. +3. **Exact progress and ETA** — the command plans the complete source snapshot, calculates exact source-row and pointer totals, and displays planning, building, resuming, publishing, percentage, and estimated time remaining. + +

+ FireXCore MailVault resumable view build with exact progress and ETA +

+ +```powershell +mailvault views ` + --destination "E:\MailVault-E" +``` + +Use `--restart` only to intentionally discard an incomplete staging build. The previous completed `views/` tree remains available until its replacement is fully written and transactionally published. + +See [Resumable navigation views](docs/RESUMABLE_VIEWS.md) for the lifecycle, state files, path-safety model, recovery behavior, operational checks, and validation commands. + ## Archive layout ```text @@ -302,7 +324,7 @@ Review [Security policy](SECURITY.md) and [Security model](docs/SECURITY_MODEL.m ## Reliability and operational limits -MailVault uses metadata-first discovery, bounded fetch batches, randomized delays, rolling 24-hour bandwidth caps, checkpointed resume, and exponential retry. Provider limits still apply. A sync can be safely stopped and restarted with the same command and destination. +MailVault uses metadata-first discovery, bounded fetch batches, randomized delays, rolling 24-hour bandwidth caps, checkpointed resume, and exponential retry. Provider limits still apply. Sync and navigation-view builds can be safely stopped and restarted with the same command and destination. View builds display exact source-row progress and ETA, write into a resumable staging tree, and publish only after the replacement snapshot is complete. Before transferring or importing an archive, run: @@ -330,6 +352,7 @@ See [Contributing](CONTRIBUTING.md) and [Development](docs/DEVELOPMENT.md). - [Getting started](docs/GETTING_STARTED.md) - [Configuration](docs/CONFIGURATION.md) - [CLI reference](docs/CLI_REFERENCE.md) +- [Resumable navigation views](docs/RESUMABLE_VIEWS.md) - [Provider support](docs/PROVIDERS.md) - [Architecture](docs/ARCHITECTURE.md) - [Archive format](docs/ARCHIVE_FORMAT.md) @@ -341,6 +364,7 @@ See [Contributing](CONTRIBUTING.md) and [Development](docs/DEVELOPMENT.md). - [Repository setup](docs/REPOSITORY_SETUP.md) - [Roadmap](docs/ROADMAP.md) - [References](docs/REFERENCES.md) +- [Release notes: 2.0.6](docs/releases/v2.0.6.md) ## License diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 748a948..7ebf7b3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -51,7 +51,7 @@ Derived: - navigation views; - reports. -Derived outputs can be removed and regenerated without contacting the mail server. +Derived outputs can be removed and regenerated without contacting the mail server. Navigation views are generated from a fingerprinted SQLite read snapshot, checkpointed by a deterministic row cursor, and published from staging only after completion. ## Identity strategy diff --git a/docs/ARCHIVE_FORMAT.md b/docs/ARCHIVE_FORMAT.md index 6b69617..c88c769 100644 --- a/docs/ARCHIVE_FORMAT.md +++ b/docs/ARCHIVE_FORMAT.md @@ -9,10 +9,14 @@ MailVault/ ├── metadata/messages/.json ├── database/mailvault.sqlite3 ├── manifests/*.jsonl -├── state/* +├── state/ +│ ├── views-rebuild-v1.json +│ ├── views-rebuild-staging-v3/ +│ └── views-previous/ ├── reports/* ├── logs/* -└── views/* +└── views/ + └── _mailvault_views.json ``` ## Raw message objects @@ -55,7 +59,11 @@ JSONL files provide streaming and portable integration surfaces. They are regene ## Navigation views -Views contain pointer JSON, not duplicated attachment bytes. Their paths are sanitized for human navigation and are disposable. +Views contain pointer JSON, not duplicated attachment bytes. Their path segments and pointer names are bounded and collision-resistant for cross-platform portability. Original filenames remain evidence fields inside pointer JSON rather than becoming unbounded storage names. + +Builds use a resumable staging tree, a durable source-row checkpoint, a deterministic source fingerprint, and a completed snapshot marker. The existing completed tree is replaced only after the new snapshot is fully written and ready for publication. An interrupted publication can restore the previous completed tree. Views remain disposable. + +See [Resumable navigation views](RESUMABLE_VIEWS.md). ## Portability diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 4237340..28e7ead 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -98,7 +98,46 @@ Regenerates portable JSONL manifests and `procurement_sources.jsonl` from SQLite ## `mailvault views` -Regenerates disposable pointer views by domain, sender, thread, mailbox, year, and label. +Builds disposable JSON pointer views by domain, sender, thread, mailbox, year, and label. + +```powershell +mailvault views ` + --destination "E:\MailVault-E" +``` + +Options: + +```text +--destination, -d Archive root +--restart Discard an incomplete staging build and start from row zero +``` + +The command performs four phases: + +1. **Planning** scans the protected SQLite snapshot to calculate exact source-row and pointer totals and a deterministic fingerprint. +2. **Building** or **Resuming** writes pointers into `state/views-rebuild-staging-v3/`. +3. **Publishing** atomically replaces the completed `views/` snapshot. +4. **Completed** reports one of `REBUILT`, `RESUMED`, or `UP TO DATE`. + +The progress display includes exact source rows, pointer writes, percentage, and ETA. + +A safe interruption checkpoints fully completed source rows in: + +```text +state/views-rebuild-v1.json +``` + +Rerun the same command to resume. Do not use `--restart` when the goal is to continue the existing staging build. + +The previous completed `views/` tree remains available until the replacement is fully built and published. If SQLite changes after interruption, the stale checkpoint is rejected automatically. If the completed marker and source fingerprint already match, the command returns `UP TO DATE` without rewriting pointer files. + +Completed snapshots contain: + +```text +views/_mailvault_views.json +``` + +See [Resumable navigation views](RESUMABLE_VIEWS.md). ## Module execution diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index abcb022..7be4c0c 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -57,6 +57,22 @@ Provider tests should cover: - provider query translation; - fallback behavior when extensions are absent. + +## View exporter validation + +Run the targeted Windows-path, interruption, resume, progress and snapshot tests: + +```powershell +python -m pytest ` + tests\test_view_exporter.py ` + tests\test_unicode_safety.py ` + -q +``` + +The view exporter tests cover bounded deterministic filenames, short atomic temporary names, durable source-row checkpoints, resume, source-fingerprint invalidation, transactional publication, up-to-date no-op behavior, and exact pointer totals. + +See [Resumable navigation views](RESUMABLE_VIEWS.md). + ## Release validation Before tagging: diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index cf1dda3..7b91214 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -108,6 +108,24 @@ mailvault export --destination E:\MailVault mailvault views --destination E:\MailVault ``` +

+ MailVault live view-build progress +

+ +`mailvault views` first plans an exact source snapshot and then displays determinate source-row progress, exact pointer writes, percentage, and ETA. After `Ctrl+C`, run the identical command to continue from the last durable source-row checkpoint. + +```powershell +mailvault views --destination E:\MailVault +``` + +Use `--restart` only when you explicitly want to discard the incomplete staging build: + +```powershell +mailvault views --destination E:\MailVault --restart +``` + +A successful result reports `REBUILT` or `RESUMED`. A second run against an unchanged archive reports `UP TO DATE` without rewriting pointers. See [Resumable navigation views](RESUMABLE_VIEWS.md). + ## Recommended first-run sequence 1. Run `doctor`. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 91530c6..d64f9d2 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -68,8 +68,27 @@ Rebuild manifests: mailvault export --destination /archive/path ``` -Rebuild navigation views: +Build or resume navigation views: ```bash mailvault views --destination /archive/path ``` + +The progress display reports exact source rows, pointer writes, percentage and ETA. A safe interruption preserves the durable cursor and staging tree; rerun the same command to continue. To intentionally discard an incomplete build: + +```bash +mailvault views --destination /archive/path --restart +``` + +The previous completed `views/` snapshot is retained until the replacement has been fully written and transactionally published. + +Operational state: + +```text +state/views-rebuild-v1.json durable checkpoint +state/views-rebuild-staging-v3/ incomplete replacement tree +state/views-previous/ temporary publication rollback tree +views/_mailvault_views.json completed snapshot marker +``` + +Do not remove the checkpoint or staging tree when the goal is to resume. A source change invalidates stale state automatically. See [Resumable navigation views](RESUMABLE_VIEWS.md) for lifecycle and recovery details. diff --git a/docs/README.fa.md b/docs/README.fa.md index 62d601c..5394ba9 100644 --- a/docs/README.fa.md +++ b/docs/README.fa.md @@ -132,10 +132,47 @@ mailvault sync آرشیو کامل و قابل Resume پیام‌ها mailvault stats نمایش آمار آرشیو mailvault verify بررسی مجدد Hash فایل‌های EML و Blobها mailvault export بازسازی JSONL و Procurement Manifest -mailvault views بازسازی Viewهای Domain، Sender، Thread، Year و Label +mailvault views ساخت Resumeپذیر Viewها با Progress دقیق، ETA و انتشار تراکنشی mailvault version نمایش نسخه نصب‌شده ``` +## Viewهای حرفه‌ای در نسخه 2.0.6 + +نسخه 2.0.6 سه ضعف عملیاتی لایه `views` را برطرف می‌کند: + +1. **مسیرهای امن برای Windows**: نام Label، Sender، Thread و Attachment دیگر نمی‌تواند مسیر نامحدود تولید کند. Segmentها محدود و همراه با Hash پایدار هستند و نام فایل موقت Atomic نیز کوتاه است. +2. **Resume واقعی بعد از توقف**: پس از `Ctrl+C`، Cursor فقط تا آخرین Source Row کاملاً نوشته‌شده جلو می‌رود. اجرای مجدد همان دستور از Checkpoint معتبر ادامه می‌دهد. +3. **Progress و ETA دقیق**: ابتدا تعداد دقیق Source Row و Pointer محاسبه می‌شود؛ سپس مرحله Planning، Building یا Resuming، درصد، تعداد Pointer و زمان تقریبی باقی‌مانده نمایش داده می‌شود. + +

+ نمایش Progress و ETA ساخت Viewهای MailVault +

+ +ساخت یا ادامه Viewها: + +```powershell +mailvault views ` + --destination "E:\MailVault-E" +``` + +توقف با `Ctrl+C` امن است. برای ادامه همان فرمان را دوباره اجرا کن. فقط برای کنارگذاشتن عمدی Build نیمه‌کاره از این گزینه استفاده می‌شود: + +```powershell +mailvault views ` + --destination "E:\MailVault-E" ` + --restart +``` + +وضعیت نهایی یکی از این موارد است: + +```text +REBUILT +RESUMED +UP TO DATE +``` + +تا وقتی Snapshot جدید کامل نشده، View کامل قبلی جایگزین یا حذف نمی‌شود. راهنمای فنی کامل در [Resumable navigation views](RESUMABLE_VIEWS.md) قرار دارد. + ## ارتباط با RMS و Procurement Intelligence فایل زیر برای تمام Bodyها و MIME Partهای قابل استفاده Evidence Record تولید می‌کند: @@ -201,3 +238,4 @@ mailvault verify --destination E:\MailVault - [امنیت](SECURITY_MODEL.md) - [آمادگی Procurement](PROCUREMENT_READINESS.md) - [رفع خطا](TROUBLESHOOTING.md) +- [ساخت Resumeپذیر Viewها](RESUMABLE_VIEWS.md) diff --git a/docs/REPOSITORY_SETUP.md b/docs/REPOSITORY_SETUP.md index a018e8c..c0ef580 100644 --- a/docs/REPOSITORY_SETUP.md +++ b/docs/REPOSITORY_SETUP.md @@ -50,7 +50,7 @@ Enable: ## Releases -Create Git tags in the form `v2.0.5`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. +Create Git tags in the form `v2.0.6`. The release workflow builds wheel and source distribution, validates metadata, writes SHA-256 checksums and attaches artifacts to the GitHub release. PyPI publishing uses a separate trusted-publishing workflow. Configure the GitHub `pypi` environment and PyPI trusted publisher, then set the repository variable `PYPI_PUBLISH_ENABLED=true`. Manual dispatch remains available for controlled validation. diff --git a/docs/RESUMABLE_VIEWS.md b/docs/RESUMABLE_VIEWS.md new file mode 100644 index 0000000..62dbcdf --- /dev/null +++ b/docs/RESUMABLE_VIEWS.md @@ -0,0 +1,330 @@ +# Resumable navigation views + +FireXCore MailVault navigation views are disposable JSON pointer trees derived from the canonical SQLite index and immutable object store. They make large archives easier to browse by sender domain, thread, year, mailbox, and Gmail label without duplicating raw EML or attachment bytes. + +Version 2.0.6 replaces the original one-shot view exporter with a Windows-safe, resumable, observable build pipeline. + +

+ MailVault 2.0.6 view build running with exact progress and ETA +

+ +## What changed in 2.0.6 + +| Change | Previous behavior | Version 2.0.6 behavior | +|---|---|---| +| Path safety | Long attachment names could produce paths that exceeded Windows limits. | Directory segments and pointer filenames are bounded and collision-resistant; atomic temporary filenames use a short constant prefix. | +| Interruption handling | `Ctrl+C` discarded all partial work on the next run. | Fully completed source rows are checkpointed and the same command resumes from the durable cursor. | +| Progress visibility | The command displayed output only after the entire build completed. | Planning, building, resuming, publishing, exact row counts, pointer counts, percentage, and ETA are displayed live. | + +These changes affect only the derived `views/` layer. Raw EML objects, content-addressed blobs, SQLite records, manifests, reports, and evidence files are not rewritten by `mailvault views`. + +## View model + +Each pointer is a small JSON document that references canonical archive objects. A pointer may include: + +- canonical message archive ID; +- provider thread identity; +- subject; +- raw EML path; +- MIME part ID and path; +- original attachment filename; +- SHA-256 digest; +- blob path; +- detected MIME type and size. + +The original attachment filename remains inside pointer JSON for evidence and inspection. It is not trusted as an unbounded filesystem filename. + +The published tree contains: + +```text +views/ +├── _mailvault_views.json +├── by-domain////.json +├── by-thread//.json +├── by-year//.json +├── by-mailbox//.json +└── by-label/