diff --git a/.github/scripts/tests/test_validate_docs_production_status.py b/.github/scripts/tests/test_validate_docs_production_status.py deleted file mode 100644 index f7a15c37c7c5..000000000000 --- a/.github/scripts/tests/test_validate_docs_production_status.py +++ /dev/null @@ -1,1010 +0,0 @@ -import importlib.util -import tempfile -import unittest -from pathlib import Path - - -BACKEND_IMAGE = ( - "ghcr.io/dos/doscan:11.2.3.commit.86fd0dd5@" - "sha256:423bab078a679d3290cc6e276774a8ed201686636933e0d066ce9859270f700d" -) -BACKEND_IMAGE_WITH_DIFFERENT_DIGEST = ( - "ghcr.io/dos/doscan:11.2.3.commit.86fd0dd5@" - "sha256:523bab078a679d3290cc6e276774a8ed201686636933e0d066ce9859270f700d" -) -BACKEND_RUNTIME_VERSION = "v11.2.3.+commit.86fd0dd5" -FRONTEND_IMAGE = ( - "metados/blockscout-frontend:2.10.0@" - "sha256:4125d49b1658ba95b81075cabbc07120bebd90be95df49440aff5fa0e7e95eed" -) -STALE_FRONTEND_IMAGE = ( - "metados/blockscout-frontend:2.9.0@" - "sha256:6125d49b1658ba95b81075cabbc07120bebd90be95df49440aff5fa0e7e95eed" -) -VALIDATOR_COMMAND = "python .github/scripts/validate-docs-production-status.py" -REQUIRED_PUSH_PATHS = ( - "docker-compose/docker-compose-mainnet.yml", - "docker-compose/docker-compose-testnet.yml", - "docker-compose/docker-compose-beta.yml", - "docker-compose/envs/common-blockscout.env", - "docker-compose/envs/common-blockscout-mainnet.env", - "docker-compose/envs/common-blockscout-testnet.env", - "docker-compose/envs/common-blockscout-beta.env", - "docker-compose/envs/common-frontend*.env", - "docs/FEATURES.md", - "docs/CHANGELOG.md", - "docs/DOScan-ARCHITECTURE.md", - "docs/reports/doscan-frontend-env-audit.vi.html", - ".github/workflows/deploy-config.yml", - ".github/workflows/dependency-build.yml", - ".github/scripts/validate-docs-production-status.py", - ".github/scripts/tests/**", -) - - -class ValidateDocsProductionStatusTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - script_path = ( - Path(__file__).resolve().parents[1] - / "validate-docs-production-status.py" - ) - spec = importlib.util.spec_from_file_location( - "validate_docs_production_status", script_path - ) - cls.module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cls.module) - - def setUp(self): - self.temporary_directory = tempfile.TemporaryDirectory() - self.repo = Path(self.temporary_directory.name) - self.write_fixture_repository() - - def tearDown(self): - self.temporary_directory.cleanup() - - def write_fixture_repository(self): - compose = """services: - backend: - image: {backend_image} - env_file: - - ./envs/common-blockscout.env - - ${{DOSCAN_BLOCKSCOUT_SECRETS_ENV:-/run/secrets/blockscout.env}} - - ./envs/common-blockscout-{environment}.env - environment: - MICROSERVICE_METADATA_ENABLED: "true" - MICROSERVICE_BENS_ENABLED: "false" - MICROSERVICE_BENS_URL: "" - MICROSERVICE_BENS_PROTOCOLS: "" - frontend: - image: {frontend_image} -""" - for environment in ("mainnet", "testnet", "beta"): - self.write( - f"docker-compose/docker-compose-{environment}.yml", - compose.format( - backend_image=BACKEND_IMAGE, - frontend_image=FRONTEND_IMAGE, - environment=environment, - ), - ) - self.write( - f"docker-compose/envs/common-blockscout-{environment}.env", - f"# {environment} overrides do not redefine protected keys.\n", - ) - - self.write( - "docs/FEATURES.md", - f"""# DOScan Feature Status - -## Runtime Baseline - -| Environment | Explorer | Chain ID | Frontend | Backend | Runtime status | -|---|---|---:|---|---|---| -| Mainnet | `https://doscan.io` | 7979 | `2.10.0` | `{BACKEND_RUNTIME_VERSION}` | healthy | -| Testnet | `https://test.doscan.io` | 3939 | `2.10.0` | `{BACKEND_RUNTIME_VERSION}` | healthy | - -Both production environments pin the custom backend image below: - -```text -{BACKEND_IMAGE} -``` - -## Backend Features - -### Backend Integrations and Services - -| Service or integration | Mainnet | Testnet | Runtime path | -|---|---|---|---| -| Metadata Service | Enabled | Enabled | `/metadata-api` | - -## Deliberately Disabled or Blocked Features - -| Feature | Status | Reason or prerequisite | -|---|---|---| -| BENS / name service | Disabled | Not configured for DOS Chain | -""", - ) - self.write( - "docs/CHANGELOG.md", - f"""# DOScan Changelog - -## [2026-08-04] - Production Documentation and Runtime Baseline - -### Deployed - -- Mainnet and Testnet run Frontend `2.10.0` and custom Backend `{BACKEND_RUNTIME_VERSION}` on GCP. -- Both production Compose files pin `{BACKEND_IMAGE}`. - -### Changed - -- Corrected feature status: the admin panel and BENS are disabled, while Metadata Service is enabled. - ---- - -## [2026-02-01] - Historical Release - -- Historical content is not the current production baseline. -""", - ) - self.write( - "docs/DOScan-ARCHITECTURE.md", - f"""# DOScan Architecture - -## Deployed Environments - -| Environment | Public origin | Chain ID | GCP host | Zone | Deployment path | -|---|---|---:|---|---|---| -| Mainnet | `https://doscan.io` | 7979 | `doscan-mainnet` | `asia-southeast1-b` | `/opt/doscan-l1` | -| Testnet | `https://test.doscan.io` | 3939 | `dos-testnet-r0` | `asia-southeast1-a` | `/opt/doscan-testnet` | -| Beta | `https://beta.doscan.io` | 7979 | `doscan-mainnet` | `asia-southeast1-b` | `/opt/doscan-beta` | - -## Runtime Versions - -| Component | Production version | -|---|---| -| Frontend | `{FRONTEND_IMAGE}` | -| Backend | `{BACKEND_IMAGE}` | - -## Backend Integrations - -### Metadata Service - -The backend enables Blockscout Metadata Service: - -```env -MICROSERVICE_METADATA_ENABLED=true -``` - -BENS is not configured. Name-service UI and backend integration remain disabled. - -## Historical Notes - -Historical records are outside the current status sections. -""", - ) - self.write( - "docs/reports/doscan-frontend-env-audit.vi.html", - f""" - - -

Backend production: {BACKEND_RUNTIME_VERSION}

-

Backend image: {BACKEND_IMAGE}

- - -""", - ) - self.write( - "docker-compose/envs/common-blockscout.env", - "# MICROSERVICE_BENS_ENABLED=\n" - "# MICROSERVICE_BENS_URL=\n" - "# MICROSERVICE_BENS_PROTOCOLS=\n" - "MICROSERVICE_METADATA_ENABLED=true\n", - ) - for filename in ( - "common-frontend.env", - "common-frontend-scan.env", - "common-frontend-testnet.env", - "common-frontend-beta.env", - ): - self.write( - f"docker-compose/envs/{filename}", - "# NEXT_PUBLIC_NAME_SERVICE_API_HOST=\n", - ) - self.write( - ".github/workflows/deploy-config.yml", - """env: - GCP_INSTANCE: doscan-mainnet - GCP_ZONE: asia-southeast1-b - GCP_TESTNET_INSTANCE: dos-testnet-r0 - GCP_TESTNET_ZONE: asia-southeast1-a - -jobs: - deploy: - steps: - - run: echo deploy -""", - ) - push_paths = "\n".join(f' - "{path}"' for path in REQUIRED_PUSH_PATHS) - self.write( - ".github/workflows/dependency-build.yml", - f"""name: Dependency build - -on: - pull_request: - push: - branches: [main] - paths: -{push_paths} - -jobs: - workflow-scripts: - runs-on: ubuntu-latest - steps: - - name: Validate production documentation status - run: {VALIDATOR_COMMAND} -""", - ) - - def write(self, relative_path, content): - path = self.repo / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - - def replace(self, relative_path, old, new): - path = self.repo / relative_path - original = path.read_text(encoding="utf-8") - self.assertIn(old, original, f"fixture mutation target missing: {old!r}") - path.write_text(original.replace(old, new), encoding="utf-8") - - def append(self, relative_path, content): - path = self.repo / relative_path - path.write_text(path.read_text(encoding="utf-8") + content, encoding="utf-8") - - def assert_diagnostic(self, errors, path, invariant, actual=None): - matches = [ - error - for error in errors - if path in error and invariant in error and "expected" in error and "actual" in error - ] - self.assertTrue(matches, "\n".join(errors)) - if actual is not None: - self.assertTrue(any(actual in error for error in matches), "\n".join(matches)) - - def test_synchronized_repository_passes(self): - self.assertEqual([], self.module.validate_repository(self.repo)) - - def test_repository_workflow_keeps_the_validator_bootstrapped(self): - repository_root = Path(__file__).resolve().parents[3] - workflow_path = repository_root / ".github/workflows/dependency-build.yml" - - self.assertEqual( - [], - self.module.dependency_workflow_errors( - workflow_path.read_text(encoding="utf-8"), - Path(".github/workflows/dependency-build.yml"), - ), - ) - - def test_backend_digest_drift_names_the_compose_file(self): - self.replace( - "docker-compose/docker-compose-testnet.yml", - BACKEND_IMAGE, - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ) - errors = self.module.validate_repository(self.repo) - self.assert_diagnostic( - errors, "docker-compose-testnet.yml", "backend image", BACKEND_IMAGE_WITH_DIFFERENT_DIGEST - ) - - def test_html_report_current_backend_version_is_guarded(self): - stale_version = "v11.2.2.+commit.deadbeef" - self.replace( - "docs/reports/doscan-frontend-env-audit.vi.html", - BACKEND_RUNTIME_VERSION, - stale_version, - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - "doscan-frontend-env-audit.vi.html", - "backend version", - stale_version, - ) - - def test_html_report_current_backend_image_is_guarded(self): - self.replace( - "docs/reports/doscan-frontend-env-audit.vi.html", - BACKEND_IMAGE, - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - "doscan-frontend-env-audit.vi.html", - "backend image", - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ) - - def test_invalid_and_missing_sources_have_complete_aggregated_diagnostics(self): - (self.repo / "docker-compose/docker-compose-beta.yml").unlink() - invalid_frontend_image = "metados/blockscout-frontend:2.10.0" - self.replace( - "docker-compose/docker-compose-testnet.yml", - FRONTEND_IMAGE, - invalid_frontend_image, - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic(errors, "docker-compose-beta.yml", "missing required file", "missing") - self.assert_diagnostic( - errors, - "docker-compose-testnet.yml", - "frontend immutable image pin", - invalid_frontend_image, - ) - - def test_common_metadata_and_bens_source_invariants(self): - self.replace( - "docker-compose/envs/common-blockscout.env", - "MICROSERVICE_METADATA_ENABLED=true", - "MICROSERVICE_METADATA_ENABLED=false\nMICROSERVICE_BENS_ENABLED=true", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic(errors, "common-blockscout.env", "metadata enabled", "false") - self.assert_diagnostic(errors, "common-blockscout.env", "BENS disabled", "true") - - def test_env_parser_accepts_whitespace_quotes_and_comments_outside_quotes(self): - self.write( - "docker-compose/envs/common-blockscout.env", - " MICROSERVICE_METADATA_ENABLED = \"true\" # required\n" - "MICROSERVICE_BENS_ENABLED = 'false' # deliberately disabled\n" - "MICROSERVICE_BENS_URL = \"\" # no endpoint\n" - "MICROSERVICE_BENS_PROTOCOLS = '' # no protocols\n", - ) - - self.assertEqual([], self.module.validate_repository(self.repo)) - - def test_frontend_name_service_host_is_rejected(self): - self.append( - "docker-compose/envs/common-frontend-testnet.env", - "NEXT_PUBLIC_NAME_SERVICE_API_HOST=https://bens.example\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - "common-frontend-testnet.env", - "NEXT_PUBLIC_NAME_SERVICE_API_HOST disabled", - "https://bens.example", - ) - - def test_document_status_mutations_report_the_parsed_current_value(self): - cases = ( - ( - "docs/FEATURES.md", - "| Metadata Service | Enabled | Enabled | `/metadata-api` |", - "| Metadata Service | Disabled | Enabled | `/metadata-api` |", - "metadata documentation status", - "Mainnet=Disabled, Testnet=Enabled", - ), - ( - "docs/FEATURES.md", - "| BENS / name service | Disabled | Not configured for DOS Chain |", - "| BENS / name service | Enabled | Not configured for DOS Chain |", - "BENS documentation status", - "Enabled", - ), - ( - "docs/CHANGELOG.md", - "while Metadata Service is enabled.", - "while Metadata Service is disabled.", - "metadata documentation status", - "Disabled", - ), - ( - "docs/CHANGELOG.md", - "BENS are disabled, while", - "BENS are enabled, while", - "BENS documentation status", - "Enabled", - ), - ( - "docs/DOScan-ARCHITECTURE.md", - "MICROSERVICE_METADATA_ENABLED=true", - "MICROSERVICE_METADATA_ENABLED=false", - "metadata documentation status", - "false", - ), - ( - "docs/DOScan-ARCHITECTURE.md", - "BENS is not configured.", - "BENS is configured.", - "BENS documentation status", - "Configured", - ), - ) - for path, old, new, invariant, actual in cases: - with self.subTest(path=path, invariant=invariant): - self.replace(path, old, new) - errors = self.module.validate_repository(self.repo) - self.assert_diagnostic(errors, path, invariant, actual) - self.replace(path, new, old) - - def test_features_current_frontend_version_uses_exact_table_fields(self): - self.replace("docs/FEATURES.md", "| `2.10.0` |", "| `2.10.01` |") - self.append("docs/FEATURES.md", "\nHistorical note: Frontend `2.10.0`.\n") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/FEATURES.md", "frontend version", "2.10.01" - ) - - def test_features_current_backend_pin_is_not_hidden_by_history(self): - self.replace( - "docs/FEATURES.md", - f"```text\n{BACKEND_IMAGE}\n```", - f"```text\n{BACKEND_IMAGE_WITH_DIFFERENT_DIGEST}\n```", - ) - self.append("docs/FEATURES.md", f"\nHistorical pin: `{BACKEND_IMAGE}`.\n") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/FEATURES.md", "backend image", BACKEND_IMAGE_WITH_DIFFERENT_DIGEST - ) - - def test_features_current_backend_version_is_not_hidden_by_another_row(self): - self.replace( - "docs/FEATURES.md", - f"| Mainnet | `https://doscan.io` | 7979 | `2.10.0` | `{BACKEND_RUNTIME_VERSION}` |", - "| Mainnet | `https://doscan.io` | 7979 | `2.10.0` | `v11.2.2` |", - ) - self.append( - "docs/FEATURES.md", - f"\nHistorical backend version: `{BACKEND_RUNTIME_VERSION}`.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/FEATURES.md", "backend version (Mainnet)", "v11.2.2" - ) - - def test_changelog_current_release_is_not_hidden_by_history(self): - self.replace( - "docs/CHANGELOG.md", - "run Frontend `2.10.0`", - "run Frontend `2.9.0`", - ) - self.append("docs/CHANGELOG.md", "\nHistorical note: Frontend `2.10.0`.\n") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic(errors, "docs/CHANGELOG.md", "frontend version", "2.9.0") - - def test_changelog_current_backend_fields_are_not_hidden_by_history(self): - cases = ( - ( - f"custom Backend `{BACKEND_RUNTIME_VERSION}`", - "custom Backend `v11.2.2`", - "backend version", - "v11.2.2", - ), - ( - f"pin `{BACKEND_IMAGE}`", - f"pin `{BACKEND_IMAGE_WITH_DIFFERENT_DIGEST}`", - "backend image", - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ), - ) - original = (self.repo / "docs/CHANGELOG.md").read_text(encoding="utf-8") - for old, new, invariant, actual in cases: - with self.subTest(invariant=invariant): - self.write( - "docs/CHANGELOG.md", - original.replace(old, new) - + f"\nHistorical backend: `{BACKEND_RUNTIME_VERSION}` `{BACKEND_IMAGE}`.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/CHANGELOG.md", invariant, actual - ) - self.write("docs/CHANGELOG.md", original) - - def test_architecture_runtime_pin_is_not_hidden_by_history(self): - self.replace("docs/DOScan-ARCHITECTURE.md", FRONTEND_IMAGE, STALE_FRONTEND_IMAGE) - self.append( - "docs/DOScan-ARCHITECTURE.md", - f"\nHistorical frontend pin: `{FRONTEND_IMAGE}`.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/DOScan-ARCHITECTURE.md", "frontend image", STALE_FRONTEND_IMAGE - ) - - def test_architecture_current_backend_pin_is_not_hidden_by_history(self): - self.replace( - "docs/DOScan-ARCHITECTURE.md", - f"| Backend | `{BACKEND_IMAGE}` |", - f"| Backend | `{BACKEND_IMAGE_WITH_DIFFERENT_DIGEST}` |", - ) - self.append( - "docs/DOScan-ARCHITECTURE.md", - f"\nHistorical backend pin: `{BACKEND_IMAGE}`.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - "docs/DOScan-ARCHITECTURE.md", - "backend image", - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ) - - def test_architecture_gcp_fields_are_not_hidden_by_history(self): - self.replace( - "docs/DOScan-ARCHITECTURE.md", - "| Testnet | `https://test.doscan.io` | 3939 | `dos-testnet-r0` |", - "| Testnet | `https://test.doscan.io` | 3939 | `stale-host` |", - ) - self.append( - "docs/DOScan-ARCHITECTURE.md", - "\nHistorical host: `dos-testnet-r0`.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/DOScan-ARCHITECTURE.md", "GCP_TESTNET_INSTANCE", "stale-host" - ) - - def test_every_architecture_gcp_field_comes_from_the_current_row(self): - cases = ( - ("`doscan-mainnet` | `asia-southeast1-b`", "`stale-mainnet` | `asia-southeast1-b`", "GCP_INSTANCE", "stale-mainnet"), - ("`doscan-mainnet` | `asia-southeast1-b`", "`doscan-mainnet` | `stale-zone`", "GCP_ZONE", "stale-zone"), - ("`dos-testnet-r0` | `asia-southeast1-a`", "`stale-testnet` | `asia-southeast1-a`", "GCP_TESTNET_INSTANCE", "stale-testnet"), - ("`dos-testnet-r0` | `asia-southeast1-a`", "`dos-testnet-r0` | `stale-zone`", "GCP_TESTNET_ZONE", "stale-zone"), - ) - original = (self.repo / "docs/DOScan-ARCHITECTURE.md").read_text( - encoding="utf-8" - ) - for old, new, invariant, actual in cases: - with self.subTest(invariant=invariant): - mutated = original.replace(old, new, 1) - self.assertNotEqual(original, mutated) - self.write( - "docs/DOScan-ARCHITECTURE.md", - mutated + f"\nHistorical topology: {old}.\n", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/DOScan-ARCHITECTURE.md", invariant, actual - ) - self.write("docs/DOScan-ARCHITECTURE.md", original) - - def test_each_backend_overlay_rejects_a_protected_override(self): - cases = ( - ("mainnet", "MICROSERVICE_METADATA_ENABLED=false", "metadata enabled", "false"), - ("testnet", "MICROSERVICE_BENS_ENABLED=true", "BENS disabled", "true"), - ( - "beta", - "MICROSERVICE_BENS_URL=https://bens.example", - "BENS disabled", - "https://bens.example", - ), - ) - for environment, assignment, invariant, actual in cases: - with self.subTest(environment=environment): - path = f"docker-compose/envs/common-blockscout-{environment}.env" - self.write(path, f"{assignment}\n") - errors = self.module.validate_repository(self.repo) - self.assert_diagnostic(errors, path, invariant, actual) - self.write(path, f"# {environment} overrides restored.\n") - - def test_every_referenced_committed_backend_env_file_is_validated(self): - cases = ( - ("mainnet", "MICROSERVICE_METADATA_ENABLED=false", "metadata enabled", "false"), - ("testnet", "MICROSERVICE_BENS_ENABLED=true", "BENS disabled", "true"), - ( - "beta", - "MICROSERVICE_BENS_PROTOCOLS=dos", - "BENS disabled", - "dos", - ), - ) - for environment, assignment, invariant, actual in cases: - with self.subTest(environment=environment): - compose_path = f"docker-compose/docker-compose-{environment}.yml" - marker = f" - ./envs/common-blockscout-{environment}.env\n" - extra_name = f"extra-blockscout-{environment}.env" - extra_reference = f" - ./envs/{extra_name}\n" - self.replace(compose_path, marker, marker + extra_reference) - self.write(f"docker-compose/envs/{extra_name}", f"{assignment}\n") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic(errors, extra_name, invariant, actual) - self.replace(compose_path, marker + extra_reference, marker) - - def test_backend_inline_protected_overrides_are_rejected(self): - cases = ( - ( - "mainnet", - 'MICROSERVICE_METADATA_ENABLED: "true"', - 'MICROSERVICE_METADATA_ENABLED: "false"', - "metadata enabled", - "false", - ), - ( - "testnet", - 'MICROSERVICE_BENS_ENABLED: "false"', - 'MICROSERVICE_BENS_ENABLED: "true"', - "BENS disabled", - "true", - ), - ( - "beta", - 'MICROSERVICE_BENS_PROTOCOLS: ""', - 'MICROSERVICE_BENS_PROTOCOLS: "dos"', - "BENS disabled", - "dos", - ), - ) - for environment, old, new, invariant, actual in cases: - with self.subTest(environment=environment): - path = f"docker-compose/docker-compose-{environment}.yml" - self.replace(path, old, new) - errors = self.module.validate_repository(self.repo) - self.assert_diagnostic(errors, path, invariant, actual) - self.replace(path, new, old) - - def test_unresolved_secret_env_source_requires_every_protected_value_resolved(self): - path = "docker-compose/docker-compose-beta.yml" - original = (self.repo / path).read_text(encoding="utf-8") - cases = ( - ("MICROSERVICE_METADATA_ENABLED", "effective metadata enabled"), - ("MICROSERVICE_BENS_ENABLED", "effective BENS disabled"), - ("MICROSERVICE_BENS_URL", "effective BENS disabled"), - ("MICROSERVICE_BENS_PROTOCOLS", "effective BENS disabled"), - ) - for key, invariant in cases: - with self.subTest(key=key): - line = next( - fixture_line - for fixture_line in original.splitlines(keepends=True) - if fixture_line.strip().startswith(f"{key}:") - ) - self.write(path, original.replace(line, "")) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - "docker-compose-beta.yml", - invariant, - "unresolved", - ) - self.write(path, original) - - def test_committed_source_after_secret_can_resolve_every_protected_key(self): - path = "docker-compose/docker-compose-mainnet.yml" - original = (self.repo / path).read_text(encoding="utf-8") - inline_environment = ( - " environment:\n" - ' MICROSERVICE_METADATA_ENABLED: "true"\n' - ' MICROSERVICE_BENS_ENABLED: "false"\n' - ' MICROSERVICE_BENS_URL: ""\n' - ' MICROSERVICE_BENS_PROTOCOLS: ""\n' - ) - marker = " - ./envs/common-blockscout-mainnet.env\n" - late_reference = " - ./envs/protected-locks.env\n" - self.assertIn(inline_environment, original) - self.assertIn(marker, original) - self.write( - path, - original.replace(marker, marker + late_reference).replace( - inline_environment, "" - ), - ) - self.write( - "docker-compose/envs/protected-locks.env", - "MICROSERVICE_METADATA_ENABLED=true\n" - "MICROSERVICE_BENS_ENABLED=false\n" - "MICROSERVICE_BENS_URL=\n" - "MICROSERVICE_BENS_PROTOCOLS=\n", - ) - - self.assertEqual([], self.module.validate_repository(self.repo)) - - def test_key_only_inline_environment_is_unresolved_not_disabled(self): - path = "docker-compose/docker-compose-testnet.yml" - original = (self.repo / path).read_text(encoding="utf-8") - mapping = ' MICROSERVICE_BENS_ENABLED: "false"\n' - environment_block = ( - " environment:\n" - ' MICROSERVICE_METADATA_ENABLED: "true"\n' - ' MICROSERVICE_BENS_ENABLED: "false"\n' - ' MICROSERVICE_BENS_URL: ""\n' - ' MICROSERVICE_BENS_PROTOCOLS: ""\n' - ) - list_environment = ( - " environment:\n" - " - MICROSERVICE_METADATA_ENABLED=true\n" - " - MICROSERVICE_BENS_ENABLED\n" - " - MICROSERVICE_BENS_URL=\n" - " - MICROSERVICE_BENS_PROTOCOLS=\n" - ) - mutations = ( - original.replace(mapping, " MICROSERVICE_BENS_ENABLED:\n"), - original.replace(environment_block, list_environment), - ) - for style, mutated in zip(("mapping", "list"), mutations, strict=True): - with self.subTest(style=style): - self.assertNotEqual(original, mutated) - self.write(path, mutated) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "effective BENS disabled", "unresolved" - ) - self.write(path, original) - - def test_backend_without_env_files_or_inline_locks_fails_closed(self): - path = "docker-compose/docker-compose-testnet.yml" - env_files = ( - " env_file:\n" - " - ./envs/common-blockscout.env\n" - " - ${DOSCAN_BLOCKSCOUT_SECRETS_ENV:-/run/secrets/blockscout.env}\n" - " - ./envs/common-blockscout-testnet.env\n" - ) - inline_environment = ( - " environment:\n" - ' MICROSERVICE_METADATA_ENABLED: "true"\n' - ' MICROSERVICE_BENS_ENABLED: "false"\n' - ' MICROSERVICE_BENS_URL: ""\n' - ' MICROSERVICE_BENS_PROTOCOLS: ""\n' - ) - original = (self.repo / path).read_text(encoding="utf-8") - self.assertIn(env_files, original) - self.assertIn(inline_environment, original) - self.write(path, original.replace(env_files, "").replace(inline_environment, "")) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "canonical backend env source", "missing" - ) - self.assert_diagnostic( - errors, path, "effective metadata enabled", "missing" - ) - - def test_backend_must_reference_the_canonical_common_env(self): - path = "docker-compose/docker-compose-mainnet.yml" - self.replace(path, " - ./envs/common-blockscout.env\n", "") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "canonical backend env source", "missing" - ) - - def test_effective_backend_metadata_must_not_be_missing(self): - path = "docker-compose/docker-compose-testnet.yml" - self.replace( - "docker-compose/envs/common-blockscout.env", - "MICROSERVICE_METADATA_ENABLED=true\n", - "# MICROSERVICE_METADATA_ENABLED=\n", - ) - self.replace( - path, - " - ${DOSCAN_BLOCKSCOUT_SECRETS_ENV:-/run/secrets/blockscout.env}\n", - "", - ) - self.replace(path, ' MICROSERVICE_METADATA_ENABLED: "true"\n', "") - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "effective metadata enabled", "missing" - ) - - def test_nested_workflow_env_cannot_overwrite_top_level_gcp_value(self): - self.replace( - ".github/workflows/deploy-config.yml", - " GCP_TESTNET_INSTANCE: dos-testnet-r0", - " GCP_TESTNET_INSTANCE: stale-host", - ) - self.append( - ".github/workflows/deploy-config.yml", - """ - nested-env-probe: - env: - GCP_TESTNET_INSTANCE: dos-testnet-r0 - steps: - - run: echo nested -""", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docs/DOScan-ARCHITECTURE.md", "GCP_TESTNET_INSTANCE", "stale-host" - ) - - def test_compose_extension_before_services_cannot_supply_runtime_image(self): - path = "docker-compose/docker-compose-testnet.yml" - compose = (self.repo / path).read_text(encoding="utf-8") - self.write( - path, - """x-image-template: - backend: - image: {historical_image} -""".format(historical_image=BACKEND_IMAGE) - + compose.replace(BACKEND_IMAGE, BACKEND_IMAGE_WITH_DIFFERENT_DIGEST), - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "backend image", BACKEND_IMAGE_WITH_DIFFERENT_DIGEST - ) - - def test_duplicate_top_level_yaml_mappings_are_rejected(self): - self.append( - "docker-compose/docker-compose-testnet.yml", - f""" -services: - backend: - image: {BACKEND_IMAGE} -""", - ) - self.append( - ".github/workflows/deploy-config.yml", - """ -env: - GCP_INSTANCE: doscan-mainnet -""", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docker-compose-testnet.yml", "Compose structure", "duplicate" - ) - self.assert_diagnostic( - errors, ".github/workflows/deploy-config.yml", "workflow structure", "duplicate" - ) - - def test_unsupported_inline_environment_structure_fails_closed(self): - self.replace( - "docker-compose/docker-compose-beta.yml", - " environment:\n" - ' MICROSERVICE_METADATA_ENABLED: "true"\n' - ' MICROSERVICE_BENS_ENABLED: "false"\n' - ' MICROSERVICE_BENS_URL: ""\n' - ' MICROSERVICE_BENS_PROTOCOLS: ""', - " environment: {MICROSERVICE_METADATA_ENABLED: true}", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, "docker-compose-beta.yml", "Compose structure", "unsupported" - ) - - def test_commented_validator_command_is_not_an_active_workflow_step(self): - self.replace( - ".github/workflows/dependency-build.yml", - f" run: {VALIDATOR_COMMAND}", - f" # run: {VALIDATOR_COMMAND}", - ) - self.append( - ".github/workflows/dependency-build.yml", - f""" - decoy-job: - runs-on: ubuntu-latest - steps: - - run: {VALIDATOR_COMMAND} -""", - ) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - ".github/workflows/dependency-build.yml", - "active validator run step", - "missing", - ) - - def test_dependency_workflow_requires_the_complete_push_path_set(self): - path = ".github/workflows/dependency-build.yml" - original = (self.repo / path).read_text(encoding="utf-8") - for missing_path in REQUIRED_PUSH_PATHS: - with self.subTest(missing_path=missing_path): - line = f' - "{missing_path}"\n' - self.assertIn(line, original) - self.write(path, original.replace(line, "")) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, - path, - "required push paths", - missing_path, - ) - self.write(path, original) - - def test_validator_job_and_step_must_be_unconditional_and_blocking(self): - path = ".github/workflows/dependency-build.yml" - original = (self.repo / path).read_text(encoding="utf-8") - cases = ( - ( - " workflow-scripts:\n", - " workflow-scripts:\n if: false\n", - "job if: false", - ), - ( - " workflow-scripts:\n", - " workflow-scripts:\n continue-on-error: true\n", - "job continue-on-error: true", - ), - ( - " - name: Validate production documentation status\n", - " - name: Validate production documentation status\n" - " if: false\n", - "step if: false", - ), - ( - " - name: Validate production documentation status\n", - " - name: Validate production documentation status\n" - " continue-on-error: true\n", - "step continue-on-error: true", - ), - ) - for old, new, actual in cases: - with self.subTest(actual=actual): - self.assertIn(old, original) - self.write(path, original.replace(old, new, 1)) - - errors = self.module.validate_repository(self.repo) - - self.assert_diagnostic( - errors, path, "active validator run step", actual - ) - self.write(path, original) - - def test_yaml_block_scalars_may_contain_multiline_shell_quotes(self): - self.replace( - ".github/workflows/dependency-build.yml", - f" run: {VALIDATOR_COMMAND}", - " run: |\n" - " value=\"$(\n" - f" {VALIDATOR_COMMAND}\n" - " )\"", - ) - - self.assertEqual([], self.module.validate_repository(self.repo)) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/validate-docs-production-status.py b/.github/scripts/validate-docs-production-status.py deleted file mode 100644 index 5cc3dd35e131..000000000000 --- a/.github/scripts/validate-docs-production-status.py +++ /dev/null @@ -1,1297 +0,0 @@ -import re -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[2] -IMMUTABLE_IMAGE_RE = re.compile( - r"^(?P[^\s:@]+(?:/[^\s:@]+)*):" - r"(?P[^\s@]+)@sha256:(?P[0-9a-f]{64})$" -) -ENV_REFERENCE_RE = re.compile( - r"^\$\{(?P[A-Za-z_][A-Za-z0-9_]*)(?::-(?P.*))?\}$" -) -YAML_KEY_RE = re.compile(r"^(?P[A-Za-z0-9_.-]+):(?P.*)$") - -COMPOSE_FILES = ( - "docker-compose/docker-compose-mainnet.yml", - "docker-compose/docker-compose-testnet.yml", - "docker-compose/docker-compose-beta.yml", -) -STATUS_DOCUMENTS = ( - "docs/FEATURES.md", - "docs/CHANGELOG.md", - "docs/DOScan-ARCHITECTURE.md", - "docs/reports/doscan-frontend-env-audit.vi.html", -) -FEATURES_DOCUMENT = "docs/FEATURES.md" -CHANGELOG_DOCUMENT = "docs/CHANGELOG.md" -ARCHITECTURE_DOCUMENT = "docs/DOScan-ARCHITECTURE.md" -HTML_REPORT_DOCUMENT = "docs/reports/doscan-frontend-env-audit.vi.html" -COMMON_BLOCKSCOUT_ENV = "docker-compose/envs/common-blockscout.env" -DEPLOY_WORKFLOW = ".github/workflows/deploy-config.yml" -DEPENDENCY_WORKFLOW = ".github/workflows/dependency-build.yml" -VALIDATOR_COMMAND = "python .github/scripts/validate-docs-production-status.py" -GCP_KEYS = ( - "GCP_INSTANCE", - "GCP_ZONE", - "GCP_TESTNET_INSTANCE", - "GCP_TESTNET_ZONE", -) -PROTECTED_BACKEND_ENV = { - "MICROSERVICE_METADATA_ENABLED": "true", - "MICROSERVICE_BENS_ENABLED": "false", - "MICROSERVICE_BENS_URL": "", - "MICROSERVICE_BENS_PROTOCOLS": "", -} -BENS_KEYS = tuple(key for key in PROTECTED_BACKEND_ENV if "BENS" in key) -NAME_SERVICE_API_HOST = "NEXT_PUBLIC_NAME_SERVICE_API_HOST" -FRONTEND_ENV_FILES = ( - "docker-compose/envs/common-frontend.env", - "docker-compose/envs/common-frontend-scan.env", - "docker-compose/envs/common-frontend-testnet.env", - "docker-compose/envs/common-frontend-beta.env", -) -REQUIRED_DEPENDENCY_PUSH_PATHS = frozenset( - ( - *COMPOSE_FILES, - COMMON_BLOCKSCOUT_ENV, - "docker-compose/envs/common-blockscout-mainnet.env", - "docker-compose/envs/common-blockscout-testnet.env", - "docker-compose/envs/common-blockscout-beta.env", - "docker-compose/envs/common-frontend*.env", - *STATUS_DOCUMENTS, - DEPLOY_WORKFLOW, - DEPENDENCY_WORKFLOW, - ".github/scripts/validate-docs-production-status.py", - ".github/scripts/tests/**", - ) -) - - -class StructureError(ValueError): - pass - - -def diagnostic(path: Path, invariant: str, expected: object, actual: object) -> str: - return ( - f"{path.as_posix()}: {invariant}; " - f"expected {expected!r}; actual {actual!r}" - ) - - -def read_required(root: Path, relative_path: str) -> str: - return (root / relative_path).read_text(encoding="utf-8") - - -def strip_inline_comment(value: str, require_closed_quote: bool = False) -> str: - quote: str | None = None - escaped = False - for index, character in enumerate(value): - if escaped: - escaped = False - continue - if quote is not None: - if quote == '"' and character == "\\": - escaped = True - elif character == quote: - quote = None - continue - if character in ("'", '"'): - quote = character - elif character == "#" and (index == 0 or value[index - 1].isspace()): - return value[:index].rstrip() - if quote is not None and require_closed_quote: - raise StructureError("unterminated quoted value") - return value.rstrip() - - -def parse_scalar(value: str) -> str: - normalized = strip_inline_comment(value, require_closed_quote=True).strip() - if not normalized: - return "" - if normalized[0] in ("'", '"'): - quote = normalized[0] - if len(normalized) < 2 or normalized[-1] != quote: - raise StructureError("unsupported characters after quoted value") - return normalized[1:-1] - return normalized - - -def read_env(path: Path) -> dict[str, str]: - values: dict[str, str] = {} - for line_number, raw_line in enumerate( - path.read_text(encoding="utf-8").splitlines(), start=1 - ): - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, raw_value = line.split("=", 1) - key = key.strip() - if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) is None: - raise StructureError(f"invalid env key at line {line_number}: {key!r}") - values[key] = parse_scalar(raw_value) - return values - - -def yaml_lines(text: str) -> list[tuple[int, int, str]]: - lines: list[tuple[int, int, str]] = [] - for line_number, raw_line in enumerate(text.splitlines(), start=1): - if not raw_line.strip() or raw_line.lstrip().startswith("#"): - continue - prefix = raw_line[: len(raw_line) - len(raw_line.lstrip())] - if "\t" in prefix: - raise StructureError(f"tab indentation at line {line_number}") - content = strip_inline_comment(raw_line.lstrip()).rstrip() - if content: - lines.append((line_number, len(prefix), content)) - return lines - - -def split_yaml_mapping(content: str) -> tuple[str, str] | None: - match = YAML_KEY_RE.fullmatch(content) - if match is None: - return None - return match.group("key"), match.group("value").strip() - - -def mapping_block( - lines: list[tuple[int, int, str]], - key: str, - indent: int, - start: int = 0, - end: int | None = None, -) -> tuple[int, int]: - if end is None: - end = len(lines) - occurrences: list[int] = [] - for index in range(start, end): - _, line_indent, content = lines[index] - entry = split_yaml_mapping(content) - if line_indent == indent and entry is not None and entry[0] == key: - occurrences.append(index) - if not occurrences: - raise StructureError(f"missing mapping {key!r}") - if len(occurrences) > 1: - raise StructureError(f"duplicate mapping {key!r}") - mapping_index = occurrences[0] - _, _, content = lines[mapping_index] - _, value = split_yaml_mapping(content) - if value: - raise StructureError(f"unsupported inline value for mapping {key!r}") - block_end = end - for index in range(mapping_index + 1, end): - if lines[index][1] <= indent: - block_end = index - break - return mapping_index + 1, block_end - - -def direct_mapping_children( - lines: list[tuple[int, int, str]], - start: int, - end: int, - indent: int, - context: str, -) -> dict[str, tuple[str, int, int]]: - positions: list[tuple[str, str, int]] = [] - for index in range(start, end): - _, line_indent, content = lines[index] - if line_indent != indent: - continue - entry = split_yaml_mapping(content) - if entry is None: - raise StructureError(f"unsupported {context} entry at line {lines[index][0]}") - positions.append((entry[0], entry[1], index)) - - children: dict[str, tuple[str, int, int]] = {} - for position, (key, value, index) in enumerate(positions): - if key in children: - raise StructureError(f"duplicate {context} entry {key!r}") - child_end = end - if position + 1 < len(positions): - child_end = positions[position + 1][2] - children[key] = (value, index + 1, child_end) - return children - - -def parse_env_file_property( - value: str, - lines: list[tuple[int, int, str]], - start: int, - end: int, -) -> list[str]: - if value: - if value.startswith(("[", "{")): - raise StructureError("unsupported inline env_file structure") - return [parse_scalar(value)] - - env_files: list[str] = [] - for index in range(start, end): - line_number, indent, content = lines[index] - if indent != 6 or not content.startswith("- "): - raise StructureError(f"unsupported env_file entry at line {line_number}") - item = content[2:].strip() - if not item or split_yaml_mapping(item) is not None: - raise StructureError(f"unsupported env_file entry at line {line_number}") - env_files.append(parse_scalar(item)) - if not env_files: - raise StructureError("empty env_file list") - return env_files - - -def parse_environment_property( - value: str, - lines: list[tuple[int, int, str]], - start: int, - end: int, -) -> dict[str, str | None]: - if value: - raise StructureError("unsupported inline environment structure") - - environment: dict[str, str | None] = {} - style: str | None = None - for index in range(start, end): - line_number, indent, content = lines[index] - if indent != 6: - raise StructureError(f"unsupported environment entry at line {line_number}") - if content.startswith("- "): - if style == "mapping": - raise StructureError("mixed environment mapping and list structures") - style = "list" - item = parse_scalar(content[2:]) - if "=" in item: - key, raw_value = item.split("=", 1) - parsed_value: str | None = raw_value - else: - key = item - parsed_value = None - else: - if style == "list": - raise StructureError("mixed environment mapping and list structures") - style = "mapping" - entry = split_yaml_mapping(content) - if entry is None: - raise StructureError(f"unsupported environment entry at line {line_number}") - key, raw_value = entry - parsed_value = parse_scalar(raw_value) if raw_value else None - key = key.strip() - if key in environment: - raise StructureError(f"duplicate environment key {key!r}") - environment[key] = parsed_value - return environment - - -def parse_compose_services(compose: str) -> dict[str, dict[str, object]]: - lines = yaml_lines(compose) - services_start, services_end = mapping_block(lines, "services", 0) - service_children = direct_mapping_children( - lines, services_start, services_end, 2, "service" - ) - services: dict[str, dict[str, object]] = {} - for service, (service_value, service_start, service_end) in service_children.items(): - if service_value: - raise StructureError(f"unsupported service structure for {service!r}") - properties = direct_mapping_children( - lines, service_start, service_end, 4, f"{service} property" - ) - parsed: dict[str, object] = { - "image": None, - "env_files": [], - "environment": {}, - } - if "image" in properties: - image_value, _, _ = properties["image"] - if not image_value: - raise StructureError(f"unsupported empty image for service {service!r}") - parsed["image"] = parse_scalar(image_value) - if "env_file" in properties: - value, start, end = properties["env_file"] - parsed["env_files"] = parse_env_file_property(value, lines, start, end) - if "environment" in properties: - value, start, end = properties["environment"] - parsed["environment"] = parse_environment_property( - value, lines, start, end - ) - services[service] = parsed - return services - - -def read_workflow_env(text: str, keys: tuple[str, ...]) -> dict[str, str]: - lines = yaml_lines(text) - env_start, env_end = mapping_block(lines, "env", 0) - children = direct_mapping_children(lines, env_start, env_end, 2, "workflow env") - values: dict[str, str] = {} - for key in keys: - if key not in children: - continue - value, child_start, child_end = children[key] - if not value or child_start != child_end: - raise StructureError(f"unsupported workflow env value for {key!r}") - values[key] = parse_scalar(value) - return values - - -def extract_service_image(compose: str, service: str) -> str | None: - service_data = parse_compose_services(compose).get(service) - return None if service_data is None else service_data["image"] - - -def parse_immutable_image(image: str) -> dict[str, str] | None: - match = IMMUTABLE_IMAGE_RE.fullmatch(image) - return match.groupdict() if match else None - - -def backend_runtime_version(image: str) -> str | None: - parsed = parse_immutable_image(image) - if parsed is None: - return None - return f"v{parsed['tag'].replace('.commit.', '.+commit.')}" - - -def markdown_section(text: str, heading_pattern: str) -> str: - lines = text.splitlines() - heading_re = re.compile(heading_pattern) - matches = [index for index, line in enumerate(lines) if heading_re.fullmatch(line)] - if not matches: - raise StructureError(f"missing Markdown section matching {heading_pattern!r}") - if len(matches) > 1: - raise StructureError(f"duplicate Markdown section matching {heading_pattern!r}") - start = matches[0] - level = len(lines[start]) - len(lines[start].lstrip("#")) - end = len(lines) - for index in range(start + 1, len(lines)): - match = re.match(r"^(?P#+)\s+", lines[index]) - if match is not None and len(match.group("marks")) <= level: - end = index - break - return "\n".join(lines[start + 1 : end]) - - -def markdown_table( - section: str, expected_header: tuple[str, ...] -) -> list[dict[str, str]]: - lines = section.splitlines() - for index, line in enumerate(lines): - if not line.strip().startswith("|"): - continue - header = tuple(markdown_cells(line)) - if header != expected_header: - continue - if index + 1 >= len(lines): - break - separator = markdown_cells(lines[index + 1]) - if len(separator) != len(header) or not all( - re.fullmatch(r":?-{3,}:?", cell) for cell in separator - ): - raise StructureError(f"invalid Markdown table separator for {header!r}") - rows: list[dict[str, str]] = [] - for row_line in lines[index + 2 :]: - if not row_line.strip().startswith("|"): - break - cells = markdown_cells(row_line) - if len(cells) != len(header): - raise StructureError(f"invalid Markdown table row for {header!r}") - rows.append(dict(zip(header, cells))) - return rows - raise StructureError(f"missing Markdown table with header {expected_header!r}") - - -def markdown_cells(line: str) -> list[str]: - stripped = line.strip() - if not stripped.startswith("|") or not stripped.endswith("|"): - return [] - cells: list[str] = [] - for cell in stripped[1:-1].split("|"): - value = cell.strip() - if len(value) >= 2 and value.startswith("`") and value.endswith("`"): - value = value[1:-1] - cells.append(value) - return cells - - -def rows_by_key( - rows: list[dict[str, str]], key_column: str, context: str -) -> dict[str, dict[str, str]]: - keyed: dict[str, dict[str, str]] = {} - for row in rows: - key = row[key_column] - if key in keyed: - raise StructureError(f"duplicate {context} row {key!r}") - keyed[key] = row - return keyed - - -def append_mismatch( - errors: list[str], - path: Path, - invariant: str, - expected: object, - actual: object, -) -> None: - if actual != expected: - errors.append(diagnostic(path, invariant, expected, actual)) - - -def validate_features_document( - document: str, - path: Path, - backend_image: str | None, - frontend_version: str | None, - errors: list[str], -) -> None: - try: - runtime = markdown_section(document, r"## Runtime Baseline") - runtime_rows = rows_by_key( - markdown_table( - runtime, - ( - "Environment", - "Explorer", - "Chain ID", - "Frontend", - "Backend", - "Runtime status", - ), - ), - "Environment", - "runtime baseline", - ) - except StructureError as error: - errors.append(diagnostic(path, "current runtime structure", "supported table", str(error))) - else: - backend_version = ( - backend_runtime_version(backend_image) if backend_image is not None else None - ) - for environment in ("Mainnet", "Testnet"): - row = runtime_rows.get(environment, {}) - if frontend_version is not None: - append_mismatch( - errors, - path, - f"frontend version ({environment})", - frontend_version, - row.get("Frontend", "missing"), - ) - if backend_version is not None: - append_mismatch( - errors, - path, - f"backend version ({environment})", - backend_version, - row.get("Backend", "missing"), - ) - pin_match = re.search( - r"Both production environments pin the custom backend image below:\s*" - r"```text\s*\n(?P[^\n]+)\n```", - runtime, - ) - if backend_image is not None: - append_mismatch( - errors, - path, - "backend image", - backend_image, - pin_match.group("image").strip() if pin_match else "missing", - ) - - try: - integrations = markdown_section( - document, r"### Backend Integrations and Services" - ) - integration_rows = rows_by_key( - markdown_table( - integrations, - ("Service or integration", "Mainnet", "Testnet", "Runtime path"), - ), - "Service or integration", - "backend integration", - ) - metadata_row = integration_rows.get("Metadata Service", {}) - metadata_actual = ( - f"Mainnet={metadata_row.get('Mainnet', 'missing')}, " - f"Testnet={metadata_row.get('Testnet', 'missing')}" - ) - append_mismatch( - errors, - path, - "metadata documentation status", - "Mainnet=Enabled, Testnet=Enabled", - metadata_actual, - ) - except StructureError as error: - errors.append( - diagnostic(path, "metadata documentation status", "Enabled", str(error)) - ) - - try: - disabled = markdown_section( - document, r"## Deliberately Disabled or Blocked Features" - ) - disabled_rows = rows_by_key( - markdown_table( - disabled, ("Feature", "Status", "Reason or prerequisite") - ), - "Feature", - "disabled feature", - ) - bens_actual = disabled_rows.get("BENS / name service", {}).get( - "Status", "missing" - ) - if bens_actual.lower() not in ("disabled", "not configured"): - errors.append( - diagnostic( - path, - "BENS documentation status", - "Disabled or Not configured", - bens_actual, - ) - ) - except StructureError as error: - errors.append( - diagnostic( - path, - "BENS documentation status", - "Disabled or Not configured", - str(error), - ) - ) - - -def validate_changelog_document( - document: str, - path: Path, - backend_image: str | None, - frontend_version: str | None, - errors: list[str], -) -> None: - try: - baseline = markdown_section( - document, - r"## \[[^]]+\] - Production Documentation and Runtime Baseline", - ) - except StructureError as error: - errors.append( - diagnostic(path, "current production baseline", "labeled release section", str(error)) - ) - return - - version_match = re.search( - r"^- Mainnet and Testnet run Frontend `(?P[^`]+)` and " - r"custom Backend `(?P[^`]+)` on GCP\.$", - baseline, - re.MULTILINE, - ) - pin_match = re.search( - r"^- Both production Compose files pin `(?P[^`]+)`\.$", - baseline, - re.MULTILINE, - ) - status_match = re.search( - r"^- Corrected feature status: the admin panel and BENS are " - r"(?Penabled|disabled|not configured), while Metadata Service is " - r"(?Penabled|disabled)\.$", - baseline, - re.MULTILINE | re.IGNORECASE, - ) - - if frontend_version is not None: - append_mismatch( - errors, - path, - "frontend version", - frontend_version, - version_match.group("frontend") if version_match else "missing", - ) - if backend_image is not None: - append_mismatch( - errors, - path, - "backend image", - backend_image, - pin_match.group("backend") if pin_match else "missing", - ) - expected_backend_version = backend_runtime_version(backend_image) - append_mismatch( - errors, - path, - "backend version", - expected_backend_version, - version_match.group("backend") if version_match else "missing", - ) - - metadata_actual = ( - status_match.group("metadata").capitalize() if status_match else "missing" - ) - append_mismatch( - errors, - path, - "metadata documentation status", - "Enabled", - metadata_actual, - ) - bens_actual = status_match.group("bens").capitalize() if status_match else "missing" - if bens_actual.lower() not in ("disabled", "not configured"): - errors.append( - diagnostic( - path, - "BENS documentation status", - "Disabled or Not configured", - bens_actual, - ) - ) - - -def validate_architecture_document( - document: str, - path: Path, - backend_image: str | None, - frontend_image: str | None, - workflow_env: dict[str, str], - errors: list[str], -) -> None: - try: - deployed = markdown_section(document, r"## Deployed Environments") - deployed_rows = rows_by_key( - markdown_table( - deployed, - ( - "Environment", - "Public origin", - "Chain ID", - "GCP host", - "Zone", - "Deployment path", - ), - ), - "Environment", - "deployed environment", - ) - except StructureError as error: - errors.append( - diagnostic(path, "deployed environment structure", "supported table", str(error)) - ) - else: - topology_fields = ( - ("Mainnet", "GCP host", "GCP_INSTANCE"), - ("Mainnet", "Zone", "GCP_ZONE"), - ("Testnet", "GCP host", "GCP_TESTNET_INSTANCE"), - ("Testnet", "Zone", "GCP_TESTNET_ZONE"), - ) - for environment, column, key in topology_fields: - if key not in workflow_env: - continue - actual = deployed_rows.get(environment, {}).get(column, "missing") - append_mismatch(errors, path, key, workflow_env[key], actual) - - try: - runtime = markdown_section(document, r"## Runtime Versions") - runtime_rows = rows_by_key( - markdown_table(runtime, ("Component", "Production version")), - "Component", - "runtime version", - ) - except StructureError as error: - errors.append(diagnostic(path, "runtime version structure", "supported table", str(error))) - else: - if frontend_image is not None: - append_mismatch( - errors, - path, - "frontend image", - frontend_image, - runtime_rows.get("Frontend", {}).get("Production version", "missing"), - ) - if backend_image is not None: - append_mismatch( - errors, - path, - "backend image", - backend_image, - runtime_rows.get("Backend", {}).get("Production version", "missing"), - ) - - try: - metadata = markdown_section(document, r"### Metadata Service") - except StructureError as error: - errors.append( - diagnostic(path, "metadata documentation status", "Enabled", str(error)) - ) - errors.append( - diagnostic( - path, - "BENS documentation status", - "Disabled or Not configured", - str(error), - ) - ) - return - - metadata_match = re.search( - r"^\s*MICROSERVICE_METADATA_ENABLED\s*=\s*(?P[^\n]+)$", - metadata, - re.MULTILINE, - ) - try: - metadata_actual = ( - parse_scalar(metadata_match.group("value")) if metadata_match else "missing" - ) - except StructureError as error: - metadata_actual = str(error) - append_mismatch( - errors, - path, - "metadata documentation status", - "true", - metadata_actual, - ) - - bens_match = re.search( - r"\bBENS is (?Pnot configured|disabled|enabled|configured)\.", - metadata, - re.IGNORECASE, - ) - bens_actual = bens_match.group("status").capitalize() if bens_match else "missing" - if bens_actual.lower() not in ("disabled", "not configured"): - errors.append( - diagnostic( - path, - "BENS documentation status", - "Disabled or Not configured", - bens_actual, - ) - ) - - -def html_code_claim(document: str, element_id: str) -> str: - match = re.search( - rf']*\bid=["\']{re.escape(element_id)}["\'])[^>]*>' - r'.*?(?P[^<]+).*?

', - document, - re.DOTALL, - ) - if match is None: - return "missing" - return match.group("value").strip() - - -def validate_html_report( - document: str, - path: Path, - backend_image: str | None, - errors: list[str], -) -> None: - if backend_image is None: - return - append_mismatch( - errors, - path, - "backend version", - backend_runtime_version(backend_image), - html_code_claim(document, "production-backend-version"), - ) - append_mismatch( - errors, - path, - "backend image", - backend_image, - html_code_claim(document, "production-backend-image"), - ) - - -def local_env_reference( - compose_path: str, reference: str -) -> tuple[str | None, bool]: - expression = ENV_REFERENCE_RE.fullmatch(reference) - unresolved = expression is not None or "$" in reference - candidate = expression.group("default") if expression is not None else reference - if not candidate or "$" in candidate or candidate.startswith(("/", "\\")): - return None, unresolved or bool(candidate) - compose_directory = Path(compose_path).parent - relative = (compose_directory / Path(candidate)).as_posix() - while relative.startswith("./"): - relative = relative[2:] - if relative == ".." or relative.startswith("../"): - return None, True - return relative, unresolved - - -def protected_invariant(key: str) -> str: - if key == "MICROSERVICE_METADATA_ENABLED": - return "metadata enabled" - return f"BENS disabled ({key})" - - -def validate_protected_values( - values: dict[str, str | None], - path: Path, - errors: list[str], - require_metadata: bool, - invariant_prefix: str = "", -) -> None: - for key, expected in PROTECTED_BACKEND_ENV.items(): - actual = values.get(key) - if actual is None and (not require_metadata or key != "MICROSERVICE_METADATA_ENABLED"): - continue - if actual != expected: - invariant = protected_invariant(key) - if invariant_prefix: - invariant = f"{invariant_prefix} {invariant}" - errors.append( - diagnostic( - path, - invariant, - expected, - actual if actual is not None else "missing", - ) - ) - - -def validate_backend_sources( - root: Path, - compose_path: str, - backend: dict[str, object], - env_cache: dict[str, dict[str, str] | None], - errors: list[str], -) -> None: - effective_values: dict[str, str | None] = {} - canonical_source_referenced = False - for reference in backend["env_files"]: - local_path, unresolved = local_env_reference(compose_path, reference) - if local_path == COMMON_BLOCKSCOUT_ENV and not unresolved: - canonical_source_referenced = True - if local_path is None: - for key in PROTECTED_BACKEND_ENV: - effective_values[key] = "unresolved" - continue - if local_path not in env_cache: - try: - env_cache[local_path] = read_env(root / local_path) - except FileNotFoundError: - env_cache[local_path] = None - errors.append( - diagnostic( - Path(local_path), "missing required file", "present", "missing" - ) - ) - except StructureError as error: - env_cache[local_path] = None - errors.append( - diagnostic(Path(local_path), "env structure", "supported env", str(error)) - ) - values = env_cache.get(local_path) - if values is not None and local_path != COMMON_BLOCKSCOUT_ENV: - validate_protected_values(values, Path(local_path), errors, False) - if unresolved: - for key in PROTECTED_BACKEND_ENV: - effective_values[key] = "unresolved" - elif values is not None: - for key in PROTECTED_BACKEND_ENV: - if key in values: - effective_values[key] = values[key] - - inline_environment = backend["environment"] - validate_protected_values( - inline_environment, Path(compose_path), errors, require_metadata=False - ) - for key in PROTECTED_BACKEND_ENV: - if key in inline_environment: - inline_value = inline_environment[key] - effective_values[key] = ( - inline_value if inline_value is not None else "unresolved" - ) - - if not canonical_source_referenced: - errors.append( - diagnostic( - Path(compose_path), - "canonical backend env source", - COMMON_BLOCKSCOUT_ENV, - "missing", - ) - ) - - validate_protected_values( - effective_values, - Path(compose_path), - errors, - require_metadata=True, - invariant_prefix="effective", - ) - - -def dependency_workflow_errors(text: str, path: Path) -> list[str]: - errors: list[str] = [] - lines = yaml_lines(text) - on_start, on_end = mapping_block(lines, "on", 0) - push_start, push_end = mapping_block(lines, "push", 2, on_start, on_end) - paths_start, paths_end = mapping_block( - lines, "paths", 4, push_start, push_end - ) - actual_paths: set[str] = set() - for index in range(paths_start, paths_end): - line_number, indent, content = lines[index] - if indent != 6 or not content.startswith("- "): - raise StructureError(f"unsupported push path at line {line_number}") - actual_paths.add(parse_scalar(content[2:])) - missing_paths = sorted(REQUIRED_DEPENDENCY_PUSH_PATHS - actual_paths) - if missing_paths: - errors.append( - diagnostic( - path, - "required push paths", - "complete guard source set", - f"missing: {', '.join(missing_paths)}", - ) - ) - - jobs_start, jobs_end = mapping_block(lines, "jobs", 0) - workflow_start, workflow_end = mapping_block( - lines, "workflow-scripts", 2, jobs_start, jobs_end - ) - job_properties = direct_mapping_children( - lines, - workflow_start, - workflow_end, - 4, - "workflow-scripts job property", - ) - if "steps" not in job_properties: - raise StructureError("missing workflow-scripts steps mapping") - steps_value, steps_start, steps_end = job_properties["steps"] - if steps_value: - raise StructureError("unsupported inline workflow-scripts steps") - - def guard_control_reasons( - properties: dict[str, tuple[str, int, int]], scope: str - ) -> list[str]: - reasons: list[str] = [] - controls = (("if", "true"), ("continue-on-error", "false")) - for key, safe_value in controls: - if key not in properties: - continue - value, child_start, child_end = properties[key] - if child_start != child_end: - raise StructureError(f"unsupported {scope} {key} structure") - actual = parse_scalar(value).lower() - if actual != safe_value: - reasons.append(f"{scope} {key}: {actual or 'missing'}") - return reasons - - job_reasons = guard_control_reasons(job_properties, "job") - step_markers = [ - index - for index in range(steps_start, steps_end) - if lines[index][1] == 6 - ] - for index in step_markers: - if not lines[index][2].startswith("- "): - raise StructureError( - f"unsupported workflow step at line {lines[index][0]}" - ) - - active_command = False - inactive_reasons: list[str] = [] - for marker_position, marker_index in enumerate(step_markers): - step_end = steps_end - if marker_position + 1 < len(step_markers): - step_end = step_markers[marker_position + 1] - inline_entry = split_yaml_mapping(lines[marker_index][2][2:].strip()) - if inline_entry is None: - raise StructureError( - f"unsupported workflow step at line {lines[marker_index][0]}" - ) - step_properties = direct_mapping_children( - lines, - marker_index + 1, - step_end, - 8, - "workflow step property", - ) - inline_key, inline_value = inline_entry - if inline_key in step_properties: - raise StructureError(f"duplicate workflow step property {inline_key!r}") - step_properties[inline_key] = ( - inline_value, - marker_index + 1, - marker_index + 1, - ) - if "run" not in step_properties: - continue - run_value, run_start, run_end = step_properties["run"] - if run_value in ("|", ">", "|-", ">-"): - block_commands = [ - lines[index][2].strip() - for index in range(run_start, run_end) - if lines[index][1] > 8 - ] - runs_validator = VALIDATOR_COMMAND in block_commands - else: - if run_start != run_end: - raise StructureError("unsupported validator run step structure") - runs_validator = parse_scalar(run_value) == VALIDATOR_COMMAND - if not runs_validator: - continue - step_reasons = guard_control_reasons(step_properties, "step") - if not job_reasons and not step_reasons: - active_command = True - break - inactive_reasons.extend(job_reasons + step_reasons) - if not active_command: - actual = ", ".join(dict.fromkeys(inactive_reasons)) or "missing" - errors.append( - diagnostic(path, "active validator run step", VALIDATOR_COMMAND, actual) - ) - return errors - - -def validate_repository(root: Path) -> list[str]: - errors: list[str] = [] - images: dict[str, dict[str, str]] = {"backend": {}, "frontend": {}} - compose_services: dict[str, dict[str, dict[str, object]]] = {} - documents: dict[str, str] = {} - - for compose_path in COMPOSE_FILES: - try: - compose = read_required(root, compose_path) - except FileNotFoundError: - errors.append( - diagnostic( - Path(compose_path), "missing required file", "present", "missing" - ) - ) - continue - try: - services = parse_compose_services(compose) - except StructureError as error: - errors.append( - diagnostic( - Path(compose_path), - "Compose structure", - "one supported top-level services mapping", - str(error), - ) - ) - continue - compose_services[compose_path] = services - for service in images: - image = services.get(service, {}).get("image") - if image is None: - errors.append( - diagnostic( - Path(compose_path), - f"{service} image", - "service image key", - "missing", - ) - ) - elif parse_immutable_image(image) is None: - errors.append( - diagnostic( - Path(compose_path), - f"{service} immutable image pin", - "tag@sha256:<64 lowercase hexadecimal characters>", - image, - ) - ) - else: - images[service][compose_path] = image - - canonical_images: dict[str, str] = {} - mainnet_path = COMPOSE_FILES[0] - for service, service_images in images.items(): - canonical_image = service_images.get(mainnet_path) - if canonical_image is None: - continue - canonical_images[service] = canonical_image - for compose_path, image in service_images.items(): - if image != canonical_image: - errors.append( - diagnostic( - Path(compose_path), f"{service} image", canonical_image, image - ) - ) - - common_env_values: dict[str, str] | None = None - try: - common_env_values = read_env(root / COMMON_BLOCKSCOUT_ENV) - except FileNotFoundError: - errors.append( - diagnostic( - Path(COMMON_BLOCKSCOUT_ENV), "missing required file", "present", "missing" - ) - ) - except StructureError as error: - errors.append( - diagnostic( - Path(COMMON_BLOCKSCOUT_ENV), "env structure", "supported env", str(error) - ) - ) - else: - validate_protected_values( - common_env_values, - Path(COMMON_BLOCKSCOUT_ENV), - errors, - require_metadata=True, - ) - - env_cache: dict[str, dict[str, str] | None] = { - COMMON_BLOCKSCOUT_ENV: common_env_values - } - for compose_path, services in compose_services.items(): - backend = services.get("backend") - if backend is not None: - validate_backend_sources( - root, compose_path, backend, env_cache, errors - ) - - for frontend_env_file in FRONTEND_ENV_FILES: - try: - frontend_env = read_env(root / frontend_env_file) - except FileNotFoundError: - errors.append( - diagnostic( - Path(frontend_env_file), "missing required file", "present", "missing" - ) - ) - continue - except StructureError as error: - errors.append( - diagnostic( - Path(frontend_env_file), "env structure", "supported env", str(error) - ) - ) - continue - name_service_api_host = frontend_env.get(NAME_SERVICE_API_HOST) - if name_service_api_host: - errors.append( - diagnostic( - Path(frontend_env_file), - f"{NAME_SERVICE_API_HOST} disabled", - "unset", - name_service_api_host, - ) - ) - - workflow_env: dict[str, str] = {} - try: - workflow_text = read_required(root, DEPLOY_WORKFLOW) - except FileNotFoundError: - errors.append( - diagnostic(Path(DEPLOY_WORKFLOW), "missing required file", "present", "missing") - ) - else: - try: - workflow_env = read_workflow_env(workflow_text, GCP_KEYS) - except StructureError as error: - errors.append( - diagnostic( - Path(DEPLOY_WORKFLOW), - "workflow structure", - "one supported top-level env mapping", - str(error), - ) - ) - else: - for key in GCP_KEYS: - if not workflow_env.get(key): - errors.append( - diagnostic( - Path(DEPLOY_WORKFLOW), - key, - "defined workflow value", - "missing", - ) - ) - - for document_path in STATUS_DOCUMENTS: - try: - documents[document_path] = read_required(root, document_path) - except FileNotFoundError: - errors.append( - diagnostic( - Path(document_path), "missing required file", "present", "missing" - ) - ) - - backend_image = canonical_images.get("backend") - frontend_image = canonical_images.get("frontend") - frontend_version = None - if frontend_image is not None: - parsed_frontend = parse_immutable_image(frontend_image) - if parsed_frontend is not None: - frontend_version = parsed_frontend["tag"] - - if FEATURES_DOCUMENT in documents: - validate_features_document( - documents[FEATURES_DOCUMENT], - Path(FEATURES_DOCUMENT), - backend_image, - frontend_version, - errors, - ) - if CHANGELOG_DOCUMENT in documents: - validate_changelog_document( - documents[CHANGELOG_DOCUMENT], - Path(CHANGELOG_DOCUMENT), - backend_image, - frontend_version, - errors, - ) - if ARCHITECTURE_DOCUMENT in documents: - validate_architecture_document( - documents[ARCHITECTURE_DOCUMENT], - Path(ARCHITECTURE_DOCUMENT), - backend_image, - frontend_image, - workflow_env, - errors, - ) - if HTML_REPORT_DOCUMENT in documents: - validate_html_report( - documents[HTML_REPORT_DOCUMENT], - Path(HTML_REPORT_DOCUMENT), - backend_image, - errors, - ) - - try: - dependency_workflow = read_required(root, DEPENDENCY_WORKFLOW) - except FileNotFoundError: - errors.append( - diagnostic( - Path(DEPENDENCY_WORKFLOW), "missing required file", "present", "missing" - ) - ) - else: - try: - errors.extend( - dependency_workflow_errors( - dependency_workflow, Path(DEPENDENCY_WORKFLOW) - ) - ) - except StructureError as error: - errors.append( - diagnostic( - Path(DEPENDENCY_WORKFLOW), - "workflow structure", - "supported workflow-scripts job and push paths", - str(error), - ) - ) - - return errors - - -def main() -> int: - errors = validate_repository(ROOT) - if errors: - print("Production documentation drift validation failed:") - for error in errors: - print(f"- {error}") - return 1 - print("Production documentation drift validation passed.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/workflows/dependency-build.yml b/.github/workflows/dependency-build.yml index 6890bcfc9a82..69f2b5aa4d38 100644 --- a/.github/workflows/dependency-build.yml +++ b/.github/workflows/dependency-build.yml @@ -13,24 +13,12 @@ on: - "apps/**/yarn.lock" - "docker/Dockerfile" - "docker/oldUI.Dockerfile" - - "docker-compose/docker-compose-mainnet.yml" - - "docker-compose/docker-compose-testnet.yml" - - "docker-compose/docker-compose-beta.yml" - - "docker-compose/envs/common-blockscout.env" - "docker-compose/envs/common-blockscout-mainnet.env" - "docker-compose/envs/common-blockscout-testnet.env" - - "docker-compose/envs/common-blockscout-beta.env" - - "docker-compose/envs/common-frontend*.env" - - "docs/FEATURES.md" - - "docs/CHANGELOG.md" - - "docs/DOScan-ARCHITECTURE.md" - - "docs/reports/doscan-frontend-env-audit.vi.html" - - ".github/workflows/deploy-config.yml" - ".github/workflows/dependency-build.yml" - ".github/workflows/publish-regular-docker-image-on-demand.yml" - ".github/workflows/sync-upstream.yml" - ".github/scripts/check-upstream-sync.py" - - ".github/scripts/validate-docs-production-status.py" - ".github/scripts/tests/**" workflow_dispatch: @@ -57,9 +45,6 @@ jobs: - name: Test upstream sync state detection run: python -m unittest discover -s .github/scripts/tests -p 'test_*.py' -v - - name: Validate production documentation status - run: python .github/scripts/validate-docs-production-status.py - token-lists: name: Validate DOS Chain token list reference runs-on: ubuntu-latest diff --git a/docs/superpowers/plans/2026-08-04-docs-production-status-drift-guard.md b/docs/superpowers/plans/2026-08-04-docs-production-status-drift-guard.md deleted file mode 100644 index fbbae2a693f3..000000000000 --- a/docs/superpowers/plans/2026-08-04-docs-production-status-drift-guard.md +++ /dev/null @@ -1,305 +0,0 @@ -# Production Status Documentation Drift Guard Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add an offline CI validator that rejects production documentation when its critical deployment status disagrees with Compose, environment, or deployment workflow configuration. - -**Architecture:** A Python standard-library script reads three Compose files, the active backend and frontend environment files, and the deployment workflow. It derives immutable image pins, required Metadata and BENS states, and GCP topology, then checks the three Markdown production status documents and returns every detected error in one run. - -**Tech Stack:** Python 3, `unittest`, GitHub Actions YAML, Markdown source documents. - -## Global Constraints - -- Treat Compose, environment files, and `.github/workflows/deploy-config.yml` as canonical. -- Validate only critical production invariants: Backend and Frontend pins, Metadata enabled, BENS disabled, and Mainnet/Testnet GCP topology. -- Validate Mainnet, Testnet, and Beta image equality. -- Use only the Python standard library and make no network requests. -- Aggregate diagnostics with file, invariant, expected value, and actual value. -- Keep the Blockscout v11.2.4 upgrade outside this change. -- Preserve all existing deployment and environment validation behavior. - ---- - -### Task 1: Parse canonical deployment state and validate immutable image pins - -**Files:** -- Create: `.github/scripts/validate-docs-production-status.py` -- Create: `.github/scripts/tests/test_validate_docs_production_status.py` - -**Interfaces:** -- Consumes: A repository root as `pathlib.Path`. -- Produces: `validate_repository(root: Path) -> list[str]` and `main() -> int`. -- Produces: Helper functions `read_required`, `read_env`, `extract_service_image`, `parse_immutable_image`, and `backend_runtime_version`. - -- [ ] **Step 1: Write baseline and image drift tests** - -Create a temporary repository fixture containing minimal Mainnet, Testnet, and Beta Compose files. Each file must define the same immutable Backend and Frontend images. Create synchronized Markdown fixtures and assert: - -```python -def test_synchronized_repository_passes(self): - self.assertEqual([], self.module.validate_repository(self.repo)) - -def test_backend_digest_drift_names_the_compose_file(self): - self.replace( - "docker-compose/docker-compose-testnet.yml", - BACKEND_IMAGE, - BACKEND_IMAGE_WITH_DIFFERENT_DIGEST, - ) - errors = self.module.validate_repository(self.repo) - self.assertTrue(any("docker-compose-testnet.yml" in error for error in errors)) - self.assertTrue(any("backend image" in error for error in errors)) - -def test_frontend_version_drift_names_the_document(self): - self.replace("docs/FEATURES.md", "2.10.0", "2.9.0") - errors = self.module.validate_repository(self.repo) - self.assertTrue(any("docs/FEATURES.md" in error for error in errors)) - self.assertTrue(any("frontend version" in error for error in errors)) -``` - -- [ ] **Step 2: Run the focused tests and verify RED** - -Run: - -```text -python -m unittest .github/scripts/tests/test_validate_docs_production_status.py -v -``` - -Expected: import or attribute failure because the validator does not exist yet. - -- [ ] **Step 3: Implement minimal source and image validation** - -Implement these exact public interfaces: - -```python -def validate_repository(root: Path) -> list[str]: - errors: list[str] = [] - # Read required files, derive canonical state, and append all mismatches. - return errors - -def main() -> int: - errors = validate_repository(ROOT) - if errors: - print("Production documentation drift validation failed:") - for error in errors: - print(f"- {error}") - return 1 - print("Production documentation drift validation passed.") - return 0 -``` - -Parse each Compose file by locating the requested service block and its `image:` key. Accept an immutable image only when it matches: - -```python -IMMUTABLE_IMAGE_RE = re.compile( - r"^(?P[^\s:@]+(?:/[^\s:@]+)*):" - r"(?P[^\s@]+)@sha256:(?P[0-9a-f]{64})$" -) -``` - -Require the same Backend image and the same Frontend image in all three Compose files. Require the full Backend pin in all three production status documents, the Frontend tag version in all three documents, and the full Frontend pin in `docs/DOScan-ARCHITECTURE.md`. - -- [ ] **Step 4: Run the focused tests and verify GREEN** - -Run: - -```text -python -m unittest .github/scripts/tests/test_validate_docs_production_status.py -v -``` - -Expected: baseline and image drift tests pass. - -- [ ] **Step 5: Commit the image invariant slice** - -```text -git add .github/scripts/validate-docs-production-status.py .github/scripts/tests/test_validate_docs_production_status.py -git commit -m "ci: validate production image documentation" -``` - ---- - -### Task 2: Validate Metadata, BENS, GCP topology, and aggregated diagnostics - -**Files:** -- Modify: `.github/scripts/validate-docs-production-status.py` -- Modify: `.github/scripts/tests/test_validate_docs_production_status.py` - -**Interfaces:** -- Consumes: `validate_repository(root: Path) -> list[str]` from Task 1. -- Produces: `read_env(path: Path) -> dict[str, str]`, `read_workflow_env(text: str, keys: tuple[str, ...]) -> dict[str, str]`, and formatted diagnostic strings. - -- [ ] **Step 1: Add failing state and error aggregation tests** - -Add these behaviors to the temporary fixture suite: - -```python -def test_metadata_must_remain_enabled(self): - self.replace( - "docker-compose/envs/common-blockscout.env", - "MICROSERVICE_METADATA_ENABLED=true", - "MICROSERVICE_METADATA_ENABLED=false", - ) - errors = self.module.validate_repository(self.repo) - self.assertTrue(any("metadata enabled" in error for error in errors)) - -def test_bens_configuration_is_rejected(self): - self.append( - "docker-compose/envs/common-blockscout.env", - "MICROSERVICE_BENS_ENABLED=true\n", - ) - errors = self.module.validate_repository(self.repo) - self.assertTrue(any("BENS disabled" in error for error in errors)) - -def test_gcp_topology_drift_names_the_architecture_document(self): - self.replace("docs/DOScan-ARCHITECTURE.md", "dos-testnet-r0", "stale-host") - errors = self.module.validate_repository(self.repo) - self.assertTrue(any("GCP_TESTNET_INSTANCE" in error for error in errors)) - -def test_missing_sources_and_multiple_drifts_are_aggregated(self): - (self.repo / "docker-compose/docker-compose-beta.yml").unlink() - self.replace("docs/FEATURES.md", "2.10.0", "2.9.0") - errors = self.module.validate_repository(self.repo) - self.assertGreaterEqual(len(errors), 2) - self.assertTrue(any("missing required file" in error for error in errors)) - self.assertTrue(any("frontend version" in error for error in errors)) -``` - -- [ ] **Step 2: Run the focused tests and verify RED** - -Run: - -```text -python -m unittest .github/scripts/tests/test_validate_docs_production_status.py -v -``` - -Expected: the new Metadata, BENS, topology, and aggregation assertions fail. - -- [ ] **Step 3: Implement the remaining critical invariants** - -Read active env assignments while ignoring blank and commented lines. Require `MICROSERVICE_METADATA_ENABLED=true`. Reject an active BENS enablement, URL, or protocols value, and reject active `NEXT_PUBLIC_NAME_SERVICE_API_HOST` in every `common-frontend*.env` file. - -Read these exact workflow variables: - -```python -GCP_KEYS = ( - "GCP_INSTANCE", - "GCP_ZONE", - "GCP_TESTNET_INSTANCE", - "GCP_TESTNET_ZONE", -) -``` - -Require every derived value in `docs/DOScan-ARCHITECTURE.md`. Format every mismatch through one helper: - -```python -def diagnostic(path: Path, invariant: str, expected: object, actual: object) -> str: - return ( - f"{path.as_posix()}: {invariant}; " - f"expected {expected!r}; actual {actual!r}" - ) -``` - -When a file is missing, append a diagnostic and continue validating every source that remains readable. - -- [ ] **Step 4: Run focused and existing script tests** - -Run: - -```text -python -m unittest discover -s .github/scripts/tests -p "test_*.py" -v -python .github/scripts/validate-docs-production-status.py -python scripts/validate-blockscout-env-parity.py -``` - -Expected: all unit tests pass, the documentation validator passes on the real repository, and env parity passes. - -- [ ] **Step 5: Commit the state invariant slice** - -```text -git add .github/scripts/validate-docs-production-status.py .github/scripts/tests/test_validate_docs_production_status.py -git commit -m "ci: guard production status invariants" -``` - ---- - -### Task 3: Wire the guard into Dependency build and verify the complete change - -**Files:** -- Modify: `.github/workflows/dependency-build.yml` - -**Interfaces:** -- Consumes: `.github/scripts/validate-docs-production-status.py` from Tasks 1 and 2. -- Produces: A PR and path-filtered `main` push CI gate. - -- [ ] **Step 1: Add a failing workflow source assertion** - -Add a unit test that reads `.github/workflows/dependency-build.yml` from a supplied fixture root and requires the command: - -```text -python .github/scripts/validate-docs-production-status.py -``` - -Expected before workflow modification: FAIL because the command is absent. - -- [ ] **Step 2: Run the workflow assertion and verify RED** - -Run: - -```text -python -m unittest .github/scripts/tests/test_validate_docs_production_status.py -v -``` - -Expected: only the workflow integration assertion fails. - -- [ ] **Step 3: Add the CI command and push path coverage** - -In `workflow-scripts`, add: - -```yaml - - name: Validate production documentation status - run: python .github/scripts/validate-docs-production-status.py -``` - -Add these push paths without changing the Elixir build change detector: - -```yaml - - "docker-compose/docker-compose-mainnet.yml" - - "docker-compose/docker-compose-testnet.yml" - - "docker-compose/docker-compose-beta.yml" - - "docker-compose/envs/common-blockscout.env" - - "docker-compose/envs/common-frontend*.env" - - "docs/FEATURES.md" - - "docs/CHANGELOG.md" - - "docs/DOScan-ARCHITECTURE.md" - - ".github/workflows/deploy-config.yml" - - ".github/scripts/validate-docs-production-status.py" -``` - -- [ ] **Step 4: Run full local verification** - -Run: - -```text -python -m unittest discover -s .github/scripts/tests -p "test_*.py" -v -python .github/scripts/validate-docs-production-status.py -python scripts/validate-blockscout-env-parity.py -git diff --check -``` - -Expected: every command exits 0. - -- [ ] **Step 5: Commit workflow integration** - -```text -git add .github/workflows/dependency-build.yml .github/scripts/tests/test_validate_docs_production_status.py -git commit -m "ci: run production documentation drift guard" -``` - -- [ ] **Step 6: Review and publish** - -Request an independent code review, fix all Critical and Important findings, rerun the complete verification set, push the branch, open a PR, wait for required CI, and merge only after review and CI are green. - ---- - -## Execution Mode - -JOY authorized implementation without an execution choice. Use inline execution with `superpowers:executing-plans`, preserving TDD evidence at every RED and GREEN boundary. diff --git a/docs/superpowers/specs/2026-08-04-docs-production-status-drift-guard-design.vi.html b/docs/superpowers/specs/2026-08-04-docs-production-status-drift-guard-design.vi.html deleted file mode 100644 index 2ec54664ed2a..000000000000 --- a/docs/superpowers/specs/2026-08-04-docs-production-status-drift-guard-design.vi.html +++ /dev/null @@ -1,308 +0,0 @@ - - - - - - - - Thiết kế CI guard chống lệch trạng thái production - - - -
-
DOScan · Đặc tả thiết kế
-

CI guard chống lệch trạng thái production

-
Thiết kế đã được JOY duyệt ở mức invariant quan trọng
-

- Guard mới sẽ lấy Compose, env và workflow làm nguồn canonical, sau đó chặn PR nếu - FEATURES.md, CHANGELOG.md hoặc DOScan-ARCHITECTURE.md - còn ghi version, image pin, Metadata, BENS hoặc topology GCP mâu thuẫn. -

- -

1. Mục tiêu và phạm vi

-
-
-

Trong phạm vi

-
    -
  • Frontend và Backend version, tag, digest trên Mainnet, Testnet và Beta.
    • -
  • Trạng thái Metadata Service đang bật.
    • -
  • BENS không được cấu hình và tài liệu phải ghi là tắt.
    • -
  • GCP host và zone của Mainnet, Testnet.
    • -
  • Tính nhất quán giữa ba file tài liệu production.
    • -
  • Thông báo lỗi chỉ rõ file, invariant, expected và actual.
    • -
-
-
-

Ngoài phạm vi

-
    -
  • Không gọi public HTTP production trong guard.
    • -
  • Không kiểm tra mọi câu chữ hoặc mọi dòng trong bảng feature.
    • -
  • Không generate lại Markdown.
    • -
  • Không thêm manifest JSON làm nguồn trạng thái thứ ba.
    • -
  • Không nâng cấp Backend lên v11.2.4 trong cùng PR.
    • -
-
-
- -

2. Phương án đã chọn

-
-
-

Validator semantic, đã chọn

-

Đọc nguồn deployment thật, suy ra invariant canonical rồi kiểm tra token cấu trúc trong tài liệu.

-

Ưu điểm: không tạo nguồn trạng thái mới, patch nhỏ, lỗi CI có ý nghĩa.

-
-
-

Manifest trung gian, không chọn

-

Dễ parse nhưng tạo thêm một file phải đồng bộ với Compose và docs.

-
-
-

Generate docs, không chọn

-

Chặt nhưng quá xâm lấn, dễ làm tài liệu khó đọc và vượt yêu cầu hiện tại.

-
-
- -

3. Kiến trúc

- 
Compose + env + deploy-config.yml
-            |
-            v
-validate-docs-production-status.py
-            |
-            +-- suy ra Frontend/Backend pins
-            +-- suy ra Metadata và BENS state
-            +-- suy ra GCP host/zone
-            |
-            v
-FEATURES.md + CHANGELOG.md + DOScan-ARCHITECTURE.md
-            |
-            +-- pass: CI tiếp tục
-            +-- fail: báo invariant, expected, actual và file
- -

Thành phần

- - - - - - - - - - - - - - - - - - -
FileTrách nhiệm
.github/scripts/validate-docs-production-status.pyĐọc nguồn canonical, suy ra trạng thái và kiểm tra tài liệu. Không gọi network.
.github/scripts/tests/test_validate_docs_production_status.pyDùng fixture tạm để kiểm tra hành vi pass/fail và nội dung lỗi.
.github/workflows/dependency-build.ymlChạy unittest và validator trên PR; thêm path phù hợp cho push vào main.
- -

4. Nguồn canonical và invariant

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
InvariantNguồn canonicalTài liệu phải khớp
Backend image tag và digestService backend trong ba file docker-compose/docker-compose-{mainnet,testnet,beta}.ymlCả ba docs phải chứa pin production hiện tại ở phần trạng thái liên quan.
Frontend image version và digestService frontend trong ba file Compose trênBa docs phải chứa version; architecture phải chứa toàn bộ image pin gồm digest.
Metadata bậtMICROSERVICE_METADATA_ENABLED=true trong docker-compose/envs/common-blockscout.envFeature và architecture phải ghi Enabled, changelog phải ghi baseline đúng.
BENS tắtKhông có active MICROSERVICE_BENS_ENABLED=true và các frontend env không có Name Service hostFeature và architecture phải ghi Disabled hoặc not configured.
GCP Mainnet/TestnetBốn biến GCP_* trong .github/workflows/deploy-config.ymlDOScan-ARCHITECTURE.md.
- -
- Nguyên tắc: guard bảo vệ trạng thái đã deploy, không tự động so sánh với release upstream mới nhất. - Khi DOScan nâng lên v11.2.4, Compose/env được cập nhật trước hoặc cùng PR với docs; guard sẽ buộc ba nguồn Markdown đi cùng thay đổi đó. -
- -

5. Xử lý lỗi

-

Validator trả exit code 1 nếu có bất kỳ drift nào và gom toàn bộ lỗi trong một lần chạy.

- 
Production documentation drift validation failed:
-- docs/FEATURES.md: backend image expected 'ghcr.io/...@sha256:...', actual token not found
-- docs/DOScan-ARCHITECTURE.md: GCP_TESTNET_ZONE expected 'asia-southeast1-a', found stale value
-

File thiếu, YAML không có service cần thiết hoặc image không có digest đều là lỗi rõ ràng, không được bỏ qua.

- -

6. TDD và test matrix

- - - - - - - - - - - - - - -
TestMutation phải bị bắt
Baseline passFixture đồng bộ hoàn toàn trả exit code 0.
Backend digest driftMột Compose hoặc docs còn digest cũ.
Frontend version driftDocs còn version cũ sau khi Compose đổi.
Metadata disabledEnv đổi thành false hoặc tài liệu còn ghi Disabled.
BENS enabledEnv bật BENS nhưng tài liệu vẫn ghi tắt.
GCP topology driftHost hoặc zone trong architecture không khớp workflow.
Missing sourceThiếu Compose, env, workflow hoặc docs cần thiết.
Aggregated diagnosticsNhiều drift được báo cùng lúc, không fail ở lỗi đầu tiên.
-

Mỗi hành vi mới phải đi qua RED, GREEN, REFACTOR. Test chạy trên fixture thật trong thư mục tạm, không mock parser.

- -

7. CI và tiêu chí hoàn tất

-
    -
  1. Unittest mới pass cùng toàn bộ test workflow script hiện có.
    2. -
  2. Validator pass trên repo production hiện tại.
    3. -
  3. Mỗi mutation trong test matrix làm validator fail vì đúng lý do.
    4. -
  4. Dependency build chạy guard trên PR và push liên quan.
    5. -
  5. Không kích hoạt Deploy Config chỉ vì thay đổi guard hoặc spec.
    6. -
  6. Superpowers reviewer không còn Critical hoặc Important issue.
    7. -
  7. PR CI xanh trước merge.
    8. -
- -

8. Trình tự với Blockscout v11.2.4

-
    -
  1. Merge drift guard.
    2. -
  2. Thực hiện task upgrade v11.2.4 riêng.
    3. -
  3. Xác minh custom NFT video patch 86fd0dd5 còn cần reapply hay đã được upstream thay thế.
    4. -
  4. Build image immutable, smoke test NFT image/video, deploy Beta trước production.
    5. -
  5. Cập nhật Compose và ba docs trong cùng chuỗi upgrade để guard bảo vệ trạng thái mới.
    6. -
- -
- Branch: codex/docs-production-status-drift-guard
- Ngày: 04/08/2026
- Trạng thái: Chờ JOY duyệt spec trước khi viết implementation plan. -
-
- -