From 548412494d650285e52274cd6b5591da10d306ff Mon Sep 17 00:00:00 2001 From: Qiancheng Wu Date: Wed, 12 Aug 2026 19:41:38 -0700 Subject: [PATCH 1/4] Simplify OSS onboarding for 1.0 Add a guarded two-command AWS evaluation with a managed canary, consolidate deployment and documentation surfaces, and make environment integrations opt-in. Co-authored-by: Cursor --- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- .github/dependabot.yml | 2 +- .github/pull_request_template.md | 2 +- .github/workflows/ci.yml | 51 - .github/workflows/containers.yml | 2 - .github/workflows/kubernetes.yml | 44 +- .github/workflows/supply-chain.yml | 2 +- .gitignore | 1 + CHANGELOG.md | 23 +- CONTRIBUTING.md | 14 +- LICENSES/Apache-2.0.txt | 201 ---- Makefile | 15 +- NOTICE | 6 - README.md | 279 ++--- REUSE.toml | 12 - SECURITY.md | 5 +- contracts/README.md | 74 ++ contracts/pyproject.toml | 4 +- contracts/src/portscanner_contracts/compat.py | 10 - contracts/src/portscanner_contracts/models.py | 6 - contracts/tests/test_compat.py | 2 +- db/migrations/000001_core.down.sql | 53 + db/migrations/000001_core.up.sql | 660 ++++++++++- db/migrations/000002_findings.down.sql | 6 - db/migrations/000002_findings.up.sql | 281 ----- db/migrations/000003_serving.down.sql | 6 - db/migrations/000003_serving.up.sql | 266 ----- .../000004_application_role_connect.down.sql | 51 - .../000004_application_role_connect.up.sql | 85 -- .../000005_reviewed_state_safety.down.sql | 100 -- .../000005_reviewed_state_safety.up.sql | 140 --- db/migrator/Dockerfile | 2 +- db/migrator/README.md | 64 +- db/migrator/pyproject.toml | 2 +- db/migrator/requirements-runtime.txt | 14 +- db/migrator/src/act_migrator/handler.py | 4 +- db/migrator/tests/test_handler.py | 50 +- db/migrator/tests/test_migrator.py | 17 + .../tests/test_postgresql_integration.py | 128 +- docs/adding-sources.md | 589 --------- docs/architecture.md | 148 --- docs/aws-deployment.md | 412 ------- docs/configuration.md | 288 ----- docs/costs.md | 137 --- docs/data-handling.md | 142 --- docs/getting-started.md | 343 ------ docs/integrating-findings.md | 137 --- docs/multi-account.md | 158 --- docs/operations.md | 240 ---- docs/scanning-safety.md | 171 --- docs/security-model.md | 152 --- docs/testing.md | 276 ----- generator/.gitignore | 7 - generator/Dockerfile | 2 +- generator/README.md | 71 +- generator/pyproject.toml | 6 +- generator/requirements-runtime.txt | 14 +- generator/src/portscanner_generator/config.py | 33 +- .../src/portscanner_generator/event_reader.py | 28 +- .../src/portscanner_generator/handler.py | 76 +- .../src/portscanner_generator/revalidation.py | 14 +- .../portscanner_generator/scanner_resource.py | 25 +- generator/tests/test_dispatch.py | 30 +- generator/tests/test_event_reader.py | 89 +- generator/tests/test_scanner_resource.py | 18 +- inventory/Dockerfile | 2 +- inventory/README.md | 79 ++ inventory/pyproject.toml | 4 +- inventory/requirements-runtime.txt | 14 +- .../aws/config_snapshot.py | 34 +- .../portscanner_inventory/aws/ec2_snapshot.py | 22 +- .../portscanner_inventory/aws/normalize.py | 58 +- .../portscanner_inventory/aws/ownership.py | 9 + .../src/portscanner_inventory/aws/resolve.py | 24 +- inventory/src/portscanner_inventory/config.py | 150 +++ inventory/src/portscanner_inventory/events.py | 11 +- .../portscanner_inventory/handlers/outbox.py | 7 +- .../portscanner_inventory/handlers/signals.py | 40 +- .../handlers/snapshot.py | 102 +- inventory/src/portscanner_inventory/state.py | 39 +- inventory/tests/helpers.py | 2 + inventory/tests/test_handlers.py | 114 +- inventory/tests/test_snapshot_handler.py | 129 +- inventory/tests/test_snapshots.py | 94 ++ inventory/tests/test_state.py | 30 - operator/.dockerignore | 4 +- operator/.gitignore | 4 +- operator/Dockerfile | 8 +- operator/LICENSE | 222 +--- operator/Makefile | 28 +- operator/NOTICE | 4 - operator/PROJECT | 4 +- operator/README.md | 29 +- operator/THIRD_PARTY_NOTICES.md | 75 ++ operator/api/v1alpha1/groupversion_info.go | 4 +- operator/api/v1alpha1/scanner_types.go | 4 +- .../api/v1alpha1/zz_generated.deepcopy.go | 4 +- operator/chart/portscanner/Chart.yaml | 10 +- .../scanning.portscanner.io_scanners.yaml | 4 +- .../chart/portscanner/templates/_helpers.tpl | 4 +- .../portscanner/templates/deployment.yaml | 4 +- .../portscanner/templates/generator-rbac.yaml | 4 +- .../portscanner/templates/operator-rbac.yaml | 4 +- .../templates/operator-serviceaccount.yaml | 4 +- .../templates/priorityclasses.yaml | 4 +- .../portscanner/templates/scanner-quota.yaml | 4 +- .../portscanner/templates/scanner-rbac.yaml | 4 +- operator/chart/portscanner/values.yaml | 4 +- operator/cmd/manager/main.go | 4 +- operator/cmd/manager/main_test.go | 4 +- .../scanning.portscanner.io_scanners.yaml | 4 +- operator/config/crd/kustomization.yaml | 7 - operator/config/default/kustomization.yaml | 17 - operator/config/default/namespace.yaml | 9 - operator/config/manager/kustomization.yaml | 7 - operator/config/manager/manager.yaml | 81 -- operator/config/priorities/kustomization.yaml | 7 - .../config/priorities/priorityclasses.yaml | 24 - operator/config/rbac/kustomization.yaml | 11 - .../config/rbac/leader_election_role.yaml | 27 - .../rbac/leader_election_role_binding.yaml | 17 - operator/config/rbac/role.yaml | 59 - operator/config/rbac/role_binding.yaml | 18 - operator/config/rbac/service_account.yaml | 10 - .../samples/scanning_v1alpha1_scanner.yaml | 4 +- operator/config/scanner/kustomization.yaml | 9 - operator/config/scanner/role.yaml | 14 - operator/config/scanner/role_binding.yaml | 17 - operator/config/scanner/service_account.yaml | 11 - operator/go.mod | 10 +- operator/go.sum | 12 +- operator/hack/boilerplate.go.txt | 4 +- operator/hack/boilerplate.yaml.txt | 4 +- .../controller/integration_boundary_test.go | 4 +- operator/internal/controller/job_builder.go | 13 +- .../internal/controller/job_builder_test.go | 8 +- .../internal/controller/scanner_controller.go | 4 +- .../controller/scanner_controller_test.go | 4 +- .../controller/scanner_envtest_test.go | 4 +- parser/Dockerfile | 2 +- parser/pyproject.toml | 2 +- parser/requirements-runtime.txt | 14 +- parser/src/act_parser/config.py | 31 +- parser/src/act_parser/database.py | 69 +- parser/src/act_parser/handler.py | 4 +- parser/src/act_parser/target_handler.py | 9 +- parser/tests/test_handler.py | 64 + parser/tests/test_postgresql_integration.py | 82 +- parser/tests/test_target_handler.py | 78 +- processor/Dockerfile | 2 +- processor/pyproject.toml | 2 +- processor/requirements-runtime.txt | 14 +- processor/src/act_processor/handler.py | 225 +++- processor/tests/test_handler.py | 120 +- pyproject.toml | 12 +- scanner/nmap/Dockerfile | 2 +- scanner/nmap/README.md | 6 +- scanner/nmap/pyproject.toml | 2 +- .../nmap/src/portscanner_scanner/__init__.py | 2 +- scanner/nmap/src/portscanner_scanner/cli.py | 19 - scanner/nmap/tests/test_worker_and_cli.py | 2 - terraform/aws/README.md | 382 +++--- terraform/aws/application/README.md | 62 - terraform/aws/application/main.tf | 159 ++- terraform/aws/application/main.tftest.hcl | 89 ++ terraform/aws/application/outputs.tf | 119 +- terraform/aws/application/variables.tf | 98 +- .../.terraform.lock.hcl | 0 .../environment.auto.tfvars.json.example | 64 + terraform/aws/deployment/main.tf | 182 +++ terraform/aws/deployment/main.tftest.hcl | 127 ++ terraform/aws/deployment/outputs.tf | 165 +++ terraform/aws/deployment/variables.tf | 304 +++++ terraform/aws/deployment/versions.tf | 12 + terraform/aws/examples/README.md | 39 - .../created-vpc/evaluation.tfvars.example | 47 - terraform/aws/examples/created-vpc/main.tf | 360 ------ terraform/aws/examples/existing-vpc/main.tf | 23 +- .../examples/multi-account-central/main.tf | 23 +- terraform/aws/member-account/README.md | 28 - terraform/aws/member-account/main.tf | 19 +- terraform/aws/modules/database/main.tf | 13 +- terraform/aws/modules/eks/main.tf | 13 +- terraform/aws/modules/functions/main.tf | 267 ++++- .../aws/modules/functions/main.tftest.hcl | 139 ++- terraform/aws/modules/identities/main.tf | 33 +- .../aws/modules/identities/main.tftest.hcl | 74 ++ terraform/aws/modules/managed-canary/main.tf | 352 ++++++ .../modules/managed-canary/main.tftest.hcl | 85 ++ terraform/aws/modules/signals/main.tf | 12 +- terraform/aws/modules/signals/main.tftest.hcl | 5 +- terraform/aws/modules/storage/main.tf | 47 +- terraform/aws/modules/storage/main.tftest.hcl | 47 + terraform/aws/scripts/bootstrap.sh | 723 ++++++++++++ terraform/aws/scripts/build-images.sh | 42 +- terraform/aws/scripts/deploy.sh | 1050 ++++++++++++----- terraform/aws/scripts/evaluate-canary.sh | 313 +++++ terraform/aws/scripts/retire-canary.sh | 340 ++++++ terraform/aws/scripts/tests/bootstrap-test.sh | 436 +++++++ .../aws/scripts/tests/build-images-test.sh | 43 +- .../aws/scripts/tests/canary-helpers-test.sh | 357 ++++++ terraform/aws/scripts/tests/deploy-test.sh | 428 ++++++- .../aws/scripts/tests/emergency-pause-test.sh | 2 +- terraform/aws/scripts/validate.sh | 2 +- terraform/aws/state-bootstrap/README.md | 19 - tools/sanitize-policy.toml | 13 +- tools/test_migrator_package.sh | 2 +- tools/tests/test_release_packaging.py | 43 +- tools/verify_python_image.py | 2 +- uv.lock | 28 +- 210 files changed, 9002 insertions(+), 7117 deletions(-) delete mode 100644 LICENSES/Apache-2.0.txt delete mode 100644 NOTICE create mode 100644 contracts/README.md delete mode 100644 contracts/src/portscanner_contracts/compat.py delete mode 100644 db/migrations/000002_findings.down.sql delete mode 100644 db/migrations/000002_findings.up.sql delete mode 100644 db/migrations/000003_serving.down.sql delete mode 100644 db/migrations/000003_serving.up.sql delete mode 100644 db/migrations/000004_application_role_connect.down.sql delete mode 100644 db/migrations/000004_application_role_connect.up.sql delete mode 100644 db/migrations/000005_reviewed_state_safety.down.sql delete mode 100644 db/migrations/000005_reviewed_state_safety.up.sql delete mode 100644 docs/adding-sources.md delete mode 100644 docs/architecture.md delete mode 100644 docs/aws-deployment.md delete mode 100644 docs/configuration.md delete mode 100644 docs/costs.md delete mode 100644 docs/data-handling.md delete mode 100644 docs/getting-started.md delete mode 100644 docs/integrating-findings.md delete mode 100644 docs/multi-account.md delete mode 100644 docs/operations.md delete mode 100644 docs/scanning-safety.md delete mode 100644 docs/security-model.md delete mode 100644 docs/testing.md delete mode 100644 generator/.gitignore create mode 100644 inventory/README.md delete mode 100644 operator/NOTICE create mode 100644 operator/THIRD_PARTY_NOTICES.md delete mode 100644 operator/config/crd/kustomization.yaml delete mode 100644 operator/config/default/kustomization.yaml delete mode 100644 operator/config/default/namespace.yaml delete mode 100644 operator/config/manager/kustomization.yaml delete mode 100644 operator/config/manager/manager.yaml delete mode 100644 operator/config/priorities/kustomization.yaml delete mode 100644 operator/config/priorities/priorityclasses.yaml delete mode 100644 operator/config/rbac/kustomization.yaml delete mode 100644 operator/config/rbac/leader_election_role.yaml delete mode 100644 operator/config/rbac/leader_election_role_binding.yaml delete mode 100644 operator/config/rbac/role.yaml delete mode 100644 operator/config/rbac/role_binding.yaml delete mode 100644 operator/config/rbac/service_account.yaml delete mode 100644 operator/config/scanner/kustomization.yaml delete mode 100644 operator/config/scanner/role.yaml delete mode 100644 operator/config/scanner/role_binding.yaml delete mode 100644 operator/config/scanner/service_account.yaml delete mode 100644 terraform/aws/application/README.md create mode 100644 terraform/aws/application/main.tftest.hcl rename terraform/aws/{examples/created-vpc => deployment}/.terraform.lock.hcl (100%) create mode 100644 terraform/aws/deployment/environment.auto.tfvars.json.example create mode 100644 terraform/aws/deployment/main.tf create mode 100644 terraform/aws/deployment/main.tftest.hcl create mode 100644 terraform/aws/deployment/outputs.tf create mode 100644 terraform/aws/deployment/variables.tf create mode 100644 terraform/aws/deployment/versions.tf delete mode 100644 terraform/aws/examples/README.md delete mode 100644 terraform/aws/examples/created-vpc/evaluation.tfvars.example delete mode 100644 terraform/aws/examples/created-vpc/main.tf delete mode 100644 terraform/aws/member-account/README.md create mode 100644 terraform/aws/modules/identities/main.tftest.hcl create mode 100644 terraform/aws/modules/managed-canary/main.tf create mode 100644 terraform/aws/modules/managed-canary/main.tftest.hcl create mode 100755 terraform/aws/scripts/bootstrap.sh create mode 100755 terraform/aws/scripts/evaluate-canary.sh create mode 100755 terraform/aws/scripts/retire-canary.sh create mode 100755 terraform/aws/scripts/tests/bootstrap-test.sh create mode 100755 terraform/aws/scripts/tests/canary-helpers-test.sh delete mode 100644 terraform/aws/state-bootstrap/README.md diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 4d5441c..5a590b5 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -11,7 +11,7 @@ body: attributes: value: | Describe the problem and safety boundary, not a live environment. New source - proposals should follow docs/adding-sources.md. + proposals should follow inventory/README.md. - type: checkboxes id: checks diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bbeb1b7..dfe67bc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -38,7 +38,7 @@ updates: - package-ecosystem: terraform directories: - - /terraform/aws/examples/created-vpc + - /terraform/aws/deployment - /terraform/aws/examples/existing-vpc - /terraform/aws/examples/member-account - /terraform/aws/examples/multi-account-central diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 40ca123..7ffe625 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -50,7 +50,7 @@ Complete this section when adding or changing a snapshot/signal source. - [ ] Metadata allowlist/redaction - [ ] Central profile/priority/deadline policy - [ ] IAM/Terraform registration -- [ ] Review checklist in `docs/adding-sources.md` +- [ ] Source-adapter requirements in `inventory/README.md` ## Canary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20ad538..8cf33e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,57 +102,6 @@ jobs: PORTSCANNER_TEST_PYTHON: ${{ github.workspace }}/.venv/bin/python run: make -C operator test-envtest - schemas: - name: JSON schemas - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7.0.1 - - - uses: actions/setup-python@v7.0.0 - with: - python-version: "3.12" - - - name: Install uv - run: python -m pip install --disable-pip-version-check "uv==0.11.30" - - - name: Install schema dependencies - shell: bash - run: | - set -euo pipefail - uv sync --frozen --all-packages --group dev - - - name: Validate schema documents - shell: bash - run: | - set -euo pipefail - if [[ ! -d schemas ]]; then - echo "No schemas directory is present." - exit 0 - fi - uv run python - <<'PY' - import json - from pathlib import Path - - from jsonschema.validators import validator_for - - schemas = sorted(Path("schemas").glob("*.schema.json")) - if not schemas: - raise SystemExit("schemas directory contains no *.schema.json files") - for path in schemas: - schema = json.loads(path.read_text(encoding="utf-8")) - validator_for(schema).check_schema(schema) - print(f"validated {path}") - PY - - - name: Run schema contract tests - shell: bash - run: | - set -euo pipefail - if [[ -f contracts/tests/test_schemas.py ]]; then - uv run --package portscanner-contracts \ - pytest contracts/tests/test_schemas.py - fi - generated-drift: name: Generated drift runs-on: ubuntu-latest diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index a957689..2066f98 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -10,7 +10,6 @@ on: - "**/*.dockerfile" - ".dockerignore" - "LICENSE" - - "NOTICE" - "THIRD_PARTY_NOTICES.md" - "contracts/**" - "db/migrations/**" @@ -39,7 +38,6 @@ on: - "**/*.dockerfile" - ".dockerignore" - "LICENSE" - - "NOTICE" - "THIRD_PARTY_NOTICES.md" - "contracts/**" - "db/migrations/**" diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml index 13fde5b..867d8b5 100644 --- a/.github/workflows/kubernetes.yml +++ b/.github/workflows/kubernetes.yml @@ -6,11 +6,11 @@ name: Kubernetes manifests on: pull_request: paths: - - "**/*.yaml" - - "**/*.yml" - - "**/Chart.yaml" - - "**/values.yaml" - - "**/*.tpl" + - "operator/api/**" + - "operator/chart/**" + - "operator/config/crd/**" + - "operator/config/samples/**" + - "operator/Makefile" - "tools/export_crd_schemas.py" - "tools/tests/test_export_crd_schemas.py" - "generator/pyproject.toml" @@ -21,8 +21,11 @@ on: branches: - main paths: - - "operator/config/**" + - "operator/api/**" - "operator/chart/**" + - "operator/config/crd/**" + - "operator/config/samples/**" + - "operator/Makefile" - "tools/export_crd_schemas.py" - "tools/tests/test_export_crd_schemas.py" - "generator/pyproject.toml" @@ -39,7 +42,7 @@ concurrency: jobs: validate: - name: Kustomize, Helm, kubeconform + name: Helm and kubeconform runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.1 @@ -62,7 +65,6 @@ jobs: shell: bash run: | set -euo pipefail - go install sigs.k8s.io/kustomize/kustomize/v5@v5.8.1 go install helm.sh/helm/v3/cmd/helm@v3.20.1 go install github.com/yannh/kubeconform/cmd/kubeconform@v0.8.0 @@ -77,32 +79,6 @@ jobs: --output-directory "${RUNNER_TEMP}/crd-schemas" \ "${crds[@]}" - - name: Build and validate Kustomize roots - shell: bash - run: | - set -euo pipefail - count=0 - while IFS= read -r file; do - directory="$(dirname "${file}")" - rendered="${RUNNER_TEMP}/kustomize-${count}.yaml" - echo "Building ${directory}" - kustomize build "${directory}" > "${rendered}" - for version in 1.35.0 1.36.0; do - kubeconform \ - -strict \ - -summary \ - -skip CustomResourceDefinition \ - -kubernetes-version "${version}" \ - -schema-location default \ - -schema-location "${RUNNER_TEMP}/crd-schemas/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ - "${rendered}" - done - count=$((count + 1)) - done < <(git ls-files 'kustomization.yaml' '**/kustomization.yaml') - if [[ "${count}" -eq 0 ]]; then - echo "No Kustomize roots are present." - fi - - name: Validate custom resource samples shell: bash run: | diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index f2cf965..57b39d3 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -28,7 +28,7 @@ jobs: python-version: "3.12" - name: Run sanitizer unit tests - run: python -m unittest discover -s tools/tests -p 'test_*.py' -v + run: python -m unittest -v tools.tests.test_sanitize - name: Scan tracked publication content run: python tools/sanitize.py diff --git a/.gitignore b/.gitignore index 029f71c..c8db373 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ htmlcov/ *.local # Terraform state, plans, local variables, and provider caches +/.portscanner/ **/.terraform/ *.tfstate *.tfstate.* diff --git a/CHANGELOG.md b/CHANGELOG.md index b0ca74c..ddb449d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,19 +8,18 @@ SPDX-License-Identifier: MIT All notable changes to this project will be documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and releases -will use [Semantic Versioning](https://semver.org/spec/v2.0.0.html) after the initial -public version is tagged. +use [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [1.0.0] - 2026-08-12 + ### Added - Public architecture and operating model using **DISCOVER → PRIORITIZE → VERIFY → ACT**. -- AWS deployment, configuration, operations, security, scanning safety, data handling, - testing, cost, and multi-account documentation. -- Implementation contract and review checklist for new inventory snapshots and change - signal sources. +- Concise root and component guides for the AWS deployment, inventory sources, + contracts, scanning, operations, safety, and finding integrations. - Versioned TargetEvent, ScanResultEnvelope, and Finding contracts with generated JSON Schemas and synthetic examples. - AWS Config and direct-EC2 inventory snapshots, CloudTrail/EventBridge signal @@ -33,10 +32,9 @@ public version is tagged. - Staged Terraform for single-account and optional hub/spoke AWS deployment, including private EKS/Aurora, secure queues and buckets, alarms, image publication, and paused activation gates. -- Guarded single-account evaluation quickstart with fail-closed account/Region checks, a - one-target snapshot invocation, separate canary/automatic dispatch gates, conservative - scanner tuning, optional CloudTrail, restricted EKS API access, explicit pause, - scanner egress output, and downstream finding handoff guidance. +- Guarded single-account evaluation with one environment file, resumable state/image + bootstrap, a Terraform-managed one-port canary, fail-closed account/Region checks, + conservative scanner tuning, restricted EKS API access, and automatic return to pause. - Hard scanner-Pod quotas, per-destination serialization, generator-side CIDR enforcement, endpoint-mode public-egress checks, and an AWS-only emergency dispatch brake. @@ -54,6 +52,8 @@ public version is tagged. disabled manual AWS sandbox. - Security, contribution, conduct, issue, pull request, dependency update, pre-commit, and secret-scanning policies. +- Helm-only Kubernetes packaging, optional finding export, and a PostgreSQL-first + default deployment boundary. ### Security @@ -62,4 +62,5 @@ public version is tagged. per-destination serialization. - Incomplete inventory or scan evidence is explicitly preserved as UNKNOWN. -[Unreleased]: https://github.com/Roblox/portscanner/commits/main +[Unreleased]: https://github.com/Roblox/portscanner/compare/v1.0.0...HEAD +[1.0.0]: https://github.com/Roblox/portscanner/releases/tag/v1.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 775de68..2c63521 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,14 +39,18 @@ python3 tools/sanitize.py --working-tree pre-commit run --all-files ``` -See [docs/testing.md](docs/testing.md) for component, schema, generated, Terraform, -Kubernetes, container, and integration checks. +The root `Makefile` is the command index for component, schema, generated, +Terraform, Helm, container, and integration checks. Run `make ci` before +requesting review and `make containers` when an image or runtime dependency +changes. ## Design rules - Keep **DISCOVER → PRIORITIZE → VERIFY → ACT** boundaries explicit. - Snapshot-driven inventory is authoritative; event-driven signals are hints. -- Apply removals only after a complete snapshot. +- Apply removals only after absence from a complete snapshot or an + authoritative resource-specific reread. Partial snapshots and raw signals + never remove targets. - Use stable Target identity and monotonic generations. - Revalidate ownership immediately before dispatch. - Preserve periodic known-door reconciliation while priority work advances. @@ -56,8 +60,8 @@ Kubernetes, container, and integration checks. - Default dispatch off and account/CIDR allowlists empty. - Minimize and redact provider metadata. -Read [docs/adding-sources.md](docs/adding-sources.md) before adding an inventory or signal -adapter. +Read [inventory/README.md](inventory/README.md) before adding an inventory or +signal adapter. ## Fixtures and examples diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt deleted file mode 100644 index 261eeb9..0000000 --- a/LICENSES/Apache-2.0.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/Makefile b/Makefile index b6a5cc6..c353df3 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ UV ?= uv PYTHON ?= python3 HELM ?= helm -KUBECTL ?= kubectl KUBECONFORM ?= kubeconform PORTSCANNER_TEST_PYTHON ?= $(CURDIR)/.venv/bin/python @@ -102,6 +101,8 @@ secret-scan: gitleaks dir --redact --config .gitleaks.toml . terraform-script-tests: + ./terraform/aws/scripts/tests/bootstrap-test.sh + ./terraform/aws/scripts/tests/canary-helpers-test.sh ./terraform/aws/scripts/tests/deploy-test.sh ./terraform/aws/scripts/tests/emergency-pause-test.sh ./terraform/aws/scripts/tests/build-images-test.sh @@ -119,18 +120,6 @@ kubernetes-validate: --output-directory "$$schema_dir" \ $$(git ls-files 'operator/config/crd/bases/*.yaml'); \ count=0; \ - for file in $$(git ls-files 'kustomization.yaml' '**/kustomization.yaml'); do \ - rendered="$$work_dir/kustomize-$$count.yaml"; \ - $(KUBECTL) kustomize "$$(dirname "$$file")" >"$$rendered"; \ - for version in 1.35.0 1.36.0; do \ - $(KUBECONFORM) -strict -summary -skip CustomResourceDefinition \ - -kubernetes-version "$$version" \ - -schema-location default \ - -schema-location "$$schema_dir/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" \ - "$$rendered"; \ - done; \ - count=$$((count + 1)); \ - done; \ for sample in $$(git ls-files 'operator/config/samples/*.yaml'); do \ for version in 1.35.0 1.36.0; do \ $(KUBECONFORM) -strict -summary \ diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 595843f..0000000 --- a/NOTICE +++ /dev/null @@ -1,6 +0,0 @@ -Portscanner Kubernetes Operator -Copyright 2026 Roblox Corporation - -The files under the operator/ boundary are licensed under the Apache License, -Version 2.0. This NOTICE applies to that boundary and does not change the -license of other repository content. diff --git a/README.md b/README.md index a307b66..b3cf974 100644 --- a/README.md +++ b/README.md @@ -5,174 +5,119 @@ SPDX-License-Identifier: MIT # Portscanner -Portscanner is a self-hosted system for continuously verifying the Internet-facing ports -of cloud assets you are authorized to test. It discovers public AWS addresses, -revalidates ownership immediately before dispatch, runs bounded Nmap jobs from an -outside vantage, and publishes normalized findings for downstream systems. - -## Start here - -Choose the smallest path that matches your goal: - -- **Evaluate without AWS:** install the locked development workspace and run the - synthetic unit, contract, migration, parser, Terraform, and Kubernetes checks. No - network target is scanned. -- **Prove one AWS canary:** follow the [getting-started guide](docs/getting-started.md). - It deploys in stages with dispatch off, restricts the evaluation to one authorized - account and public `/32`, verifies that canary, and pauses again. -- **Connect your system:** consume immutable S3 finding documents through the external - SQS handoff described in [integrating findings](docs/integrating-findings.md). - -The AWS stack includes EKS, Aurora, NAT, queues, buckets, supporting services, and -optional CloudTrail API hints, so it incurs charges before scanning is activated. Use a -dedicated sandbox, review the plan and [cost model](docs/costs.md), and scan only assets -you own or are explicitly authorized to test. - -Its operating loop uses the exact lifecycle: - -> **DISCOVER → PRIORITIZE → VERIFY → ACT** - -The design is both **snapshot-driven** and **event-driven**: - -- Snapshot-driven discovery is authoritative. Complete snapshots find targets that an - event stream missed and remove targets that no longer belong to the inventory. -- Event-driven signals are hints that can move likely changes ahead of the baseline - queue. A signal never proves current ownership or exposure by itself. -- Periodic reconciliation compares complete snapshots, repairs missed or duplicated - events, and keeps the baseline sweep moving. - -> **The sweep continues. The change jumps the line.** - -## The model - -- A **Target** is an authorized provider resource and network address eligible for - verification. It has a stable identity, an ownership record, and a monotonic - generation. -- An **Exposure** is outside-vantage evidence about whether a transport/port on a Target - was reachable at a specific time. Missing, expired, failed, or incomplete evidence is - **UNKNOWN**—never silently “closed.” -- A **Finding** is a policy-relevant interpretation of one or more Exposures plus current - context. A Finding is not the raw scanner output. - -Inventory and prior evidence classify each door: - -- **new door** (`new_target`): a Target first discovered, or reactivated after its - previously accepted state was removed; -- **changed door** (`target_change`): an active known Target whose address binding or - scan-relevant non-policy lifecycle state changed; -- **changed door** (`policy_change`): an active known Target whose attached security - groups or effective ingress policy changed; and -- **known door** (`coverage`): an unchanged Target that remains in the periodic - reconciliation sweep. - -New and changed doors receive priority. Known doors retain scheduled coverage so an -event path cannot starve the baseline. - -## How the lifecycle works - -1. **DISCOVER** — a provider adapter reads a complete inventory snapshot. Optional - control-plane signals identify resources worth rereading sooner. -2. **PRIORITIZE** — policy assigns a profile, priority, and deadline to each new door, - changed door, or known door. Deployment-wide Nmap rates plus hard scanner-Pod and - Job-object quotas bound aggregate work; required anti-affinity serializes work for - the same public destination. -3. **VERIFY** — immediately before dispatch, the system rereads current provider state - and revalidates that the address is still owned by the expected Target generation. - An outside-vantage worker then performs the authorized scan. -4. **ACT** — normalized evidence updates Exposure state. Detection policy may open, - update, or resolve a Finding; inconclusive work remains UNKNOWN and is retried or - reconciled. - -Ownership revalidation is a hard dispatch gate. This matters for recycled public -addresses: a queued Target can become stale between discovery and scanning. A stale, -moved, missing, or ambiguous ownership verdict must not be scanned. - -## First public release scope - -The first release is intentionally narrow: - -- self-hosted deployment in AWS; -- a single AWS account by default, with an optional hub/spoke pattern; -- complete snapshots of supported AWS resources with public IPv4 addresses; -- optional AWS-native change signals treated only as acceleration hints; -- bounded Nmap jobs orchestrated on Kubernetes from an outside vantage; -- durable queues, idempotency records, object evidence, and relational state; and -- provider-neutral source and evidence contracts intended for additional adapters. - -The first release does **not** promise a p95 detection time or any universal latency -SLO. End-to-end time depends on provider freshness, snapshot cadence, queue depth, -profile, rate limits, target behavior, and scanner placement. Operators must measure -their own environment. - -The first release also excludes: - -- Internet-wide or third-party scanning; -- private-address or IPv6 scanning; -- production-ready Azure or Google Cloud adapters; -- automatic remediation or enforcement; -- a hosted control plane, turnkey organization rollout, or compliance certification; -- guarantees that an open service is vulnerable, or that no observed port means a host - is safe. - -Only scan assets you own or are explicitly authorized to test. Start with -[scanning safety](docs/scanning-safety.md). - -## Repository layout - -- `contracts/` and `schemas/`: versioned TargetEvent, ScanResultEnvelope, and Finding - wire contracts. -- `inventory/`: AWS Config/direct-EC2 snapshots, CloudTrail signal resolution, - generation state, ownership revalidation, and the transactional outbox. -- `generator/`: freshness-gated dispatch into one-shot Scanner resources. -- `operator/`: the Kubernetes controller, CRD, Helm chart, and hardened Job policy. -- `scanner/nmap/`: the single-target Nmap worker and immutable evidence publisher. -- `parser/`, `processor/`, and `db/`: generation-safe Exposure state, generic rules, - Findings, handoff, and migrations. -- `terraform/aws/`: staged single-account deployment and optional hub/spoke onboarding. - -## Validate and deploy - -Python 3.12 and `uv` are required for the local workspace; the operator checks require -Go and the PostgreSQL integration target requires Docker: - -```bash +Portscanner is a self-hosted system for continuously verifying the +Internet-facing ports of cloud assets you are authorized to test. It discovers +current AWS ownership, dispatches bounded Nmap jobs from Kubernetes, and stores +generation-safe exposures and findings in PostgreSQL. + +> Only scan assets you own or are explicitly authorized to test. The AWS +> evaluation creates EKS, Aurora, NAT, EC2, queues, buckets, and supporting +> services that incur charges. Dispatch stays off except for one managed +> canary until you explicitly activate an integration. + +## Try it locally + +Local checks do not scan a network target. Install Python 3.12, `uv`, Go, and +Docker, then run: + +```sh make sync make check -make test-go -make test-go-envtest -make vet-go -make test-postgresql ``` -`make ci` adds backend-disabled Terraform validation/contract tests and Kubernetes -rendering. -Terraform provider installation can require network access; it is not an offline check. -`make containers` builds all seven images from committed, tracked-only contexts. Python -Lambda dependencies come from checked-in hash-verified `uv.lock` exports. Deployment is -intentionally staged with dispatch disabled until images, migrations, the operator, and -an authorized canary are ready; follow the [AWS deployment guide](docs/aws-deployment.md). - -## Documentation - -- [Architecture](docs/architecture.md) -- [Getting started](docs/getting-started.md) -- [AWS deployment](docs/aws-deployment.md) -- [Configuration](docs/configuration.md) -- [Operations](docs/operations.md) -- [Security model](docs/security-model.md) -- [Scanning safety](docs/scanning-safety.md) -- [Data handling](docs/data-handling.md) -- [Testing](docs/testing.md) -- [Costs](docs/costs.md) -- [Multi-account deployment](docs/multi-account.md) -- [Integrating findings](docs/integrating-findings.md) -- [Adding inventory or signal sources](docs/adding-sources.md) - -## Project status - -The project is pre-1.0. Treat contracts, migrations, deployment manifests, and container -images as a coordinated release. Review the staged activation procedure before using a -real account. - -See [CONTRIBUTING.md](CONTRIBUTING.md) for development and review requirements and -[SECURITY.md](SECURITY.md) for private vulnerability reporting. +`make ci` adds PostgreSQL, Go/envtest, Terraform, Helm, and publication checks. +See [CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow. + +## Evaluate the AWS stack + +The canonical evaluation creates its own low-cost VPC and a small isolated EC2 +canary that exposes only TCP 18080 to the scanner's NAT address. It never scans +localhost, an arbitrary public host, or the rest of your account. + +Install the versions checked by the bootstrap script, configure the standard +AWS CLI credential chain, and copy the environment template: + +```sh +cp terraform/aws/deployment/environment.auto.tfvars.json.example \ + terraform/aws/deployment/environment.auto.tfvars.json +$EDITOR terraform/aws/deployment/environment.auto.tfvars.json +``` + +The JSON file is the only user-maintained deployment configuration. It contains +non-secret account, Region, access, capacity, retention, and integration +policy; credentials stay outside Terraform variables. + +Deploy and evaluate: + +```sh +./terraform/aws/scripts/bootstrap.sh +./terraform/aws/scripts/deploy.sh evaluate +``` + +Bootstrap verifies the live AWS identity, creates remote state, applies a +paused foundation, and builds, scans, and publishes immutable images. +Evaluation migrates PostgreSQL, installs the Helm operator, runs one targeted +canary scan, verifies its finding, and returns dispatch to paused state. +Each saved Terraform plan still requires an explicit confirmation. + +Read [terraform/aws/README.md](terraform/aws/README.md) before applying. It +covers exact prerequisites, costs, failure recovery, emergency pause, data +retention, and guarded destruction. + +## Next steps + +After the canary succeeds: + +1. Add explicit AWS account, CIDR, ENI class, and opt-in tag scope to the same + environment JSON. +2. Enable periodic snapshots, review the plan, and activate the integration. +3. Optionally add Config/CloudTrail hints, multi-account collection, or S3/SQS + finding export; retire the managed canary when it is no longer needed. + +```sh +./terraform/aws/scripts/deploy.sh activate +./terraform/aws/scripts/deploy.sh pause +``` + +Activation is never implied by installation. Signals only accelerate work; +authoritative snapshots and immediate provider ownership rereads remain the +scan gate. + +## How it works + +1. **Discover** complete provider inventory and record stable target + generations. +2. **Prioritize** new, changed, and periodic coverage work under explicit + account, CIDR, profile, rate, Pod, and Job limits. +3. **Verify** current ownership immediately before creating one digest-pinned + Scanner Job. +4. **Act** on complete evidence; failed or incomplete work remains `UNKNOWN` + and never silently closes an exposure. + +## Supported scope + +The 1.0 release supports self-hosted AWS, public IPv4 EC2 targets, TCP +connect scanning, Kubernetes orchestration, and PostgreSQL findings. It does +not authorize Internet-wide scanning, support private/IPv6 targets, provide +production-ready GCP/Azure adapters, remediate resources, or promise a +universal detection-time SLO. + +## Components + +- [Terraform AWS deployment](terraform/aws/README.md) +- [Inventory and source adapters](inventory/README.md) +- [Contracts and finding consumers](contracts/README.md) +- [Generator](generator/README.md) +- [Kubernetes operator](operator/README.md) +- [Nmap scanner](scanner/nmap/README.md) +- [Database migrator](db/migrator/README.md) + +Synthetic contracts are in `examples/`; generated JSON Schemas are in +`schemas/`. + +## Project policy + +Portscanner is MIT licensed. Report vulnerabilities privately through +[SECURITY.md](SECURITY.md), follow [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md), +and review [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for bundled +dependencies. diff --git a/REUSE.toml b/REUSE.toml index 93e7ce1..3f68f1c 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -7,15 +7,3 @@ path = "**" precedence = "aggregate" SPDX-FileCopyrightText = "2026 Roblox" SPDX-License-Identifier = "MIT" - -[[annotations]] -path = "NOTICE" -precedence = "override" -SPDX-FileCopyrightText = "2026 Roblox Corporation" -SPDX-License-Identifier = "Apache-2.0" - -[[annotations]] -path = "operator/**" -precedence = "override" -SPDX-FileCopyrightText = "2026 Roblox Corporation" -SPDX-License-Identifier = "Apache-2.0" diff --git a/SECURITY.md b/SECURITY.md index b0108e0..f02c227 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,9 +7,8 @@ SPDX-License-Identifier: MIT ## Supported versions -The project is pre-1.0. Security fixes are provided on the default branch until the -first versioned release. After releases begin, this section will list supported release -lines explicitly. +Security fixes are provided for the latest 1.x release and the default branch. +Older release lines should be upgraded before reporting an issue. ## Report a vulnerability privately diff --git a/contracts/README.md b/contracts/README.md new file mode 100644 index 0000000..e51a6eb --- /dev/null +++ b/contracts/README.md @@ -0,0 +1,74 @@ + + +# Portscanner contracts + +This package owns the provider-neutral records exchanged between inventory, +dispatch, scanning, ingestion, and optional finding consumers. Runtime +components must parse these models instead of maintaining looser local copies. + +## Contracts + +- `TargetEvent` declares one authoritative target generation and, when + applicable, one bounded scan directive. +- `ScanResultEnvelope` references immutable raw evidence and contains one + normalized `ScanResult`. +- `Finding` is a policy result derived from accepted exposure state. It is not + raw Nmap output. + +Canonical Python models live in `src/portscanner_contracts`. Generated JSON +Schemas live in the repository `schemas/` directory, and synthetic examples +live in `examples/`. Examples contain documentation-only identifiers and are +contract fixtures, not runnable scan requests. + +Identifiers are deterministic. Timestamps are UTC, object shapes are closed, +and unknown fields are rejected. A failed or incomplete scan remains +`UNKNOWN`; consumers must not infer that an unobserved port is closed. + +## Compatibility + +The 1.x contracts follow Semantic Versioning. A contract change must update +the model, generated Schema, examples, producers, consumers, and compatibility +tests together. Never reuse a schema version for an incompatible shape. + +Regenerate and test from the repository root: + +```sh +make generate +make test-contracts +``` + +Generated files are committed so deployments and non-Python consumers can pin +the exact release contract. + +## Consuming finding exports + +PostgreSQL findings are the default system boundary. S3/SQS export is an +optional integration enabled in the AWS environment configuration. + +When export is enabled, SQS messages are S3 object notifications, not Finding +payloads. A consumer must: + +1. Verify the configured bucket, prefix, object version, and size before + download. +2. Fetch that exact object version and verify its declared payload SHA-256. +3. Validate the JSON Schema and semantic model. +4. Commit downstream state idempotently using the finding event key and + finding version. +5. Delete the SQS message only after the downstream commit succeeds. + +S3 notifications and SQS delivery are at least once and may be delayed or +reordered. Consumers must tolerate duplicates, reject malformed pointers, and +send repeatedly failing messages to their own reviewed quarantine path. + +Finding documents intentionally omit raw XML, credentials, private cloud +metadata, and scanner command output. Integrations should preserve that +minimized boundary rather than joining directly to internal tables. + +## Security review + +Changes to identity, generation, freshness, coverage, evidence eligibility, +hashing, or finding resolution semantics are security-sensitive. Use only +synthetic fixtures and run the full repository checks before review. diff --git a/contracts/pyproject.toml b/contracts/pyproject.toml index cbff9c6..7be1b43 100644 --- a/contracts/pyproject.toml +++ b/contracts/pyproject.toml @@ -7,14 +7,14 @@ build-backend = "hatchling.build" [project] name = "portscanner-contracts" -version = "0.1.0" +version = "1.0.0" description = "Typed public data contracts for Portscanner" requires-python = ">=3.12" license = "MIT" license-files = ["LICENSE"] authors = [{ name = "Roblox" }] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", diff --git a/contracts/src/portscanner_contracts/compat.py b/contracts/src/portscanner_contracts/compat.py deleted file mode 100644 index fb31d5d..0000000 --- a/contracts/src/portscanner_contracts/compat.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Roblox -# SPDX-License-Identifier: MIT - -"""Backward-compatible import for the public scanner envelope validator.""" - -from __future__ import annotations - -from portscanner_contracts.models import validate_scan_result - -__all__ = ["validate_scan_result"] diff --git a/contracts/src/portscanner_contracts/models.py b/contracts/src/portscanner_contracts/models.py index 10bb862..f669a9c 100644 --- a/contracts/src/portscanner_contracts/models.py +++ b/contracts/src/portscanner_contracts/models.py @@ -603,12 +603,6 @@ class Target(DeterministicModel): transport: TransportProtocol generation: Generation - @property - def address(self) -> str: - """Compatibility alias for the current public scan address.""" - - return self.public_address - def _identity_payload(self) -> Mapping[str, Any]: return { "provider": self.provider, diff --git a/contracts/tests/test_compat.py b/contracts/tests/test_compat.py index 5c187cf..1ab4f90 100644 --- a/contracts/tests/test_compat.py +++ b/contracts/tests/test_compat.py @@ -24,7 +24,7 @@ def test_target_event_accepts_a_decoded_json_mapping() -> None: event = TargetEvent.model_validate(payload) - assert event.target.address == "192.0.2.44" + assert event.target.public_address == "192.0.2.44" assert event.trace_id == event.event_id assert event.provider == event.target.provider diff --git a/db/migrations/000001_core.down.sql b/db/migrations/000001_core.down.sql index 2933125..14218fc 100644 --- a/db/migrations/000001_core.down.sql +++ b/db/migrations/000001_core.down.sql @@ -1,3 +1,56 @@ +DO $$ +DECLARE + managed_role NAME; +BEGIN + FOR managed_role IN + SELECT role_name FROM act.application_role_grants + LOOP + IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = managed_role) THEN + EXECUTE format( + 'REVOKE CONNECT ON DATABASE %I FROM %I', + current_database(), + managed_role + ); + END IF; + END LOOP; +END; +$$; + +DO $$ +DECLARE + restore_public_connect BOOLEAN; +BEGIN + SELECT public_connect + INTO STRICT restore_public_connect + FROM act.database_connect_baseline + WHERE database_name = current_database(); + + IF restore_public_connect THEN + EXECUTE format('GRANT CONNECT ON DATABASE %I TO PUBLIC', current_database()); + ELSE + EXECUTE format('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', current_database()); + END IF; +END; +$$; + +DROP FUNCTION act.grant_application_role(NAME); +DROP FUNCTION act.revoke_application_role(NAME); +DROP FUNCTION act.grant_application_role_objects(NAME); +DROP FUNCTION act.revoke_application_role_objects(NAME); + +DROP VIEW act.pipeline_latency; +DROP VIEW act.current_findings; +DROP VIEW act.current_exposures; +DROP VIEW act.serving_targets; + +DROP TABLE act.application_role_grants; +DROP TABLE act.database_connect_baseline; +DROP TABLE act.finding_handoffs; +DROP TABLE act.reconciliation_runs; +DROP TRIGGER finding_events_append_only ON act.finding_events; +DROP TABLE act.finding_events; +DROP TABLE act.findings; +DROP TABLE act.detection_rules; DROP TRIGGER exposure_events_append_only ON act.exposure_events; DROP TABLE act.exposure_events; DROP TABLE act.exposure_state; diff --git a/db/migrations/000001_core.up.sql b/db/migrations/000001_core.up.sql index db54017..7384947 100644 --- a/db/migrations/000001_core.up.sql +++ b/db/migrations/000001_core.up.sql @@ -66,6 +66,7 @@ CREATE TABLE act.target_events ( received_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), accepted BOOLEAN NOT NULL, stale BOOLEAN NOT NULL, + removed_at TIMESTAMPTZ, CONSTRAINT target_events_event_id_nonempty CHECK (btrim(event_id) <> ''), CONSTRAINT target_events_generation_positive CHECK (generation > 0), CONSTRAINT target_events_type_valid CHECK (event_type IN ('upsert', 'remove')), @@ -78,7 +79,11 @@ CREATE TABLE act.target_events ( AND source_observed_at <= source_collected_at AND source_collected_at <= dispatched_at ), - CONSTRAINT target_events_disposition_valid CHECK (NOT (accepted AND stale)) + CONSTRAINT target_events_disposition_valid CHECK (NOT (accepted AND stale)), + CONSTRAINT target_events_removal_consistent CHECK ( + (event_type = 'upsert' AND removed_at IS NULL) + OR (event_type = 'remove' AND removed_at IS NOT NULL) + ) ); CREATE INDEX target_events_target_generation_idx @@ -127,6 +132,7 @@ CREATE TABLE act.scan_attempts ( stale_generation BOOLEAN NOT NULL, state_eligible BOOLEAN NOT NULL, ingested_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + xml_completion_validated BOOLEAN NOT NULL, CONSTRAINT scan_attempts_attempt_id_nonempty CHECK (btrim(attempt_id) <> ''), CONSTRAINT scan_attempts_result_id_valid CHECK (result_id ~ '^[0-9a-f]{64}$'), CONSTRAINT scan_attempts_run_id_nonempty CHECK (btrim(run_id) <> ''), @@ -188,7 +194,11 @@ CREATE TABLE act.scan_attempts ( ), CONSTRAINT scan_attempts_state_eligibility_valid CHECK ( NOT state_eligible - OR (NOT stale_generation AND outcome = 'complete') + OR ( + NOT stale_generation + AND outcome = 'complete' + AND xml_completion_validated + ) ) ); @@ -198,6 +208,15 @@ CREATE INDEX scan_attempts_target_generation_idx CREATE INDEX scan_attempts_pending_latency_idx ON act.scan_attempts (ingested_at DESC, source_observed_at); +CREATE INDEX scan_attempts_state_order_idx + ON act.scan_attempts ( + target_id, + generation, + scan_completed_at DESC, + attempt_id COLLATE "C" DESC + ) + WHERE state_eligible; + CREATE TABLE act.scan_attempt_coverage ( attempt_id TEXT NOT NULL REFERENCES act.scan_attempts (attempt_id) ON DELETE RESTRICT, protocol TEXT NOT NULL, @@ -297,6 +316,7 @@ CREATE TABLE act.exposure_state ( closure_reason TEXT, last_attempt_id TEXT NOT NULL REFERENCES act.scan_attempts (attempt_id), state_version BIGINT NOT NULL DEFAULT 1, + last_generation BIGINT NOT NULL, PRIMARY KEY (target_id, protocol, port), CONSTRAINT exposure_state_protocol_valid CHECK ( protocol ~ '^[a-z][a-z0-9_-]{0,15}$' @@ -307,6 +327,7 @@ CREATE TABLE act.exposure_state ( service_identity_sha256 ~ '^[0-9a-f]{64}$' ), CONSTRAINT exposure_state_version_positive CHECK (state_version > 0), + CONSTRAINT exposure_state_last_generation_positive CHECK (last_generation > 0), CONSTRAINT exposure_state_closure_valid CHECK ( (state = 'open' AND closed_at IS NULL AND closure_reason IS NULL) OR ( @@ -316,7 +337,9 @@ CREATE TABLE act.exposure_state ( 'explicit_closed', 'explicit_filtered', 'coverage_absence', - 'ownership_removed' + 'ownership_removed', + 'address_binding_changed', + 'generation_gap_reactivation' ) ) ) @@ -363,7 +386,9 @@ CREATE TABLE act.exposure_events ( 'explicit_closed', 'explicit_filtered', 'coverage_absence', - 'ownership_removed' + 'ownership_removed', + 'address_binding_changed', + 'generation_gap_reactivation' ) ), CONSTRAINT exposure_events_previous_hash_valid CHECK ( @@ -385,3 +410,630 @@ CREATE INDEX exposure_events_attempt_idx CREATE TRIGGER exposure_events_append_only BEFORE UPDATE OR DELETE ON act.exposure_events FOR EACH ROW EXECUTE FUNCTION act.reject_append_only_mutation(); + +CREATE TABLE act.detection_rules ( + rule_key TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + severity TEXT NOT NULL, + rule_type TEXT NOT NULL, + match_criteria JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT detection_rules_key_nonempty CHECK (btrim(rule_key) <> ''), + CONSTRAINT detection_rules_name_nonempty CHECK (btrim(name) <> ''), + CONSTRAINT detection_rules_description_nonempty CHECK (btrim(description) <> ''), + CONSTRAINT detection_rules_severity_valid CHECK ( + severity IN ('informational', 'low', 'medium', 'high', 'critical') + ), + CONSTRAINT detection_rules_type_valid CHECK ( + rule_type IN ('exposure', 'port') + ), + CONSTRAINT detection_rules_criteria_object CHECK ( + jsonb_typeof(match_criteria) = 'object' + ) +); + +COMMENT ON COLUMN act.detection_rules.match_criteria IS + 'Editable generic predicate. exposure rules use protocols; port rules use protocols and ports.'; + +INSERT INTO act.detection_rules ( + rule_key, + name, + description, + severity, + rule_type, + match_criteria +) +VALUES + ( + 'new-or-reopened-exposure', + 'New or reopened network exposure', + 'A network service is currently reachable and was newly observed or reopened.', + 'low', + 'exposure', + '{"protocols":["tcp","udp"]}'::JSONB + ), + ( + 'common-public-management-port', + 'Common public management port', + 'A commonly used remote administration or management port is currently reachable.', + 'high', + 'port', + '{"protocols":["tcp"],"ports":[22,23,2375,2376,3389,5900,5985,5986,6443]}'::JSONB + ), + ( + 'common-public-database-port', + 'Common public database port', + 'A commonly used database or data service port is currently reachable.', + 'high', + 'port', + '{"protocols":["tcp"],"ports":[1433,1521,3306,5432,6379,9042,9200,27017]}'::JSONB + ); + +CREATE TABLE act.findings ( + fingerprint CHAR(64) PRIMARY KEY, + target_id TEXT NOT NULL REFERENCES act.targets (target_id), + protocol TEXT NOT NULL, + port INTEGER NOT NULL, + rule_key TEXT NOT NULL REFERENCES act.detection_rules (rule_key) ON UPDATE CASCADE, + status TEXT NOT NULL, + severity TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL, + observed_address INET NOT NULL, + service_name TEXT, + service_product TEXT, + service_version TEXT, + certificate_sha256 CHAR(64), + ssh_host_key_sha256 CHAR(64), + banner_sha256 CHAR(64), + service_identity_sha256 CHAR(64) NOT NULL, + first_opened_at TIMESTAMPTZ NOT NULL, + last_seen_at TIMESTAMPTZ NOT NULL, + last_changed_at TIMESTAMPTZ NOT NULL, + resolved_at TIMESTAMPTZ, + resolution_reason TEXT, + finding_version BIGINT NOT NULL DEFAULT 1, + last_event_key CHAR(64) NOT NULL, + CONSTRAINT findings_fingerprint_valid CHECK (fingerprint ~ '^[0-9a-f]{64}$'), + CONSTRAINT findings_protocol_valid CHECK ( + protocol ~ '^[a-z][a-z0-9_-]{0,15}$' + ), + CONSTRAINT findings_port_valid CHECK (port BETWEEN 1 AND 65535), + CONSTRAINT findings_status_valid CHECK (status IN ('open', 'resolved')), + CONSTRAINT findings_severity_valid CHECK ( + severity IN ('informational', 'low', 'medium', 'high', 'critical') + ), + CONSTRAINT findings_title_nonempty CHECK (btrim(title) <> ''), + CONSTRAINT findings_description_nonempty CHECK (btrim(description) <> ''), + CONSTRAINT findings_identity_hash_valid CHECK ( + service_identity_sha256 ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT findings_version_positive CHECK (finding_version > 0), + CONSTRAINT findings_last_event_key_valid CHECK (last_event_key ~ '^[0-9a-f]{64}$'), + CONSTRAINT findings_resolution_valid CHECK ( + (status = 'open' AND resolved_at IS NULL AND resolution_reason IS NULL) + OR ( + status = 'resolved' + AND resolved_at IS NOT NULL + AND resolution_reason IN ( + 'exposure_closed', + 'ownership_removed', + 'rule_disabled', + 'rule_no_longer_matches' + ) + ) + ), + CONSTRAINT findings_rule_service_unique + UNIQUE (target_id, protocol, port, rule_key) +); + +CREATE INDEX findings_open_target_idx + ON act.findings (target_id, protocol, port) + WHERE status = 'open'; + +CREATE INDEX findings_rule_status_idx + ON act.findings (rule_key, status); + +CREATE TABLE act.finding_events ( + event_sequence BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, + event_key CHAR(64) PRIMARY KEY, + fingerprint CHAR(64) NOT NULL REFERENCES act.findings (fingerprint), + target_id TEXT NOT NULL REFERENCES act.targets (target_id), + target_generation BIGINT NOT NULL, + protocol TEXT NOT NULL, + port INTEGER NOT NULL, + rule_key TEXT NOT NULL REFERENCES act.detection_rules (rule_key) ON UPDATE CASCADE, + event_type TEXT NOT NULL, + previous_status TEXT, + new_status TEXT NOT NULL, + severity TEXT NOT NULL, + reason TEXT NOT NULL, + finding_version BIGINT NOT NULL, + source_kind TEXT NOT NULL, + source_key TEXT NOT NULL, + source_attempt_id TEXT REFERENCES act.scan_attempts (attempt_id), + service_identity_sha256 CHAR(64) NOT NULL, + occurred_at TIMESTAMPTZ NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT finding_events_key_valid CHECK (event_key ~ '^[0-9a-f]{64}$'), + CONSTRAINT finding_events_generation_positive CHECK (target_generation > 0), + CONSTRAINT finding_events_protocol_valid CHECK ( + protocol ~ '^[a-z][a-z0-9_-]{0,15}$' + ), + CONSTRAINT finding_events_port_valid CHECK (port BETWEEN 1 AND 65535), + CONSTRAINT finding_events_type_valid CHECK ( + event_type IN ('opened', 'reopened', 'updated', 'resolved') + ), + CONSTRAINT finding_events_previous_status_valid CHECK ( + previous_status IS NULL OR previous_status IN ('open', 'resolved') + ), + CONSTRAINT finding_events_new_status_valid CHECK ( + new_status IN ('open', 'resolved') + ), + CONSTRAINT finding_events_severity_valid CHECK ( + severity IN ('informational', 'low', 'medium', 'high', 'critical') + ), + CONSTRAINT finding_events_reason_valid CHECK ( + reason IN ( + 'exposure_open', + 'exposure_reopened', + 'service_changed', + 'address_changed', + 'rule_changed', + 'exposure_closed', + 'ownership_removed', + 'rule_disabled', + 'rule_no_longer_matches', + 'reconciliation_repair' + ) + ), + CONSTRAINT finding_events_version_positive CHECK (finding_version > 0), + CONSTRAINT finding_events_source_kind_valid CHECK ( + source_kind IN ('scan_attempt', 'target_event', 'reconciliation') + ), + CONSTRAINT finding_events_source_key_nonempty CHECK (btrim(source_key) <> ''), + CONSTRAINT finding_events_identity_hash_valid CHECK ( + service_identity_sha256 ~ '^[0-9a-f]{64}$' + ) +); + +CREATE INDEX finding_events_fingerprint_idx + ON act.finding_events (fingerprint, event_sequence DESC); + +CREATE INDEX finding_events_attempt_idx + ON act.finding_events (source_attempt_id) + WHERE source_attempt_id IS NOT NULL; + +CREATE TRIGGER finding_events_append_only +BEFORE UPDATE OR DELETE ON act.finding_events +FOR EACH ROW EXECUTE FUNCTION act.reject_append_only_mutation(); + +CREATE TABLE act.reconciliation_runs ( + run_key CHAR(64) PRIMARY KEY, + invocation_key TEXT NOT NULL UNIQUE, + started_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + completed_at TIMESTAMPTZ, + findings_examined INTEGER, + handoffs_queued INTEGER, + CONSTRAINT reconciliation_runs_key_valid CHECK (run_key ~ '^[0-9a-f]{64}$'), + CONSTRAINT reconciliation_runs_invocation_nonempty CHECK (btrim(invocation_key) <> ''), + CONSTRAINT reconciliation_runs_counts_valid CHECK ( + (findings_examined IS NULL OR findings_examined >= 0) + AND (handoffs_queued IS NULL OR handoffs_queued >= 0) + ), + CONSTRAINT reconciliation_runs_completion_valid CHECK ( + completed_at IS NULL + OR ( + findings_examined IS NOT NULL + AND handoffs_queued IS NOT NULL + AND completed_at >= started_at + ) + ) +); + +CREATE TABLE act.finding_handoffs ( + handoff_key CHAR(64) PRIMARY KEY, + fingerprint CHAR(64) NOT NULL REFERENCES act.findings (fingerprint), + finding_event_key CHAR(64) REFERENCES act.finding_events (event_key), + run_key CHAR(64) REFERENCES act.reconciliation_runs (run_key), + source_attempt_id TEXT REFERENCES act.scan_attempts (attempt_id), + handoff_kind TEXT NOT NULL, + object_bucket TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + payload JSONB NOT NULL, + payload_sha256 CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + first_attempted_at TIMESTAMPTZ, + last_attempted_at TIMESTAMPTZ, + published_at TIMESTAMPTZ, + publish_attempts INTEGER NOT NULL DEFAULT 0, + last_error_code TEXT, + CONSTRAINT finding_handoffs_key_valid CHECK (handoff_key ~ '^[0-9a-f]{64}$'), + CONSTRAINT finding_handoffs_kind_valid CHECK ( + handoff_kind IN ('finding_event', 'current_snapshot') + ), + CONSTRAINT finding_handoffs_source_valid CHECK ( + ( + handoff_kind = 'finding_event' + AND finding_event_key IS NOT NULL + AND run_key IS NULL + ) + OR ( + handoff_kind = 'current_snapshot' + AND finding_event_key IS NULL + AND run_key IS NOT NULL + ) + ), + CONSTRAINT finding_handoffs_bucket_nonempty CHECK (btrim(object_bucket) <> ''), + CONSTRAINT finding_handoffs_object_key_nonempty CHECK (btrim(object_key) <> ''), + CONSTRAINT finding_handoffs_payload_object CHECK (jsonb_typeof(payload) = 'object'), + CONSTRAINT finding_handoffs_payload_hash_valid CHECK ( + payload_sha256 ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT finding_handoffs_attempts_nonnegative CHECK (publish_attempts >= 0), + CONSTRAINT finding_handoffs_attempt_times_valid CHECK ( + (publish_attempts = 0 AND first_attempted_at IS NULL AND last_attempted_at IS NULL) + OR ( + publish_attempts > 0 + AND first_attempted_at IS NOT NULL + AND last_attempted_at IS NOT NULL + AND last_attempted_at >= first_attempted_at + ) + ) +); + +CREATE INDEX finding_handoffs_pending_idx + ON act.finding_handoffs (created_at, handoff_key) + WHERE published_at IS NULL; + +CREATE INDEX finding_handoffs_attempt_idx + ON act.finding_handoffs (source_attempt_id) + WHERE source_attempt_id IS NOT NULL; + +CREATE VIEW act.serving_targets AS +SELECT + target_id, + provider, + provider_scope_id, + provider_target_id, + location, + current_generation, + current_addresses, + context, + source_observed_at, + last_attempt_at, + last_confirmed_at, + created_at, + updated_at +FROM act.targets +WHERE status = 'active'; + +CREATE VIEW act.current_exposures AS +SELECT + exposure.target_id, + target.provider, + target.provider_scope_id, + target.provider_target_id, + target.location, + target.current_generation, + exposure.protocol, + exposure.port, + exposure.observed_address, + exposure.service_name, + exposure.service_product, + exposure.service_version, + exposure.certificate_sha256, + exposure.ssh_host_key_sha256, + exposure.banner_sha256, + exposure.service_identity_sha256, + exposure.first_opened_at, + exposure.last_confirmed_at, + exposure.last_changed_at, + exposure.state_version, + exposure.last_attempt_id +FROM act.exposure_state AS exposure +JOIN act.targets AS target USING (target_id) +WHERE exposure.state = 'open' + AND target.status = 'active'; + +CREATE VIEW act.current_findings AS +SELECT + finding.fingerprint, + finding.target_id, + target.provider, + target.provider_scope_id, + target.provider_target_id, + target.location, + target.current_generation, + finding.protocol, + finding.port, + finding.rule_key, + finding.severity, + finding.title, + finding.description, + finding.observed_address, + finding.service_name, + finding.service_product, + finding.service_version, + finding.certificate_sha256, + finding.ssh_host_key_sha256, + finding.banner_sha256, + finding.service_identity_sha256, + finding.first_opened_at, + finding.last_seen_at, + finding.last_changed_at, + finding.finding_version, + finding.last_event_key +FROM act.findings AS finding +JOIN act.targets AS target USING (target_id) +WHERE finding.status = 'open' + AND target.status = 'active'; + +CREATE VIEW act.pipeline_latency AS +WITH first_finding AS ( + SELECT + source_attempt_id, + min(recorded_at) AS first_finding_recorded_at + FROM act.finding_events + WHERE source_attempt_id IS NOT NULL + GROUP BY source_attempt_id +), +first_handoff AS ( + SELECT + source_attempt_id, + min(published_at) AS first_handoff_published_at + FROM act.finding_handoffs + WHERE source_attempt_id IS NOT NULL + AND published_at IS NOT NULL + GROUP BY source_attempt_id +) +SELECT + attempt.attempt_id, + attempt.result_id, + attempt.run_id, + attempt.directive_id, + attempt.target_event_id, + attempt.trace_id, + attempt.target_id, + attempt.generation, + attempt.outcome, + attempt.stale_generation, + attempt.state_eligible, + attempt.source_observed_at, + attempt.dispatched_at, + attempt.scan_started_at, + attempt.scan_completed_at, + attempt.result_uploaded_at, + attempt.ingested_at, + first_finding.first_finding_recorded_at, + first_handoff.first_handoff_published_at, + round( + extract(epoch FROM (attempt.dispatched_at - attempt.source_observed_at)) * 1000 + )::BIGINT AS dispatch_latency_ms, + round( + extract(epoch FROM (attempt.scan_completed_at - attempt.scan_started_at)) * 1000 + )::BIGINT AS scan_latency_ms, + round( + extract(epoch FROM (attempt.ingested_at - attempt.scan_completed_at)) * 1000 + )::BIGINT AS ingestion_latency_ms, + CASE + WHEN first_finding.first_finding_recorded_at IS NULL THEN NULL + ELSE round( + extract( + epoch FROM ( + first_finding.first_finding_recorded_at - attempt.source_observed_at + ) + ) * 1000 + )::BIGINT + END AS finding_latency_ms, + CASE + WHEN first_handoff.first_handoff_published_at IS NULL THEN NULL + ELSE round( + extract( + epoch FROM ( + first_handoff.first_handoff_published_at - attempt.source_observed_at + ) + ) * 1000 + )::BIGINT + END AS handoff_latency_ms +FROM act.scan_attempts AS attempt +LEFT JOIN first_finding + ON first_finding.source_attempt_id = attempt.attempt_id +LEFT JOIN first_handoff + ON first_handoff.source_attempt_id = attempt.attempt_id; + +COMMENT ON VIEW act.current_exposures IS + 'Serving view of open services on active targets, keyed by target_id, protocol, and port.'; +COMMENT ON VIEW act.current_findings IS + 'Serving view containing only unresolved findings on active targets.'; +COMMENT ON VIEW act.pipeline_latency IS + 'End-to-end event, scan, ingestion, finding, and handoff timestamps and latency.'; + +CREATE TABLE act.database_connect_baseline ( + database_name NAME PRIMARY KEY, + public_connect BOOLEAN NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE act.application_role_grants ( + role_name NAME PRIMARY KEY, + granted_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +INSERT INTO act.database_connect_baseline (database_name, public_connect) +SELECT + database.datname, + EXISTS ( + SELECT 1 + FROM aclexplode(COALESCE(database.datacl, acldefault('d', database.datdba))) + WHERE grantee = 0 + AND privilege_type = 'CONNECT' + ) +FROM pg_catalog.pg_database AS database +WHERE database.datname = current_database(); + +DO $$ +BEGIN + EXECUTE format('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', current_database()); +END; +$$; + +CREATE FUNCTION act.grant_application_role_objects(role_to_grant NAME) +RETURNS void +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_grant) THEN + RAISE EXCEPTION 'application role % does not exist', role_to_grant + USING ERRCODE = '42704'; + END IF; + + EXECUTE format('GRANT USAGE ON SCHEMA act TO %I', role_to_grant); + EXECUTE format( + 'GRANT SELECT ON TABLE ' + 'act.detection_rules, act.serving_targets, act.current_exposures, ' + 'act.current_findings, act.pipeline_latency TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT SELECT, INSERT ON TABLE ' + 'act.targets, act.exposure_state, act.findings, ' + 'act.finding_handoffs, act.reconciliation_runs TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT SELECT, INSERT ON TABLE ' + 'act.target_events, act.scan_attempts, act.scan_attempt_coverage, ' + 'act.observations, act.exposure_events, act.finding_events TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT UPDATE (' + 'location, current_generation, status, current_addresses, context, ' + 'source_observed_at, last_attempt_at, last_confirmed_at, updated_at, removed_at' + ') ON TABLE act.targets TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT UPDATE (' + 'state, observed_address, service_name, service_product, service_version, ' + 'certificate_sha256, ssh_host_key_sha256, banner_sha256, ' + 'service_identity_sha256, last_confirmed_at, last_changed_at, closed_at, ' + 'closure_reason, last_attempt_id, state_version' + ') ON TABLE act.exposure_state TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT UPDATE (' + 'status, severity, title, description, observed_address, service_name, ' + 'service_product, service_version, certificate_sha256, ssh_host_key_sha256, ' + 'banner_sha256, service_identity_sha256, last_seen_at, last_changed_at, ' + 'resolved_at, resolution_reason, finding_version, last_event_key' + ') ON TABLE act.findings TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT UPDATE (' + 'first_attempted_at, last_attempted_at, published_at, ' + 'publish_attempts, last_error_code' + ') ON TABLE act.finding_handoffs TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT UPDATE (completed_at, findings_examined, handoffs_queued) ' + 'ON TABLE act.reconciliation_runs TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA act TO %I', + role_to_grant + ); +END; +$$; + +CREATE FUNCTION act.revoke_application_role_objects(role_to_revoke NAME) +RETURNS void +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN + RETURN; + END IF; + + EXECUTE format( + 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA act FROM %I', + role_to_revoke + ); + EXECUTE format( + 'REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA act FROM %I', + role_to_revoke + ); + EXECUTE format('REVOKE USAGE ON SCHEMA act FROM %I', role_to_revoke); +END; +$$; + +CREATE FUNCTION act.grant_application_role(role_to_grant NAME) +RETURNS void +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $$ +BEGIN + PERFORM act.grant_application_role_objects(role_to_grant); + EXECUTE format( + 'GRANT UPDATE (last_generation) ON TABLE act.exposure_state TO %I', + role_to_grant + ); + EXECUTE format( + 'GRANT CONNECT ON DATABASE %I TO %I', + current_database(), + role_to_grant + ); + INSERT INTO act.application_role_grants (role_name) + VALUES (role_to_grant) + ON CONFLICT (role_name) DO UPDATE + SET granted_at = clock_timestamp(); +END; +$$; + +CREATE FUNCTION act.revoke_application_role(role_to_revoke NAME) +RETURNS void +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $$ +BEGIN + PERFORM act.revoke_application_role_objects(role_to_revoke); + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN + EXECUTE format( + 'REVOKE CONNECT ON DATABASE %I FROM %I', + current_database(), + role_to_revoke + ); + END IF; + DELETE FROM act.application_role_grants WHERE role_name = role_to_revoke; +END; +$$; + +REVOKE ALL ON SCHEMA act FROM PUBLIC; +REVOKE ALL ON ALL TABLES IN SCHEMA act FROM PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA act FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION act.reject_append_only_mutation() FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION act.grant_application_role_objects(NAME) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION act.revoke_application_role_objects(NAME) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; + +COMMENT ON FUNCTION act.grant_application_role_objects(NAME) IS + 'Owner-invoked, fail-closed grant for an existing runtime role; grants no DDL, DELETE, or rule editing.'; +COMMENT ON FUNCTION act.grant_application_role(NAME) IS + 'Owner-invoked runtime grant with effective dedicated-database CONNECT and no DDL, DELETE, or rule editing.'; +COMMENT ON TABLE act.application_role_grants IS + 'Roles whose direct CONNECT grant is managed by the ACT application-role functions.'; diff --git a/db/migrations/000002_findings.down.sql b/db/migrations/000002_findings.down.sql deleted file mode 100644 index dced5f3..0000000 --- a/db/migrations/000002_findings.down.sql +++ /dev/null @@ -1,6 +0,0 @@ -DROP TABLE act.finding_handoffs; -DROP TABLE act.reconciliation_runs; -DROP TRIGGER finding_events_append_only ON act.finding_events; -DROP TABLE act.finding_events; -DROP TABLE act.findings; -DROP TABLE act.detection_rules; diff --git a/db/migrations/000002_findings.up.sql b/db/migrations/000002_findings.up.sql deleted file mode 100644 index 83fa5f5..0000000 --- a/db/migrations/000002_findings.up.sql +++ /dev/null @@ -1,281 +0,0 @@ -CREATE TABLE act.detection_rules ( - rule_key TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT TRUE, - severity TEXT NOT NULL, - rule_type TEXT NOT NULL, - match_criteria JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - CONSTRAINT detection_rules_key_nonempty CHECK (btrim(rule_key) <> ''), - CONSTRAINT detection_rules_name_nonempty CHECK (btrim(name) <> ''), - CONSTRAINT detection_rules_description_nonempty CHECK (btrim(description) <> ''), - CONSTRAINT detection_rules_severity_valid CHECK ( - severity IN ('informational', 'low', 'medium', 'high', 'critical') - ), - CONSTRAINT detection_rules_type_valid CHECK ( - rule_type IN ('exposure', 'port') - ), - CONSTRAINT detection_rules_criteria_object CHECK ( - jsonb_typeof(match_criteria) = 'object' - ) -); - -COMMENT ON COLUMN act.detection_rules.match_criteria IS - 'Editable generic predicate. exposure rules use protocols; port rules use protocols and ports.'; - -INSERT INTO act.detection_rules ( - rule_key, - name, - description, - severity, - rule_type, - match_criteria -) -VALUES - ( - 'new-or-reopened-exposure', - 'New or reopened network exposure', - 'A network service is currently reachable and was newly observed or reopened.', - 'low', - 'exposure', - '{"protocols":["tcp","udp"]}'::JSONB - ), - ( - 'common-public-management-port', - 'Common public management port', - 'A commonly used remote administration or management port is currently reachable.', - 'high', - 'port', - '{"protocols":["tcp"],"ports":[22,23,2375,2376,3389,5900,5985,5986,6443]}'::JSONB - ), - ( - 'common-public-database-port', - 'Common public database port', - 'A commonly used database or data service port is currently reachable.', - 'high', - 'port', - '{"protocols":["tcp"],"ports":[1433,1521,3306,5432,6379,9042,9200,27017]}'::JSONB - ); - -CREATE TABLE act.findings ( - fingerprint CHAR(64) PRIMARY KEY, - target_id TEXT NOT NULL REFERENCES act.targets (target_id), - protocol TEXT NOT NULL, - port INTEGER NOT NULL, - rule_key TEXT NOT NULL REFERENCES act.detection_rules (rule_key) ON UPDATE CASCADE, - status TEXT NOT NULL, - severity TEXT NOT NULL, - title TEXT NOT NULL, - description TEXT NOT NULL, - observed_address INET NOT NULL, - service_name TEXT, - service_product TEXT, - service_version TEXT, - certificate_sha256 CHAR(64), - ssh_host_key_sha256 CHAR(64), - banner_sha256 CHAR(64), - service_identity_sha256 CHAR(64) NOT NULL, - first_opened_at TIMESTAMPTZ NOT NULL, - last_seen_at TIMESTAMPTZ NOT NULL, - last_changed_at TIMESTAMPTZ NOT NULL, - resolved_at TIMESTAMPTZ, - resolution_reason TEXT, - finding_version BIGINT NOT NULL DEFAULT 1, - last_event_key CHAR(64) NOT NULL, - CONSTRAINT findings_fingerprint_valid CHECK (fingerprint ~ '^[0-9a-f]{64}$'), - CONSTRAINT findings_protocol_valid CHECK ( - protocol ~ '^[a-z][a-z0-9_-]{0,15}$' - ), - CONSTRAINT findings_port_valid CHECK (port BETWEEN 1 AND 65535), - CONSTRAINT findings_status_valid CHECK (status IN ('open', 'resolved')), - CONSTRAINT findings_severity_valid CHECK ( - severity IN ('informational', 'low', 'medium', 'high', 'critical') - ), - CONSTRAINT findings_title_nonempty CHECK (btrim(title) <> ''), - CONSTRAINT findings_description_nonempty CHECK (btrim(description) <> ''), - CONSTRAINT findings_identity_hash_valid CHECK ( - service_identity_sha256 ~ '^[0-9a-f]{64}$' - ), - CONSTRAINT findings_version_positive CHECK (finding_version > 0), - CONSTRAINT findings_last_event_key_valid CHECK (last_event_key ~ '^[0-9a-f]{64}$'), - CONSTRAINT findings_resolution_valid CHECK ( - (status = 'open' AND resolved_at IS NULL AND resolution_reason IS NULL) - OR ( - status = 'resolved' - AND resolved_at IS NOT NULL - AND resolution_reason IN ( - 'exposure_closed', - 'ownership_removed', - 'rule_disabled', - 'rule_no_longer_matches' - ) - ) - ), - CONSTRAINT findings_rule_service_unique - UNIQUE (target_id, protocol, port, rule_key) -); - -CREATE INDEX findings_open_target_idx - ON act.findings (target_id, protocol, port) - WHERE status = 'open'; - -CREATE INDEX findings_rule_status_idx - ON act.findings (rule_key, status); - -CREATE TABLE act.finding_events ( - event_sequence BIGINT GENERATED ALWAYS AS IDENTITY UNIQUE, - event_key CHAR(64) PRIMARY KEY, - fingerprint CHAR(64) NOT NULL REFERENCES act.findings (fingerprint), - target_id TEXT NOT NULL REFERENCES act.targets (target_id), - target_generation BIGINT NOT NULL, - protocol TEXT NOT NULL, - port INTEGER NOT NULL, - rule_key TEXT NOT NULL REFERENCES act.detection_rules (rule_key) ON UPDATE CASCADE, - event_type TEXT NOT NULL, - previous_status TEXT, - new_status TEXT NOT NULL, - severity TEXT NOT NULL, - reason TEXT NOT NULL, - finding_version BIGINT NOT NULL, - source_kind TEXT NOT NULL, - source_key TEXT NOT NULL, - source_attempt_id TEXT REFERENCES act.scan_attempts (attempt_id), - service_identity_sha256 CHAR(64) NOT NULL, - occurred_at TIMESTAMPTZ NOT NULL, - recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - CONSTRAINT finding_events_key_valid CHECK (event_key ~ '^[0-9a-f]{64}$'), - CONSTRAINT finding_events_generation_positive CHECK (target_generation > 0), - CONSTRAINT finding_events_protocol_valid CHECK ( - protocol ~ '^[a-z][a-z0-9_-]{0,15}$' - ), - CONSTRAINT finding_events_port_valid CHECK (port BETWEEN 1 AND 65535), - CONSTRAINT finding_events_type_valid CHECK ( - event_type IN ('opened', 'reopened', 'updated', 'resolved') - ), - CONSTRAINT finding_events_previous_status_valid CHECK ( - previous_status IS NULL OR previous_status IN ('open', 'resolved') - ), - CONSTRAINT finding_events_new_status_valid CHECK ( - new_status IN ('open', 'resolved') - ), - CONSTRAINT finding_events_severity_valid CHECK ( - severity IN ('informational', 'low', 'medium', 'high', 'critical') - ), - CONSTRAINT finding_events_reason_valid CHECK ( - reason IN ( - 'exposure_open', - 'exposure_reopened', - 'service_changed', - 'address_changed', - 'rule_changed', - 'exposure_closed', - 'ownership_removed', - 'rule_disabled', - 'rule_no_longer_matches', - 'reconciliation_repair' - ) - ), - CONSTRAINT finding_events_version_positive CHECK (finding_version > 0), - CONSTRAINT finding_events_source_kind_valid CHECK ( - source_kind IN ('scan_attempt', 'target_event', 'reconciliation') - ), - CONSTRAINT finding_events_source_key_nonempty CHECK (btrim(source_key) <> ''), - CONSTRAINT finding_events_identity_hash_valid CHECK ( - service_identity_sha256 ~ '^[0-9a-f]{64}$' - ) -); - -CREATE INDEX finding_events_fingerprint_idx - ON act.finding_events (fingerprint, event_sequence DESC); - -CREATE INDEX finding_events_attempt_idx - ON act.finding_events (source_attempt_id) - WHERE source_attempt_id IS NOT NULL; - -CREATE TRIGGER finding_events_append_only -BEFORE UPDATE OR DELETE ON act.finding_events -FOR EACH ROW EXECUTE FUNCTION act.reject_append_only_mutation(); - -CREATE TABLE act.reconciliation_runs ( - run_key CHAR(64) PRIMARY KEY, - invocation_key TEXT NOT NULL UNIQUE, - started_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - completed_at TIMESTAMPTZ, - findings_examined INTEGER, - handoffs_queued INTEGER, - CONSTRAINT reconciliation_runs_key_valid CHECK (run_key ~ '^[0-9a-f]{64}$'), - CONSTRAINT reconciliation_runs_invocation_nonempty CHECK (btrim(invocation_key) <> ''), - CONSTRAINT reconciliation_runs_counts_valid CHECK ( - (findings_examined IS NULL OR findings_examined >= 0) - AND (handoffs_queued IS NULL OR handoffs_queued >= 0) - ), - CONSTRAINT reconciliation_runs_completion_valid CHECK ( - completed_at IS NULL - OR ( - findings_examined IS NOT NULL - AND handoffs_queued IS NOT NULL - AND completed_at >= started_at - ) - ) -); - -CREATE TABLE act.finding_handoffs ( - handoff_key CHAR(64) PRIMARY KEY, - fingerprint CHAR(64) NOT NULL REFERENCES act.findings (fingerprint), - finding_event_key CHAR(64) REFERENCES act.finding_events (event_key), - run_key CHAR(64) REFERENCES act.reconciliation_runs (run_key), - source_attempt_id TEXT REFERENCES act.scan_attempts (attempt_id), - handoff_kind TEXT NOT NULL, - object_bucket TEXT NOT NULL, - object_key TEXT NOT NULL UNIQUE, - payload JSONB NOT NULL, - payload_sha256 CHAR(64) NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), - first_attempted_at TIMESTAMPTZ, - last_attempted_at TIMESTAMPTZ, - published_at TIMESTAMPTZ, - publish_attempts INTEGER NOT NULL DEFAULT 0, - last_error_code TEXT, - CONSTRAINT finding_handoffs_key_valid CHECK (handoff_key ~ '^[0-9a-f]{64}$'), - CONSTRAINT finding_handoffs_kind_valid CHECK ( - handoff_kind IN ('finding_event', 'current_snapshot') - ), - CONSTRAINT finding_handoffs_source_valid CHECK ( - ( - handoff_kind = 'finding_event' - AND finding_event_key IS NOT NULL - AND run_key IS NULL - ) - OR ( - handoff_kind = 'current_snapshot' - AND finding_event_key IS NULL - AND run_key IS NOT NULL - ) - ), - CONSTRAINT finding_handoffs_bucket_nonempty CHECK (btrim(object_bucket) <> ''), - CONSTRAINT finding_handoffs_object_key_nonempty CHECK (btrim(object_key) <> ''), - CONSTRAINT finding_handoffs_payload_object CHECK (jsonb_typeof(payload) = 'object'), - CONSTRAINT finding_handoffs_payload_hash_valid CHECK ( - payload_sha256 ~ '^[0-9a-f]{64}$' - ), - CONSTRAINT finding_handoffs_attempts_nonnegative CHECK (publish_attempts >= 0), - CONSTRAINT finding_handoffs_attempt_times_valid CHECK ( - (publish_attempts = 0 AND first_attempted_at IS NULL AND last_attempted_at IS NULL) - OR ( - publish_attempts > 0 - AND first_attempted_at IS NOT NULL - AND last_attempted_at IS NOT NULL - AND last_attempted_at >= first_attempted_at - ) - ) -); - -CREATE INDEX finding_handoffs_pending_idx - ON act.finding_handoffs (created_at, handoff_key) - WHERE published_at IS NULL; - -CREATE INDEX finding_handoffs_attempt_idx - ON act.finding_handoffs (source_attempt_id) - WHERE source_attempt_id IS NOT NULL; diff --git a/db/migrations/000003_serving.down.sql b/db/migrations/000003_serving.down.sql deleted file mode 100644 index b843dc7..0000000 --- a/db/migrations/000003_serving.down.sql +++ /dev/null @@ -1,6 +0,0 @@ -DROP FUNCTION act.revoke_application_role(NAME); -DROP FUNCTION act.grant_application_role(NAME); -DROP VIEW act.pipeline_latency; -DROP VIEW act.current_findings; -DROP VIEW act.current_exposures; -DROP VIEW act.serving_targets; diff --git a/db/migrations/000003_serving.up.sql b/db/migrations/000003_serving.up.sql deleted file mode 100644 index dedce1b..0000000 --- a/db/migrations/000003_serving.up.sql +++ /dev/null @@ -1,266 +0,0 @@ -CREATE VIEW act.serving_targets AS -SELECT - target_id, - provider, - provider_scope_id, - provider_target_id, - location, - current_generation, - current_addresses, - context, - source_observed_at, - last_attempt_at, - last_confirmed_at, - created_at, - updated_at -FROM act.targets -WHERE status = 'active'; - -CREATE VIEW act.current_exposures AS -SELECT - exposure.target_id, - target.provider, - target.provider_scope_id, - target.provider_target_id, - target.location, - target.current_generation, - exposure.protocol, - exposure.port, - exposure.observed_address, - exposure.service_name, - exposure.service_product, - exposure.service_version, - exposure.certificate_sha256, - exposure.ssh_host_key_sha256, - exposure.banner_sha256, - exposure.service_identity_sha256, - exposure.first_opened_at, - exposure.last_confirmed_at, - exposure.last_changed_at, - exposure.state_version, - exposure.last_attempt_id -FROM act.exposure_state AS exposure -JOIN act.targets AS target USING (target_id) -WHERE exposure.state = 'open' - AND target.status = 'active'; - -CREATE VIEW act.current_findings AS -SELECT - finding.fingerprint, - finding.target_id, - target.provider, - target.provider_scope_id, - target.provider_target_id, - target.location, - target.current_generation, - finding.protocol, - finding.port, - finding.rule_key, - finding.severity, - finding.title, - finding.description, - finding.observed_address, - finding.service_name, - finding.service_product, - finding.service_version, - finding.certificate_sha256, - finding.ssh_host_key_sha256, - finding.banner_sha256, - finding.service_identity_sha256, - finding.first_opened_at, - finding.last_seen_at, - finding.last_changed_at, - finding.finding_version, - finding.last_event_key -FROM act.findings AS finding -JOIN act.targets AS target USING (target_id) -WHERE finding.status = 'open' - AND target.status = 'active'; - -CREATE VIEW act.pipeline_latency AS -WITH first_finding AS ( - SELECT - source_attempt_id, - min(recorded_at) AS first_finding_recorded_at - FROM act.finding_events - WHERE source_attempt_id IS NOT NULL - GROUP BY source_attempt_id -), -first_handoff AS ( - SELECT - source_attempt_id, - min(published_at) AS first_handoff_published_at - FROM act.finding_handoffs - WHERE source_attempt_id IS NOT NULL - AND published_at IS NOT NULL - GROUP BY source_attempt_id -) -SELECT - attempt.attempt_id, - attempt.result_id, - attempt.run_id, - attempt.directive_id, - attempt.target_event_id, - attempt.trace_id, - attempt.target_id, - attempt.generation, - attempt.outcome, - attempt.stale_generation, - attempt.state_eligible, - attempt.source_observed_at, - attempt.dispatched_at, - attempt.scan_started_at, - attempt.scan_completed_at, - attempt.result_uploaded_at, - attempt.ingested_at, - first_finding.first_finding_recorded_at, - first_handoff.first_handoff_published_at, - round( - extract(epoch FROM (attempt.dispatched_at - attempt.source_observed_at)) * 1000 - )::BIGINT AS dispatch_latency_ms, - round( - extract(epoch FROM (attempt.scan_completed_at - attempt.scan_started_at)) * 1000 - )::BIGINT AS scan_latency_ms, - round( - extract(epoch FROM (attempt.ingested_at - attempt.scan_completed_at)) * 1000 - )::BIGINT AS ingestion_latency_ms, - CASE - WHEN first_finding.first_finding_recorded_at IS NULL THEN NULL - ELSE round( - extract( - epoch FROM ( - first_finding.first_finding_recorded_at - attempt.source_observed_at - ) - ) * 1000 - )::BIGINT - END AS finding_latency_ms, - CASE - WHEN first_handoff.first_handoff_published_at IS NULL THEN NULL - ELSE round( - extract( - epoch FROM ( - first_handoff.first_handoff_published_at - attempt.source_observed_at - ) - ) * 1000 - )::BIGINT - END AS handoff_latency_ms -FROM act.scan_attempts AS attempt -LEFT JOIN first_finding - ON first_finding.source_attempt_id = attempt.attempt_id -LEFT JOIN first_handoff - ON first_handoff.source_attempt_id = attempt.attempt_id; - -COMMENT ON VIEW act.current_exposures IS - 'Serving view of open services on active targets, keyed by target_id, protocol, and port.'; -COMMENT ON VIEW act.current_findings IS - 'Serving view containing only unresolved findings on active targets.'; -COMMENT ON VIEW act.pipeline_latency IS - 'End-to-end event, scan, ingestion, finding, and handoff timestamps and latency.'; - -CREATE FUNCTION act.grant_application_role(role_to_grant NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_grant) THEN - RAISE EXCEPTION 'application role % does not exist', role_to_grant - USING ERRCODE = '42704'; - END IF; - - EXECUTE format('GRANT USAGE ON SCHEMA act TO %I', role_to_grant); - EXECUTE format( - 'GRANT SELECT ON TABLE ' - 'act.detection_rules, act.serving_targets, act.current_exposures, ' - 'act.current_findings, act.pipeline_latency TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT SELECT, INSERT ON TABLE ' - 'act.targets, act.exposure_state, act.findings, ' - 'act.finding_handoffs, act.reconciliation_runs TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT SELECT, INSERT ON TABLE ' - 'act.target_events, act.scan_attempts, act.scan_attempt_coverage, ' - 'act.observations, act.exposure_events, act.finding_events TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT UPDATE (' - 'location, current_generation, status, current_addresses, context, ' - 'source_observed_at, last_attempt_at, last_confirmed_at, updated_at, removed_at' - ') ON TABLE act.targets TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT UPDATE (' - 'state, observed_address, service_name, service_product, service_version, ' - 'certificate_sha256, ssh_host_key_sha256, banner_sha256, ' - 'service_identity_sha256, last_confirmed_at, last_changed_at, closed_at, ' - 'closure_reason, last_attempt_id, state_version' - ') ON TABLE act.exposure_state TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT UPDATE (' - 'status, severity, title, description, observed_address, service_name, ' - 'service_product, service_version, certificate_sha256, ssh_host_key_sha256, ' - 'banner_sha256, service_identity_sha256, last_seen_at, last_changed_at, ' - 'resolved_at, resolution_reason, finding_version, last_event_key' - ') ON TABLE act.findings TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT UPDATE (' - 'first_attempted_at, last_attempted_at, published_at, ' - 'publish_attempts, last_error_code' - ') ON TABLE act.finding_handoffs TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT UPDATE (completed_at, findings_examined, handoffs_queued) ' - 'ON TABLE act.reconciliation_runs TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA act TO %I', - role_to_grant - ); -END; -$$; - -CREATE FUNCTION act.revoke_application_role(role_to_revoke NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN - RETURN; - END IF; - - EXECUTE format( - 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA act FROM %I', - role_to_revoke - ); - EXECUTE format( - 'REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA act FROM %I', - role_to_revoke - ); - EXECUTE format('REVOKE USAGE ON SCHEMA act FROM %I', role_to_revoke); -END; -$$; - -REVOKE ALL ON SCHEMA act FROM PUBLIC; -REVOKE ALL ON ALL TABLES IN SCHEMA act FROM PUBLIC; -REVOKE ALL ON ALL SEQUENCES IN SCHEMA act FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.reject_append_only_mutation() FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; - -COMMENT ON FUNCTION act.grant_application_role(NAME) IS - 'Owner-invoked, fail-closed grant for an existing runtime role; grants no DDL, DELETE, or rule editing.'; diff --git a/db/migrations/000004_application_role_connect.down.sql b/db/migrations/000004_application_role_connect.down.sql deleted file mode 100644 index 2c16352..0000000 --- a/db/migrations/000004_application_role_connect.down.sql +++ /dev/null @@ -1,51 +0,0 @@ -DO $$ -DECLARE - managed_role NAME; -BEGIN - FOR managed_role IN - SELECT role_name FROM act.application_role_grants - LOOP - IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = managed_role) THEN - EXECUTE format( - 'REVOKE CONNECT ON DATABASE %I FROM %I', - current_database(), - managed_role - ); - END IF; - END LOOP; -END; -$$; - -DO $$ -DECLARE - restore_public_connect BOOLEAN; -BEGIN - SELECT public_connect - INTO STRICT restore_public_connect - FROM act.database_connect_baseline - WHERE database_name = current_database(); - - IF restore_public_connect THEN - EXECUTE format('GRANT CONNECT ON DATABASE %I TO PUBLIC', current_database()); - ELSE - EXECUTE format('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', current_database()); - END IF; -END; -$$; - -DROP FUNCTION act.grant_application_role(NAME); -DROP FUNCTION act.revoke_application_role(NAME); - -ALTER FUNCTION act.grant_application_role_objects(NAME) - RENAME TO grant_application_role; -ALTER FUNCTION act.revoke_application_role_objects(NAME) - RENAME TO revoke_application_role; - -REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; - -DROP TABLE act.application_role_grants; -DROP TABLE act.database_connect_baseline; - -COMMENT ON FUNCTION act.grant_application_role(NAME) IS - 'Owner-invoked, fail-closed grant for an existing runtime role; grants no DDL, DELETE, or rule editing.'; diff --git a/db/migrations/000004_application_role_connect.up.sql b/db/migrations/000004_application_role_connect.up.sql deleted file mode 100644 index 585e39e..0000000 --- a/db/migrations/000004_application_role_connect.up.sql +++ /dev/null @@ -1,85 +0,0 @@ -ALTER FUNCTION act.grant_application_role(NAME) - RENAME TO grant_application_role_objects; -ALTER FUNCTION act.revoke_application_role(NAME) - RENAME TO revoke_application_role_objects; - -CREATE TABLE act.database_connect_baseline ( - database_name NAME PRIMARY KEY, - public_connect BOOLEAN NOT NULL, - recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() -); - -CREATE TABLE act.application_role_grants ( - role_name NAME PRIMARY KEY, - granted_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() -); - -REVOKE ALL ON TABLE act.database_connect_baseline FROM PUBLIC; -REVOKE ALL ON TABLE act.application_role_grants FROM PUBLIC; - -INSERT INTO act.database_connect_baseline (database_name, public_connect) -SELECT - database.datname, - EXISTS ( - SELECT 1 - FROM aclexplode(COALESCE(database.datacl, acldefault('d', database.datdba))) - WHERE grantee = 0 - AND privilege_type = 'CONNECT' - ) -FROM pg_catalog.pg_database AS database -WHERE database.datname = current_database(); - -DO $$ -BEGIN - EXECUTE format('REVOKE CONNECT ON DATABASE %I FROM PUBLIC', current_database()); -END; -$$; - -CREATE FUNCTION act.grant_application_role(role_to_grant NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.grant_application_role_objects(role_to_grant); - EXECUTE format( - 'GRANT CONNECT ON DATABASE %I TO %I', - current_database(), - role_to_grant - ); - INSERT INTO act.application_role_grants (role_name) - VALUES (role_to_grant) - ON CONFLICT (role_name) DO UPDATE - SET granted_at = clock_timestamp(); -END; -$$; - -CREATE FUNCTION act.revoke_application_role(role_to_revoke NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.revoke_application_role_objects(role_to_revoke); - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN - EXECUTE format( - 'REVOKE CONNECT ON DATABASE %I FROM %I', - current_database(), - role_to_revoke - ); - END IF; - DELETE FROM act.application_role_grants WHERE role_name = role_to_revoke; -END; -$$; - -REVOKE EXECUTE ON FUNCTION act.grant_application_role_objects(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role_objects(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; - -COMMENT ON FUNCTION act.grant_application_role(NAME) IS - 'Owner-invoked, fail-closed runtime grant with explicit database connectivity and no DDL, DELETE, or rule editing.'; -COMMENT ON TABLE act.application_role_grants IS - 'Roles whose direct CONNECT grant is managed by the ACT application-role functions.'; diff --git a/db/migrations/000005_reviewed_state_safety.down.sql b/db/migrations/000005_reviewed_state_safety.down.sql deleted file mode 100644 index 71d8f2e..0000000 --- a/db/migrations/000005_reviewed_state_safety.down.sql +++ /dev/null @@ -1,100 +0,0 @@ -CREATE OR REPLACE FUNCTION act.grant_application_role(role_to_grant NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.grant_application_role_objects(role_to_grant); - EXECUTE format( - 'GRANT CONNECT ON DATABASE %I TO %I', - current_database(), - role_to_grant - ); - INSERT INTO act.application_role_grants (role_name) - VALUES (role_to_grant) - ON CONFLICT (role_name) DO UPDATE - SET granted_at = clock_timestamp(); -END; -$$; - -CREATE OR REPLACE FUNCTION act.revoke_application_role(role_to_revoke NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.revoke_application_role_objects(role_to_revoke); - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN - EXECUTE format( - 'REVOKE CONNECT ON DATABASE %I FROM %I', - current_database(), - role_to_revoke - ); - END IF; - DELETE FROM act.application_role_grants WHERE role_name = role_to_revoke; -END; -$$; - -REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; - -ALTER TABLE act.exposure_events - DROP CONSTRAINT exposure_events_reason_valid, - ADD CONSTRAINT exposure_events_reason_valid CHECK ( - reason IN ( - 'open_observation', - 'service_changed', - 'address_changed', - 'explicit_closed', - 'explicit_filtered', - 'coverage_absence', - 'ownership_removed', - 'address_binding_changed', - 'generation_gap_reactivation' - ) - ); - -ALTER TABLE act.exposure_state - DROP CONSTRAINT exposure_state_closure_valid, - ADD CONSTRAINT exposure_state_closure_valid CHECK ( - (state = 'open' AND closed_at IS NULL AND closure_reason IS NULL) - OR ( - state = 'closed' - AND closed_at IS NOT NULL - AND closure_reason IN ( - 'explicit_closed', - 'explicit_filtered', - 'coverage_absence', - 'ownership_removed', - 'address_binding_changed', - 'generation_gap_reactivation' - ) - ) - ), - DROP CONSTRAINT exposure_state_last_generation_positive, - DROP COLUMN last_generation; - -DROP INDEX act.scan_attempts_state_order_idx; - -ALTER TABLE act.scan_attempts - DROP CONSTRAINT scan_attempts_state_eligibility_valid, - ADD CONSTRAINT scan_attempts_state_eligibility_valid CHECK ( - NOT state_eligible - OR (NOT stale_generation AND outcome = 'complete') - ), - DROP COLUMN xml_completion_validated; - -DROP TRIGGER target_events_append_only ON act.target_events; - -ALTER TABLE act.target_events - DROP CONSTRAINT target_events_removal_consistent, - DROP COLUMN removed_at; - -CREATE TRIGGER target_events_append_only -BEFORE UPDATE OR DELETE ON act.target_events -FOR EACH ROW EXECUTE FUNCTION act.reject_append_only_mutation(); - -COMMENT ON FUNCTION act.grant_application_role(NAME) IS - 'Owner-invoked, fail-closed runtime grant with explicit database connectivity and no DDL, DELETE, or rule editing.'; diff --git a/db/migrations/000005_reviewed_state_safety.up.sql b/db/migrations/000005_reviewed_state_safety.up.sql deleted file mode 100644 index 5c94cb2..0000000 --- a/db/migrations/000005_reviewed_state_safety.up.sql +++ /dev/null @@ -1,140 +0,0 @@ -DROP TRIGGER target_events_append_only ON act.target_events; - -ALTER TABLE act.target_events - ADD COLUMN removed_at TIMESTAMPTZ; - -UPDATE act.target_events -SET removed_at = dispatched_at -WHERE event_type = 'remove'; - -ALTER TABLE act.target_events - ADD CONSTRAINT target_events_removal_consistent CHECK ( - (event_type = 'upsert' AND removed_at IS NULL) - OR (event_type = 'remove' AND removed_at IS NOT NULL) - ); - -CREATE TRIGGER target_events_append_only -BEFORE UPDATE OR DELETE ON act.target_events -FOR EACH ROW EXECUTE FUNCTION act.reject_append_only_mutation(); - -ALTER TABLE act.scan_attempts - ADD COLUMN xml_completion_validated BOOLEAN; - -UPDATE act.scan_attempts -SET xml_completion_validated = state_eligible; - -ALTER TABLE act.scan_attempts - ALTER COLUMN xml_completion_validated SET NOT NULL, - DROP CONSTRAINT scan_attempts_state_eligibility_valid, - ADD CONSTRAINT scan_attempts_state_eligibility_valid CHECK ( - NOT state_eligible - OR ( - NOT stale_generation - AND outcome = 'complete' - AND xml_completion_validated - ) - ); - -CREATE INDEX scan_attempts_state_order_idx - ON act.scan_attempts ( - target_id, - generation, - scan_completed_at DESC, - attempt_id COLLATE "C" DESC - ) - WHERE state_eligible; - -ALTER TABLE act.exposure_state - ADD COLUMN last_generation BIGINT; - -UPDATE act.exposure_state AS exposure -SET last_generation = attempt.generation -FROM act.scan_attempts AS attempt -WHERE attempt.attempt_id = exposure.last_attempt_id; - -ALTER TABLE act.exposure_state - ALTER COLUMN last_generation SET NOT NULL, - ADD CONSTRAINT exposure_state_last_generation_positive CHECK (last_generation > 0), - DROP CONSTRAINT exposure_state_closure_valid, - ADD CONSTRAINT exposure_state_closure_valid CHECK ( - (state = 'open' AND closed_at IS NULL AND closure_reason IS NULL) - OR ( - state = 'closed' - AND closed_at IS NOT NULL - AND closure_reason IN ( - 'explicit_closed', - 'explicit_filtered', - 'coverage_absence', - 'ownership_removed', - 'address_binding_changed', - 'generation_gap_reactivation' - ) - ) - ); - -ALTER TABLE act.exposure_events - DROP CONSTRAINT exposure_events_reason_valid, - ADD CONSTRAINT exposure_events_reason_valid CHECK ( - reason IN ( - 'open_observation', - 'service_changed', - 'address_changed', - 'explicit_closed', - 'explicit_filtered', - 'coverage_absence', - 'ownership_removed', - 'address_binding_changed', - 'generation_gap_reactivation' - ) - ); - -CREATE OR REPLACE FUNCTION act.grant_application_role(role_to_grant NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.grant_application_role_objects(role_to_grant); - EXECUTE format( - 'GRANT UPDATE (last_generation) ON TABLE act.exposure_state TO %I', - role_to_grant - ); - EXECUTE format( - 'GRANT CONNECT ON DATABASE %I TO %I', - current_database(), - role_to_grant - ); - INSERT INTO act.application_role_grants (role_name) - VALUES (role_to_grant) - ON CONFLICT (role_name) DO UPDATE - SET granted_at = clock_timestamp(); -END; -$$; - -CREATE OR REPLACE FUNCTION act.revoke_application_role(role_to_revoke NAME) -RETURNS void -LANGUAGE plpgsql -SECURITY INVOKER -SET search_path = pg_catalog -AS $$ -BEGIN - PERFORM act.revoke_application_role_objects(role_to_revoke); - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = role_to_revoke) THEN - EXECUTE format( - 'REVOKE CONNECT ON DATABASE %I FROM %I', - current_database(), - role_to_revoke - ); - END IF; - DELETE FROM act.application_role_grants WHERE role_name = role_to_revoke; -END; -$$; - -REVOKE EXECUTE ON FUNCTION act.grant_application_role(NAME) FROM PUBLIC; -REVOKE EXECUTE ON FUNCTION act.revoke_application_role(NAME) FROM PUBLIC; - -COMMENT ON TABLE act.application_role_grants IS - 'Roles whose direct CONNECT grant is managed by the ACT application-role functions.'; -COMMENT ON FUNCTION act.grant_application_role(NAME) IS - 'Owner-invoked runtime grant with effective dedicated-database CONNECT and no DDL, DELETE, or rule editing.'; diff --git a/db/migrator/Dockerfile b/db/migrator/Dockerfile index b8a4bf2..b995264 100644 --- a/db/migrator/Dockerfile +++ b/db/migrator/Dockerfile @@ -22,7 +22,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps /tmp/por && rm -rf /tmp/portscanner-migrator RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ USER 65532:65532 ENTRYPOINT ["/usr/local/bin/python", "-m", "awslambdaric"] diff --git a/db/migrator/README.md b/db/migrator/README.md index 7b9ebdc..1f94a0b 100644 --- a/db/migrator/README.md +++ b/db/migrator/README.md @@ -3,27 +3,43 @@ SPDX-FileCopyrightText: 2026 Portscanner contributors SPDX-License-Identifier: MIT --> -# ACT database migrator - -The migrator applies paired PostgreSQL migrations under a database-scoped session -advisory lock. Application credential repair uses the same lock from secret read through -role grants and secret publication. - -Lambda and CLI invocations require the expected `migration_checksum` (or -`--migration-checksum`). The migrator recomputes the ordered, length-prefixed artifact -hash before connecting to PostgreSQL and rejects mismatches or symlinked migration -files. The artifact set must contain only contiguous, paired migrations beginning at -`000001`; unpaired files, version gaps, directories, and unrelated files are rejected -before hashing. - -The `portscanner-migrator` wheel embeds the same canonical SQL files from -`db/migrations`, so a clean `act-migrate` installation has a safe default payload. -`MIGRATIONS_PATH` or `--migrations` may select a separately reviewed artifact set; the -checksum remains mandatory. - -An `up` invocation targeted below migration `000004` applies the requested schema prefix -but deliberately skips application credential provisioning because the earlier object -grants do not guarantee database `CONNECT`. Lambda responses report -`credentials_status: "skipped_target_before_application_connect"` and -`credentials_repaired: false` in that case. Full ups and targets at or above `000004` -repair credentials normally. +# Portscanner database migrator + +The migrator applies paired PostgreSQL migrations under a database-scoped +session advisory lock. The canonical AWS deployment invokes it before +installing the operator, while all scan dispatch is paused. + +## Integrity and locking + +Every Lambda or CLI invocation supplies an expected migration checksum. Before +connecting, the migrator recomputes the ordered, length-prefixed hash of the +complete `db/migrations` artifact set and rejects: + +- a checksum mismatch; +- a missing up/down pair or version gap; +- an unrelated file, directory, or symlink; or +- a previously applied migration whose recorded checksum changed. + +Schema changes and application credential repair share the same advisory lock. +The owner role creates or repairs a least-privilege runtime role, grants +database connectivity and object privileges, then publishes that credential +to Secrets Manager. Runtime components never receive the migration owner +credential. + +The packaged `portscanner-migrator` wheel embeds the same SQL payload. +`MIGRATIONS_PATH` or `--migrations` may select a separately reviewed set, but +the checksum remains mandatory. + +## Development + +From the repository root: + +```sh +uv run --package portscanner-migrator pytest db/migrator/tests +./tools/test_migrator_package.sh +./tools/test_postgresql.sh +``` + +The PostgreSQL test uses a disposable local container. Migration changes must +remain reversible, regenerate the deployment checksum through the image build +flow, and be applied only while dispatch is paused. diff --git a/db/migrator/pyproject.toml b/db/migrator/pyproject.toml index c591ba5..bc955fe 100644 --- a/db/migrator/pyproject.toml +++ b/db/migrator/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "portscanner-migrator" -version = "0.1.0" +version = "1.0.0" description = "Transactional PostgreSQL migrator for the ACT data plane" requires-python = ">=3.12" license = "MIT" diff --git a/db/migrator/requirements-runtime.txt b/db/migrator/requirements-runtime.txt index 79c94b3..7cfc3d5 100644 --- a/db/migrator/requirements-runtime.txt +++ b/db/migrator/requirements-runtime.txt @@ -5,17 +5,29 @@ awslambdaric==4.0.2 \ --hash=sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5 \ --hash=sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948 \ + --hash=sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e \ + --hash=sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f \ --hash=sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378 \ + --hash=sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594 \ + --hash=sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd \ + --hash=sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e \ + --hash=sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f \ --hash=sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb \ + --hash=sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e \ --hash=sha256:92cceacf2e37d455dd1c90a771b0b518f336971dace4cb771a5e43f9a595e867 \ --hash=sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318 \ --hash=sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005 \ + --hash=sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c \ + --hash=sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5 \ --hash=sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35 \ --hash=sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5 \ + --hash=sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c \ + --hash=sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68 \ --hash=sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372 \ --hash=sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6 \ --hash=sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3 \ - --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 + --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 \ + --hash=sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693 boto3==1.43.65 \ --hash=sha256:a8217fb68cae3f8a7575eef395383b68b846d92d29b0e8f6e948e6e9e08dcc3f \ --hash=sha256:f2331154aee1ae97ece48077d77f41d3bd5ea39eb4e3037030448b58695a3a79 diff --git a/db/migrator/src/act_migrator/handler.py b/db/migrator/src/act_migrator/handler.py index 8030a2b..ba59fcc 100644 --- a/db/migrator/src/act_migrator/handler.py +++ b/db/migrator/src/act_migrator/handler.py @@ -15,7 +15,7 @@ from .credentials import DEFAULT_APPLICATION_USERNAME, provision_application_credentials from .migrator import MigrationError, Migrator, discover_migrations, migration_set_checksum -APPLICATION_CONNECT_MIGRATION_VERSION = "000004" +APPLICATION_CONNECT_MIGRATION_VERSION = "000001" def default_migrations_path() -> Path: @@ -35,7 +35,7 @@ def default_migrations_path() -> Path: def target_supports_credential_provisioning(target: str | None) -> bool: - """A partial up below explicit CONNECT cannot safely provision credentials.""" + """The fresh-install baseline includes explicit database CONNECT grants.""" return target is None or target >= APPLICATION_CONNECT_MIGRATION_VERSION diff --git a/db/migrator/tests/test_handler.py b/db/migrator/tests/test_handler.py index eddb60d..796e5c6 100644 --- a/db/migrator/tests/test_handler.py +++ b/db/migrator/tests/test_handler.py @@ -70,13 +70,13 @@ def up(self, *, target: str | None) -> list[str]: def down(self, *, target: str | None, steps: int | None) -> list[str]: assert target is None assert steps == 1 - return ["000004"] + return ["000001"] monkeypatch.setenv("DB_APPLICATION_SECRET_ID", "application-secret") monkeypatch.setattr( handler, "discover_migrations", - lambda _path: [SimpleNamespace(version="000005")], + lambda _path: [SimpleNamespace(version="000001")], ) monkeypatch.setattr(handler, "migration_set_checksum", lambda _path: MIGRATION_CHECKSUM) monkeypatch.setattr(handler.boto3, "client", lambda *_args, **_kwargs: object()) @@ -108,7 +108,7 @@ def test_up_repairs_application_credentials_even_when_schema_is_current( assert result == { "direction": "up", "changed_versions": [], - "latest_version": "000005", + "latest_version": "000001", "credentials_repaired": True, "credentials_status": "repaired", "migration_checksum": MIGRATION_CHECKSUM, @@ -131,54 +131,12 @@ def test_down_never_alters_application_credentials( None, ) - assert result["changed_versions"] == ["000004"] + assert result["changed_versions"] == ["000001"] assert not result["credentials_repaired"] assert result["credentials_status"] == "not_applicable" assert provisions == [] -@pytest.mark.parametrize("target", ["000002", "000003"]) -def test_partial_up_before_connect_migration_explicitly_skips_credentials( - monkeypatch: pytest.MonkeyPatch, - target: str, -) -> None: - provisions = _configure(monkeypatch) - - result = handler.lambda_handler( - { - "direction": "up", - "target": target, - "migration_checksum": MIGRATION_CHECKSUM, - }, - None, - ) - - assert result["changed_versions"] == [target] - assert not result["credentials_repaired"] - assert result["credentials_status"] == "skipped_target_before_application_connect" - assert provisions == [] - - -def test_partial_up_at_connect_migration_repairs_credentials( - monkeypatch: pytest.MonkeyPatch, -) -> None: - provisions = _configure(monkeypatch) - - result = handler.lambda_handler( - { - "direction": "up", - "target": "000004", - "migration_checksum": MIGRATION_CHECKSUM, - }, - None, - ) - - assert result["changed_versions"] == ["000004"] - assert result["credentials_repaired"] - assert result["credentials_status"] == "repaired" - assert len(provisions) == 1 - - def test_rejects_mismatched_migration_artifact_before_connecting( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/db/migrator/tests/test_migrator.py b/db/migrator/tests/test_migrator.py index c653677..c78b28b 100644 --- a/db/migrator/tests/test_migrator.py +++ b/db/migrator/tests/test_migrator.py @@ -13,6 +13,23 @@ def _write_pair(root: Path, version: str, name: str, up: str, down: str) -> None (root / f"{version}_{name}.down.sql").write_text(down, encoding="utf-8") +def test_repository_migration_artifact_is_one_baseline_pair() -> None: + migrations_path = Path(__file__).resolve().parents[2] / "migrations" + + migrations = discover_migrations(migrations_path) + + assert [(migration.version, migration.name) for migration in migrations] == [("000001", "core")] + checksums = [migration.checksum for migration in migrations] + checksums.append(migration_set_checksum(migrations_path)) + assert all( + len(checksum) == 64 and set(checksum) <= set("0123456789abcdef") for checksum in checksums + ) + assert sorted(path.name for path in migrations_path.iterdir()) == [ + "000001_core.down.sql", + "000001_core.up.sql", + ] + + def test_discovers_ordered_pairs_and_hashes_both_directions(tmp_path: Path) -> None: _write_pair(tmp_path, "000002", "second", "SELECT 2;", "SELECT -2;") _write_pair(tmp_path, "000001", "first", "SELECT 1;", "SELECT -1;") diff --git a/db/migrator/tests/test_postgresql_integration.py b/db/migrator/tests/test_postgresql_integration.py index ed813f7..bf1e0f6 100644 --- a/db/migrator/tests/test_postgresql_integration.py +++ b/db/migrator/tests/test_postgresql_integration.py @@ -42,13 +42,13 @@ def _has_direct_connect(connection: Any, role_name: str) -> bool: ) -def _assert_direct_rollback_revokes_managed_connect( +def _assert_baseline_rollback( connection: Any, migrator: Migrator, *, public_connect_baseline: bool, ) -> None: - assert migrator.down(target="000003", steps=None) == ["000004"] + assert migrator.down(target="000000", steps=None) == ["000001"] assert not _has_direct_connect(connection, "act_runtime_test") assert ( connection.execute( @@ -62,6 +62,56 @@ def _assert_direct_rollback_revokes_managed_connect( ).fetchone()[0] is public_connect_baseline ) + assert connection.execute("SELECT to_regnamespace('act')").fetchone()[0] is None + assert connection.execute( + "SELECT rolcanlogin FROM pg_catalog.pg_roles WHERE rolname = 'act_runtime_test'" + ).fetchone() == (True,) + + +def _assert_seeded_detection_rules(connection: Any) -> None: + rows = connection.execute( + """ + SELECT rule_key, name, description, enabled, severity, rule_type, match_criteria + FROM act.detection_rules + ORDER BY rule_key + """ + ).fetchall() + + assert rows == [ + ( + "common-public-database-port", + "Common public database port", + "A commonly used database or data service port is currently reachable.", + True, + "high", + "port", + { + "protocols": ["tcp"], + "ports": [1433, 1521, 3306, 5432, 6379, 9042, 9200, 27017], + }, + ), + ( + "common-public-management-port", + "Common public management port", + "A commonly used remote administration or management port is currently reachable.", + True, + "high", + "port", + { + "protocols": ["tcp"], + "ports": [22, 23, 2375, 2376, 3389, 5900, 5985, 5986, 6443], + }, + ), + ( + "new-or-reopened-exposure", + "New or reopened network exposure", + "A network service is currently reachable and was newly observed or reopened.", + True, + "low", + "exposure", + {"protocols": ["tcp", "udp"]}, + ), + ] def test_fresh_up_down_and_repeat_execution() -> None: @@ -85,23 +135,18 @@ def test_fresh_up_down_and_repeat_execution() -> None: """ ).fetchone()[0] - assert migrator.up() == ["000001", "000002", "000003", "000004", "000005"] + assert migrator.up() == ["000001"] assert migrator.up() == [] rows = connection.execute( - "SELECT version, checksum FROM public.schema_migrations ORDER BY version" + "SELECT version, name, checksum FROM public.schema_migrations ORDER BY version" ).fetchall() - assert [row[0] for row in rows] == [ - "000001", - "000002", - "000003", - "000004", - "000005", - ] - assert [row[1].strip() for row in rows] == [migration.checksum for migration in migrations] + assert [(row[0], row[1]) for row in rows] == [("000001", "core")] + assert [row[2].strip() for row in rows] == [migration.checksum for migration in migrations] assert ( connection.execute("SELECT to_regclass('act.current_findings')").fetchone()[0] == "act.current_findings" ) + _assert_seeded_detection_rules(connection) connection.execute( """ UPDATE act.detection_rules @@ -254,25 +299,6 @@ def put_secret_value(self, **kwargs: Any) -> None: """ ).fetchone()[0] - assert migrator.down(target="000004", steps=None) == ["000005"] - assert _has_direct_connect(connection, "act_runtime_test") - assert connection.execute( - """ - SELECT has_database_privilege( - 'act_runtime_test', - current_database(), - 'CONNECT' - ) - """ - ).fetchone()[0] - - _assert_direct_rollback_revokes_managed_connect( - connection, - migrator, - public_connect_baseline=public_connect_baseline, - ) - - assert migrator.up() == ["000004", "000005"] provision_application_credentials( connection, secrets, @@ -287,20 +313,32 @@ def put_secret_value(self, **kwargs: Any) -> None: application_username="act_runtime_test", password_factory=lambda: pytest.fail("password must not be regenerated"), ) + _assert_baseline_rollback( + connection, + migrator, + public_connect_baseline=public_connect_baseline, + ) finally: - connection.execute("SELECT act.revoke_application_role('act_runtime_test')") + revoke_function_exists = connection.execute( + """ + SELECT EXISTS ( + SELECT 1 + FROM pg_catalog.pg_proc AS procedure + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname = 'act' + AND procedure.proname = 'revoke_application_role' + ) + """ + ).fetchone()[0] + if revoke_function_exists: + connection.execute("SELECT act.revoke_application_role('act_runtime_test')") + migrator.down(target="000000", steps=None) + role_exists = connection.execute( + "SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'act_runtime_test'" + ).fetchone() + if role_exists is not None: + connection.execute("DROP OWNED BY act_runtime_test") + connection.execute("DROP ROLE act_runtime_test") - assert migrator.down(target="000000", steps=None) == [ - "000005", - "000004", - "000003", - "000002", - "000001", - ] - assert connection.execute("SELECT to_regnamespace('act')").fetchone()[0] is None - assert connection.execute( - "SELECT rolcanlogin FROM pg_catalog.pg_roles WHERE rolname = 'act_runtime_test'" - ).fetchone() == (True,) - connection.execute("DROP OWNED BY act_runtime_test") - connection.execute("DROP ROLE act_runtime_test") assert migrator.down(target="000000", steps=None) == [] diff --git a/docs/adding-sources.md b/docs/adding-sources.md deleted file mode 100644 index 6c00ec5..0000000 --- a/docs/adding-sources.md +++ /dev/null @@ -1,589 +0,0 @@ - - -# Adding inventory and signal sources - -This guide defines the implementation contract for a new cloud, on-premises, or asset -inventory source. A source can provide: - -- a **complete inventory snapshot**; -- an optional **change signal** that accelerates a current-state reread; or -- both. - -Every production source needs a complete snapshot path. A signal-only integration is -not sufficient because events can be missed, duplicated, reordered, delayed, or -disabled. - -## Architectural boundary - -A provider adapter ends at normalized source contracts. It must not import or call: - -- scanner command construction; -- Kubernetes Job or custom-resource types; -- result parser/database models; -- exposure reconciliation code; or -- finding/detection policy. - -Conversely, scanners and parsers must not import provider SDKs or understand provider -resource payloads. The scanner consumes a normalized dispatch record. The parser -consumes a versioned result envelope and scanner evidence. - -This boundary keeps a new provider adapter independent of scanner/parser internals. A -source change can add Targets without changing how packets are generated or evidence is -parsed. - -## Required contracts - -The current extension point is -`portscanner_inventory.base.SnapshotBackend`: - -```python -class SnapshotBackend(Protocol): - def collect(self) -> SnapshotBatch: ... -``` - -`collect()` owns the provider API page walk and returns one bounded decision: - -```python -@dataclass(frozen=True) -class SnapshotBatch: - scope: SnapshotScope - targets: tuple[NormalizedTarget, ...] - completion: ScopeCompletion # COMPLETE, PARTIAL, or FAILED - pages: int - malformed_records: int - failure_code: str | None -``` - -The signal and ownership side uses the concrete source primitives -`SignalHint`, `Resolution`, and `OwnershipCheck`. A provider implementation supplies -equivalents of: - -```python -def parse_signal(envelope: Mapping[str, object]) -> SignalHint: ... -def resolve_hint(hint: SignalHint) -> Resolution: ... - -class OwnershipValidator: - def validate(self, target_id: str, generation: int) -> OwnershipCheck: ... -``` - -The first release wires these functions directly for AWS. A second provider should add -an explicit inventory-side provider registry/factory rather than adding provider -conditionals to scanner or parser code. - -### SnapshotScope - -`SnapshotScope` defines the exact state domain covered by one `SnapshotBatch`: - -- source/backend type; -- account/project/tenant boundary supplied through protected configuration; -- Region/location or named aggregator; -- supported resource types; and -- metadata allowlist version, when it affects coverage. - -The current type recognizes the AWS Config and direct EC2 scopes. A new provider must -extend scope validation and key serialization without changing existing keys. If a -provider cannot return one consistent global snapshot, divide it into stable named -partitions and reconcile removals independently in each partition. - -### Pages and completion - -Pagination tokens remain private to the backend. A page is never returned as a complete -snapshot. The backend follows every opaque cursor, detects loops/limits, normalizes and -deduplicates the collected records, and returns `ScopeCompletion.COMPLETE` only after -the provider confirms the final page with no malformed record. Any bounded failure -returns `PARTIAL` or `FAILED` plus a non-sensitive `failure_code`. - -### Target - -The adapter first produces a `NormalizedTarget` containing: - -- stable internal `target_id`; -- provider scope and resource/address-binding locator; -- typed address, resource type, and location; -- lifecycle and scan-relevant policy fingerprint; -- bounded candidate ports derived from current provider state; and -- allowlisted metadata. - -The state layer compares the normalized signature, allocates the monotonic generation -with a conditional write, and builds the versioned public `Target`/`TargetEvent` -contract. Do not let an adapter self-assign a generation or serialize directly to the -scanner queue. - -Keep identity and change reason separate. For an existing stable Target identity, a -changed address binding or scan-relevant non-policy lifecycle field is -`target_change`, not `new_target`. A changed attached-policy set or effective ingress -fingerprint is `policy_change`. `new_target` is reserved for first discovery or -reactivation after a previously removed state. - -The first release accepts public IPv4 Targets only. Reject private, reserved, -documentation, multicast, link-local, loopback, malformed, hostname, wildcard, and IPv6 -Targets at normalization even if later safety gates would also reject them. - -### SignalHint - -A hint contains: - -- signal schema version and source ID; -- provider event ID; -- provider event/observation time and collection time; -- one or more allowlisted resource locators; -- coarse change kind; -- optional bounded candidate-port hints; and -- trace correlation. - -It does not contain authoritative address ownership, Target metadata, scan profile, -scanner arguments, or a deletion instruction. Candidate ports are hints only and must be -recomputed/bounded from the current provider reread. - -### OwnershipCheck - -The current contract returns an `OwnershipCheck` with: - -- verdict: `ACTIVE`, `STALE`, `MOVED`, `INACTIVE`, or `UNKNOWN`; -- normalized current state when it is safe and useful; and -- a bounded non-sensitive reason code. - -The validator receives Target ID and expected generation. Only `ACTIVE` authorizes -dispatch. A reason string is diagnostic and must not contain a -raw provider payload or sensitive identifier. - -## Stable Target identity - -Identity answers “is this the same logical address binding?” It must not depend on -display name, mutable tags, collection time, signal ID, page order, or serialization -order. - -A typical canonical identity input is: - -```text -provider -+ source boundary ID -+ location -+ provider resource type -+ provider resource identity -+ stable address-binding slot -``` - -Hash the length-delimited canonical form if an opaque public ID is needed. Do not -concatenate ambiguous free-form strings. - -For a resource with several address bindings, use a provider-native allocation/binding -identifier. If the provider exposes no stable slot, include the normalized address in -the identity and model an address change as one removal plus one new Target. - -Never use an account number or raw provider resource ID as the externally logged Target -ID. - -## Monotonic generations - -Generation answers “which accepted state of this Target is current?” For one -`target_id`, it must: - -- be an integer; -- never decrease; -- remain the same for an idempotent reread of identical relevant state; and -- increase whenever address ownership or scan-relevant normalized state changes. - -Use the existing persisted compare-and-swap counter keyed by Target ID and normalized -signature. A provider-native revision can contribute to the signature only when its -semantics are documented; it does not replace the state-layer generation fence. - -Do not derive generation from wall-clock seconds, an unordered event timestamp, a random -number, or a hash. Signals can arrive out of order. Competing updates retry the -conditional read/write and either allocate exactly one next generation or observe the -winner. - -The relevant fingerprint should include only normalized fields that affect ownership, -authorization, or scan policy. Metadata that is intentionally informational should not -create endless changed-door work. - -## Complete snapshots and removals - -Removal is the most dangerous snapshot operation. Follow this rule: - -> A Target may be removed only because it is absent from a successfully completed -> snapshot of the same declared boundary and partition. - -Implementation sequence: - -1. Establish one stable `SnapshotScope`. -2. Walk every page using opaque cursors. -3. Validate and stage normalized Targets before returning a batch. -4. Detect cursor loops, duplicate pages, unexpected scope changes, and source - freshness violations. -5. Return `COMPLETE` only after the provider confirms the final page with no malformed - record; otherwise return `PARTIAL`/`FAILED`. -6. Reconcile upserts through conditional state-and-outbox transactions. -7. Consider absence/removal only when the batch is `COMPLETE`. -8. Revalidate each absent Target and conditionally fence its removal by the expected - active generation. - -Timeout, throttling, parse failure, process exit, permission denial, page-limit breach, -or provider inconsistency makes the batch non-complete and emits no removals. Retain a -bounded failure code and retry the entire affected partition. - -An empty complete snapshot can be valid, but an unexpected large drop should enter -quarantine for operator review without being reclassified as success. Document the -source-specific anomaly threshold; it is an acceptance guard, not a substitute for -completeness. - -Signals, point reads, and partial snapshots never remove Targets. - -## Signals are hints - -Signal handling is always: - -```text -authenticate envelope -→ validate source and event shape -→ deduplicate event ID -→ extract resource locator -→ reread current provider state -→ normalize current state -→ compare/allocate generation -→ apply central priority policy -``` - -Do not trust an event's embedded address, tags, owner, security rule, deletion marker, -or timestamp as current state. If the current reread is missing, ambiguous, stale, or -unavailable, record that outcome and let snapshot reconciliation decide lifecycle. - -Coalesce bursts by source and stable resource locator. Reordered older hints should -produce the same current reread result and no additional effective work. - -## Ownership revalidation - -The source must support a cheap, strongly scoped reread suitable immediately before -dispatch. Validate all of: - -- same provider and source boundary; -- same provider resource identity; -- same address-binding identity and address; -- same expected generation/fingerprint; -- current resource class and location; -- current account/CIDR authorization; and -- source freshness within policy. - -Do not cache an `ACTIVE` verdict beyond the configured dispatch window. A retry, -reschedule, or lease takeover repeats the reread. - -Map failures explicitly: - -- generation or scan-relevant policy changed → `STALE`; -- address/resource binding changed → `MOVED`; -- confirmed absence or inactive lifecycle → `INACTIVE`; -- multiple possible owners or inconsistent current state → `UNKNOWN`; -- timeout, throttling, permission failure, malformed response, or stale provider view → - `UNKNOWN`. - -Only a trustworthy positive match is `ACTIVE`. - -## Metadata allowlist and redaction - -Define the adapter allowlist next to the adapter version. For every field specify: - -- normalized key and type; -- source path; -- maximum length/cardinality; -- whether it affects generation or policy; -- whether it may appear in logs/Findings; and -- redaction behavior. - -Drop all unlisted fields before constructing a Target. In particular, do not retain full -provider payloads, arbitrary labels/tags, descriptions, user data, hostnames, personal -contact information, credentials, or unbounded network policy bodies. Provider account -and resource identifiers needed for identity/reread belong only in typed contract/state -fields; never duplicate them into free-form metadata or logs. - -Use opaque references for owner and source. Logs use Target ID, trace ID, verdict, and -bounded reason codes. Tests must prove that an unexpected source field does not survive -normalization. - -## Profile, priority, and deadline policy - -The adapter reports normalized facts and a coarse change kind. Central policy—not the -adapter or signal—selects: - -- `new_target` (**New door**) for first discovery or reactivation; -- `target_change` (**Changed door**) for address-binding or scan-relevant non-policy - lifecycle changes; -- `policy_change` (**Changed door**) for attached-policy or effective-ingress changes; -- `coverage` (**Known door**) for periodic work on unchanged active Targets; -- `manual` (**Manual verification**) only for an explicit operator request; -- priority class; -- named scan profile; -- deadline; -- retry limit; and -- periodic reconciliation cadence. - -The deployment-wide Nmap rates, active-Pod quota, and destination serialization remain -central controls; the adapter must never emit scanner flags, executable command -fragments, or a request to bypass them. - -Compute freshness from source observation and collection timestamps. If an item cannot -be verified before its deadline, expire it without dispatch and rely on current -snapshot state. Reserve scheduler capacity for known-door reconciliation: -**The sweep continues. The change jumps the line.** - -## Registration and configuration - -Adding code is not enough. The first release has direct AWS wiring, so a new provider -must make the inventory boundary explicitly extensible: - -1. Put provider SDK interaction and normalization under - `inventory/src/portscanner_inventory//`. -2. Extend `SnapshotScope` source validation/key encoding without changing existing AWS - keys, and register a `SnapshotBackend` factory in the inventory snapshot handler. -3. Register the signal parser and current-state resolver in the inventory signal - handler. Do not route raw provider events outside `inventory/`. -4. Register a provider-specific ownership validator returning the shared - `OwnershipCheck`. -5. Extend inventory `Settings` with deny-by-default source ID, boundaries, locations, - cadence, signal enablement, metadata allowlist, and freshness limits. -6. Add/extend shared contract provider/resource enums only when needed, regenerate JSON - Schemas/examples, and preserve the rolling compatibility window. -7. Map normalized change facts to the central `ScanReason`/`ScanProfile` policy: - first discovery/reactivation to `new_target`, address or non-policy lifecycle - changes to `target_change`, attached-policy/ingress changes to `policy_change`, and - periodic unchanged work to `coverage`. `target_change` must request fast full TCP, - use `target.upsert`, and omit `PolicyChange`. Default unknown facts to no dispatch. -8. Add metrics with bounded labels; never label by address or raw resource ID. -9. Add dead-letter/quarantine routing and operator runbook entries. -10. Document provider consistency, pagination, revision, deletion, throttling, and - current-state-read semantics. - -Unknown adapter types, schema versions, resource types, or change kinds fail closed. - -No registration step belongs in `scanner/`, `parser/`, the Kubernetes operator, or the -finding engine. Those components continue to consume the existing provider-neutral -TargetEvent/ScanResult boundaries. - -## IAM and Terraform wiring - -Create a dedicated least-privilege source identity. Separate permissions for: - -- complete inventory list/query; -- one-resource current-state get; -- optional event receive/forward; -- queue/object publication; and -- metrics/log writes. - -Do not give the adapter packet-sending, Kubernetes Job creation, evidence parsing, -database migration, or finding-publication permissions. Scanner workers receive no -provider inventory role. - -Terraform changes should include: - -1. variables with no live defaults for source boundary and role references; -2. source module/resources behind `enabled = false` by default; -3. existing-service reference mode before create mode for account-wide inventory or - audit services; -4. least-privilege policy tests; -5. encrypted queues and dead-letter queues; -6. filtered event rules only when signals are enabled; -7. OIDC/workload identity rather than static keys; -8. outputs that avoid account numbers and role/resource IDs where possible; and -9. staged activation documented in [aws-deployment.md](aws-deployment.md). - -For hub/spoke, add spoke trust and event-bus policy independently. A forwarded event -still requires a hub-readable current-state path. Never commit a live account, -organization, role, resource, state, variable file, key, or kubeconfig. - -## Fixtures and tests - -### Fixtures - -Create minimal synthetic fixtures for: - -- one complete page and a multi-page snapshot; -- duplicate resources/pages; -- no public address; -- multiple address bindings; -- changed relevant state; -- unrelated metadata; -- create/update/delete-like signals; -- stale and reordered events; -- active, stale, moved, inactive, and unavailable/ambiguous rereads; and -- malformed/oversized provider payloads. - -Use only RFC 5737/3849 addresses and synthetic identifiers. Keep payloads minimal; do -not copy provider-console exports or production events. - -### Contract tests - -Prove that serialized Targets and hints: - -- validate against the current schema; -- have deterministic canonical serialization and IDs; -- preserve timestamp meaning and ordering; -- reject unknown versions and fields where required; -- contain only allowed metadata; and -- remain readable across the documented rolling-upgrade window. - -### Unit tests - -Prove: - -- every pagination cursor is followed once and cursor loops fail; -- duplicate and reordered input is idempotent; -- identity is stable under name/tag/order changes; -- relevant change increments generation exactly once; -- public-address and non-policy lifecycle changes emit `target_change`, never - `new_target`; -- attached-policy or effective-ingress changes emit `policy_change`; -- first discovery and reactivation after removal emit `new_target`; -- generation never decreases; -- incomplete snapshots remove nothing; -- only complete-snapshot absence emits removal; -- signals force a current-state reread; -- no signal field becomes authoritative Target state; -- each ownership failure maps to the correct non-`ACTIVE` verdict; -- account/CIDR/resource gates fail closed; -- metadata allowlist and bounds hold; -- throttling/backoff respects deadlines; and -- exceptions do not leak sensitive payloads. - -### Integration tests - -With a fake provider and local queues/state: - -1. commit a complete snapshot; -2. replay it and observe no duplicate effective work; -3. change one resource and observe one changed-door item; -4. deliver duplicate/reordered hints and observe one reread/current generation; -5. advance ownership between scheduling and dispatch and observe suppression; -6. abort a later snapshot and observe no removals; -7. complete the snapshot and observe deterministic removal; -8. replay work/results and observe idempotent state; -9. simulate provider/database/queue outages and preserve UNKNOWN; and -10. verify the known-door sweep progresses under a hint burst. - -A live provider test is manual, read-only until separately authorized, uses protected -environment variables, and always cleans up project-created resources. Public CI never -scans a live destination. - -## Failure and UNKNOWN expectations - -Classify failures so operators can distinguish source health from network evidence: - -- source timeout/throttling/permission/schema failure: source UNKNOWN, no removal; -- signal parse/auth failure: quarantine, no reread or dispatch; -- ownership UNKNOWN: no dispatch; -- expired work: no dispatch; -- scanner failure/timeout/partial coverage: Exposure UNKNOWN; -- evidence upload or parse failure: durable retry/quarantine, Exposure unchanged; -- unsupported result/schema version: quarantine, no state mutation; and -- finding publication failure: retry from durable normalized state. - -Never convert absence of a provider response, event, result object, XML/JSON field, -scanner process, or parser record into a closed Exposure. - -## Idempotency expectations - -Use deterministic keys at every boundary: - -```text -snapshot state = source + scope + target + normalized signature -signal hint = source + provider event ID -work = target ID + generation + profile + policy version -dispatch = work ID + ownership-check version -result = dispatch ID + attempt -state update = target ID + generation + declared coverage -``` - -Hash canonical, length-delimited values. Idempotency records must be conditional writes, -not check-then-write races. A retry can refresh a lease or return the previous result, -but cannot create a second effective dispatch or regress state. - -## End-to-end pseudocode - -```python -def reconcile_source(backend, state, ownership, source): - batch = backend.collect() - for target in batch.targets: - state.reconcile(target, source=source) # conditional state + outbox - - if batch.completion != ScopeCompletion.COMPLETE: - return # partial/failed collections remove nothing - - observed = {target.target_id for target in batch.targets} - for prior in state.active_in_scope(batch.scope): - if prior.target_id in observed: - continue - check = ownership.validate(prior.target_id, prior.generation) - if check.verdict == OwnershipVerdict.INACTIVE: - state.remove(prior, source=source) # generation-fenced - - -def handle_signal(parse_signal, resolver, state, envelope): - hint = parse_signal(envelope) - if not state.claim_signal(hint.event_id): - return - resolution = resolver.resolve(hint) # provider current-state reads - if resolution.status == ResolutionStatus.UNKNOWN: - raise RetryableSourceError - for target in resolution.targets: - state.reconcile(target, reason="signal_hint") - - -def dispatch(ownership, work, policy): - require_unexpired(work) - require_policy_scope(work.target, policy) - check = ownership.validate(work.target_id, work.target_generation) - if check.verdict != OwnershipVerdict.ACTIVE: - record_suppressed(work, check) - return - require_policy_scope(check.current, policy) # check again after reread - create_normalized_scan_job(work, policy.named_profile(work.profile)) -``` - -The final function receives no provider payload and emits no provider-specific scanner -arguments. - -## Review checklist - -Source contract: - -- [ ] Source boundary and partition semantics are explicit. -- [ ] Snapshot completeness and consistency are documented. -- [ ] Stable Target identity excludes mutable fields. -- [ ] Generation is monotonic, persisted, and duplicate-safe. -- [ ] Multi-address resources have a stable binding rule. -- [ ] Removal occurs only after a complete snapshot. -- [ ] Signals are hints followed by current-state rereads. -- [ ] Dispatch-time ownership supports every non-`ACTIVE` verdict. - -Safety and data: - -- [ ] Supported resource/address types are allowlisted. -- [ ] Account and CIDR gates default empty. -- [ ] Provider policy and authorization are documented. -- [ ] Metadata is allowlisted, bounded, and redacted. -- [ ] No raw payload reaches logs, scanner, parser, or Findings. -- [ ] Profiles are named and centrally selected. -- [ ] Priority, fairness, deadlines, rates, and retries are bounded. -- [ ] UNKNOWN is preserved for every incomplete path. - -Implementation: - -- [ ] Adapter is registered without scanner/parser imports. -- [ ] Configuration schema and feature gates default disabled. -- [ ] IAM separates list, reread, event, and publication permissions. -- [ ] Terraform supports existing-service mode and staged activation. -- [ ] Hub/spoke trust is scoped and signals retain a reread path. -- [ ] Metrics have bounded, non-sensitive labels. -- [ ] Dead-letter, quarantine, rollback, and cleanup are documented. - -Verification: - -- [ ] Synthetic fixtures use only RFC 5737/3849 addresses and invented IDs. -- [ ] Contract, unit, integration, failure, and idempotency tests pass. -- [ ] Incomplete snapshots are proven not to remove Targets. -- [ ] Duplicate/reordered signals are proven harmless. -- [ ] Ownership changes between queue and dispatch suppress the scan. -- [ ] Known-door reconciliation progresses under priority load. -- [ ] Sanitizer, secret scan, IaC, manifest, image, and generated-drift checks pass. -- [ ] A separately authorized low-rate canary and cleanup plan are reviewed. diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 324334d..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,148 +0,0 @@ - - -# Architecture - -Portscanner separates provider inventory, scheduling, network verification, and finding -policy. The separation is deliberate: a cloud API observation is not network evidence, -and scanner output is not automatically a security finding. - -## Lifecycle - -The system follows **DISCOVER → PRIORITIZE → VERIFY → ACT**. - -### DISCOVER - -A snapshot adapter enumerates every supported resource in its configured boundary and -emits a complete, generation-bearing set of Targets. The reconciler compares that set -with the last accepted complete snapshot: - -- first discovery or reactivation after removal becomes a **new door** - (`new_target`); -- an existing Target's public-address binding or scan-relevant non-policy lifecycle - change becomes a **changed door** (`target_change`); -- an existing Target's attached security-group set or effective ingress change becomes - a **changed door** (`policy_change`); -- unchanged entries remain **known doors** and receive scheduled `coverage`; and -- an absent entry is removed only after the adapter declares the snapshot complete. - -Control-plane signals may request an early reread of one resource. They are hints, not -partial snapshots and not proof of a Target. - -### PRIORITIZE - -Policy converts inventory state into immutable work: - -- new doors use fast full-TCP priority work; -- `target_change` uses fast full-TCP priority work with the same short freshness window - as policy changes; -- `policy_change` uses priority work bounded by the current ingress-derived candidates, - or fast full TCP when those candidates are not narrower; -- known doors stay on a periodic reconciliation cadence; -- every item has a deadline, maximum attempts, and scan profile; deployment-wide - scanner rates, a hard active-Pod quota, and per-destination serialization bound - execution; -- deterministic keys collapse duplicate snapshot pages, signals, and deliveries. - -The scheduling rule is: **The sweep continues. The change jumps the line.** - -Priority work may overtake scheduled work, but cannot consume the baseline queue -indefinitely. - -### VERIFY - -Before dispatch, the source adapter rereads the current provider object and returns an -ownership verdict for the exact Target identity, address, and generation: - -- `ACTIVE`: the binding still belongs to the expected active Target generation; -- `STALE`: the generation or scan-relevant current state changed; -- `MOVED`: the address or resource binding moved; -- `INACTIVE`: the resource/binding is confirmed absent or inactive; -- `UNKNOWN`: the source could not return one trustworthy current owner. - -Only `ACTIVE` can reach a scanner. All other verdicts suppress dispatch and trigger -reconciliation or bounded retry according to policy. - -Workers run from an explicitly approved outside vantage. They receive only normalized -Targets and scan policy; they do not call provider inventory APIs. - -### ACT - -The evidence path stores an immutable result envelope and raw scanner evidence, then -normalizes observations: - -- successful, complete evidence can establish a reachable or not-observed Exposure - within its declared coverage; -- failed, timed-out, stale, missing, malformed, or incomplete evidence remains - **UNKNOWN**; -- detection policy combines Exposure state with current context to create or update a - Finding; -- later complete evidence may resolve a Finding, while missing evidence cannot. - -## Logical components - -1. **Source adapters** normalize complete inventory snapshots and optional signal - payloads into provider-neutral contracts. -2. **Snapshot reconciler** atomically accepts complete snapshots, derives upserts and - removals, and schedules periodic coverage. -3. **Priority scheduler** applies profiles, deadlines, fairness, leases, and retry - policy. -4. **Ownership gate** calls the originating adapter immediately before dispatch. -5. **Orchestrator** converts accepted work into bounded Kubernetes Jobs. -6. **Scanner workers** perform policy-constrained outside-vantage verification and - upload immutable evidence. -7. **Evidence parser** validates envelopes, records coverage and outcome, and updates - Exposure state idempotently. -8. **Finding engine** applies operator-owned policy and emits lifecycle events. -9. **State stores** hold queues, leases, snapshots, evidence, normalized state, and - audit history. - -## Contract boundaries - -Provider adapters depend on source contracts, not scanner or parser implementation. -Scanner workers consume only a normalized dispatch contract. Parsers consume only a -versioned result envelope and evidence format. This permits a new provider adapter -without importing scanner command construction, Kubernetes types, parser database -models, or finding rules. - -Every cross-component record carries: - -- a schema version; -- deterministic event and idempotency identifiers; -- Target identity and monotonic generation; -- source, observation, collection, dispatch, start, and completion timestamps as - applicable; -- scan profile and declared coverage; -- trace correlation; and -- an explicit outcome, including UNKNOWN-causing failures. - -`TargetEvent` keeps the reason distinction on the wire. `new_target` and -`target_change` use `target.upsert`; only `policy_change` uses `policy.changed` and -includes a `PolicyChange` object. `coverage` and `manual` use `rescan.requested`. -Generator-facing readable statuses map `new_target` to **New door**, both change -reasons to **Changed door**, `coverage` to **Known door**, and `manual` to -**Manual verification**. - -## Core invariants - -- A complete snapshot is replaced atomically; an interrupted page walk removes nothing. -- A signal causes a current-state reread; its payload is never accepted as current truth. -- Target identity does not change when mutable labels or display names change. -- Generation never decreases for one Target identity. -- Ownership is revalidated immediately before dispatch. -- One deterministic work key produces at most one effective dispatch. -- A result can only update the Target generation it names. -- Negative Exposure state requires complete declared coverage and successful evidence. -- UNKNOWN is preserved through storage, metrics, and APIs; it is not coerced to closed. -- Priority scheduling never eliminates periodic reconciliation. - -## Default deployment boundary - -The default deployment is one AWS account containing source, queue, orchestration, -evidence, and state resources. An optional hub/spoke design centralizes scheduling and -scanning while source accounts expose narrowly scoped read roles or forward normalized -events. See [multi-account deployment](multi-account.md). - -The trust boundary and threats are detailed in [security-model.md](security-model.md). diff --git a/docs/aws-deployment.md b/docs/aws-deployment.md deleted file mode 100644 index f9d8684..0000000 --- a/docs/aws-deployment.md +++ /dev/null @@ -1,412 +0,0 @@ - - -# AWS deployment - -This guide describes the first-release AWS deployment model. It is a staged procedure, -not a promise that one Terraform apply can safely adopt every existing account. - -Read [scanning-safety.md](scanning-safety.md), configure the mandatory account gate and -any optional CIDR defense, and obtain written authorization before creating scanner -capacity. - -## Deployment modes - -### Single-account default - -The default places inventory readers, queues, idempotency records, Kubernetes -orchestration, evidence storage, and relational state in one AWS account. Use this mode -for an evaluation and for an account whose public assets can be governed by one owner. - -The scanner still needs an outside vantage: packets must reach the target through its -publicly routed edge, not through a private address or an internal load-balancer path. - -### Optional hub/spoke - -A hub can own scheduling, scanning, evidence, and state while spoke accounts either: - -- expose a least-privilege read role that the hub assumes for snapshots and - current-state ownership checks; or -- forward normalized change notifications to the hub, with a read role retained for - authoritative rereads. - -Signals alone are insufficient. The hub must be able to obtain complete snapshots and -revalidate ownership immediately before dispatch. See [multi-account.md](multi-account.md). - -## Clean account and existing-service choices - -Record each choice before planning Terraform. Do not let an evaluation overwrite -account-wide services. - -### AWS Config - -- **Clean-account mode:** set `config_mode = "create"` so the application creates the - recorder, delivery channel, aggregation/query resources, required bucket policy, and - same-account aggregation authorization. One module instance records only its provider - Region, so the created aggregator explicitly includes that Region rather than claiming - all-Region coverage. Each additional member account in that same source Region must - run the member module there and authorize the exact central account and aggregator - Region before the central aggregator includes it. For other source Regions, use - direct EC2 snapshots or an existing aggregator whose per-Region recorders and - authorizations are provisioned outside this one-provider module instance. -- **Existing Config mode:** set `config_mode = "existing"` and provide - `existing_config_aggregator_name`; the adapter receives narrow query/read - permissions. Verify every intended account/Region has a healthy recorder and source - authorization for that aggregator. AWS Config has account/Region singleton constraints; - a second recorder or delivery channel can fail or disrupt established collection. -- **No Config mode:** set `config_mode = "disabled"` and configure the direct EC2 - paginated snapshot backend. Keep periodic reconciliation; a CloudTrail event stream - is not a replacement for inventory. - -### CloudTrail and change signals - -- **Create mode:** set `cloudtrail_mode = "create"` to provision a project-owned - multi-Region management trail. Narrowly filtered EventBridge routes still receive only - events emitted in the Terraform provider Region; a multi-Region trail does not make - one regional EventBridge rule global. -- **Existing mode:** set `cloudtrail_mode = "existing"` and provide - `existing_cloudtrail_arn` for a validated multi-Region management trail rather than - creating a duplicate trail. -- **Disabled mode:** set `cloudtrail_mode = "disabled"` to omit the local CloudTrail and - API-call rule. Native EC2 state hints may still be enabled later, while snapshot-driven - discovery and reconciliation remain authoritative. - -For hot-path signals in another Region, deploy a uniquely named member forwarding root -in that account and Region, pointing it at the central bus. It must create or explicitly -reference a member/organization multi-Region management trail. Periodic Config or direct -EC2 snapshots remain authoritative in every onboarded Region even when no forwarding -root is deployed there. - -Do not enable broad data-event logging merely for this project. Review duplicate trail, -archive, event-bus, and delivery costs before activation. - -### VPC and Kubernetes - -- **Managed VPC mode:** set `create_vpc = true`; Terraform creates dedicated public NAT, - private workload, and isolated database subnets plus controlled routing/endpoints. - `vpc_cidr` must be a canonical AWS IPv4 network from `/16` through `/24`, leaving four - subnet bits for valid AWS subnets. -- **Existing VPC mode:** set `create_vpc = false` and provide the existing VPC and - subnet IDs through protected variables. The module validates VPC membership, - availability-zone spread, public-IP behavior, route shape, and that both VPC DNS - support and DNS hostnames are enabled, without taking ownership of the shared network. - It also requires - `existing_private_subnet_egress_mode`: every private route table must have a matching - NAT-gateway or transit-gateway default route, or `vpc_endpoints` must provide explicit - endpoint IDs for Config, DynamoDB, EC2, ECR API/DKR, EKS, EKS Auth, Logs, S3, Secrets - Manager, SQS, and STS. Terraform verifies each endpoint is available, in the selected - VPC, and exposes the expected regional service. Interface endpoints must have private - DNS and an explicitly selected attached security group; Terraform manages TCP/443 - ingress to that group from the application Lambda and EKS workload security groups. - The S3 and DynamoDB gateway endpoints must be associated with every selected private - route table. AWS China names are checked per service because required endpoints use a - mix of `com.amazonaws` and `cn.com.amazonaws` prefixes. Review endpoint policies - separately because network reachability does not prove that a policy permits every - required API action. Endpoint-only mode can support paused AWS control-plane - infrastructure, but it cannot route scanner traffic to public IPv4 targets. Terraform - rejects canary or full dispatch in that mode; scanner subnets must use validated NAT - gateway or transit-gateway public egress before dispatch. NAT mode requires - `existing_public_nat_gateway_ids` to match every selected private default route and - verifies that each gateway is available, public, and in the selected VPC. Because - Terraform cannot prove the downstream topology of a TGW, that mode additionally - requires a reviewed attestation that the default routes are active/non-blackhole and - the downstream path reaches public egress via - `existing_transit_gateway_public_egress_acknowledged = true`. - -The first release's application module creates its EKS cluster. Adopting an unrelated -existing cluster is not a documented deployment mode. - -The EKS API is private by default. A disposable evaluation may explicitly set -`eks_endpoint_public_access = true` with one or more restricted canonical IPv4 prefixes -in `eks_public_access_cidrs`; private access remains enabled and `0.0.0.0/0` is rejected. -Production should use the private endpoint with approved runner or VPN security groups. - -Confirm that egress has a stable, documented source, packet rates can be limited, return -traffic is allowed, and the path genuinely exercises the public edge. - -### Target authorization - -`authorized_account_ids` is the mandatory inventory and dispatch boundary, and -activation still fails when it is empty. The generator must reread current provider -state and receive an unambiguous `ACTIVE` ownership verdict immediately before creating -scanner work. - -`allowed_target_cidrs` and `denied_target_cidrs` are optional sets of canonical IPv4 -prefixes. Terraform passes them to the generator and through EKS/Helm to the scanner. -The generator applies deny-first membership before ownership API calls or Scanner -creation, and the worker repeats it before Nmap as defense in depth. An empty allowlist -means no additional CIDR restriction; it does not bypass the account or -current-ownership gates. - -CIDR allowlisting is useful defense-in-depth for fixed Elastic IP ranges, but is often -impractical for dynamic public addresses. It is not part of a `Scanner` resource or -user event, and matching a prefix never authorizes scanning a third party. - -## Prerequisites - -- Terraform and the provider versions pinned by the repository; -- an AWS deployment role obtained through short-lived credentials or OIDC; -- Git, the AWS CLI, Docker buildx, `jq`, and Python 3 for the image publication - helper; -- the AWS CLI on every staged runner, with - `PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID` and `PORTSCANNER_EXPECTED_AWS_REGION` set; the - helper verifies STS identity before planning, and Helm later refreshes EKS credentials - with `aws eks get-token`; -- a remote Terraform backend with locking and restricted access; -- a container registry and immutable image-digest policy; -- a Kubernetes cluster or approval to create one; -- a supported PostgreSQL service and an isolated migration identity; -- an authorization record covering every account, Region, any optional CIDR - constraints, rate, port profile, and test window; and -- repository/environment variables for account-specific values. Never commit account - numbers, role ARNs, resource IDs, state, or variable files. - -## Staged activation - -Use separate reviewed plans for each stage. Preserve the redacted textual plan output and -deployment identity in the change record; saved binary plans can contain sensitive -values and the helper removes its temporary plan after applying it. The public helper -accepts `foundation`, `runtime`, `migrate`, `canary`, `pause-canary`, `activate`, or -`pause`: - -```bash -terraform/aws/scripts/deploy.sh "${TF_ROOT}" foundation -``` - -The stages map to six explicit application flags: - -- `deploy_runtime`; -- `run_migration`; -- `install_operator`; -- `enable_event_dispatch`; -- `enable_automatic_inventory`; and -- `canary_mode`. - -The foundation keeps all six false. All examples contain synthetic defaults and expose -names, accounts, VPC/subnets, member maps, API-client security groups, and installer -principals as variables. Supply live values, `image_digests`, and `migration_checksum` -through a reviewed ignored private variable file, never by editing or committing an -example. - -The helper accepts multiple variable files in order, so private environment values can -remain separate from the generated image digest/checksum values: - -```bash -terraform/aws/scripts/deploy.sh \ - "${TF_ROOT}" runtime "${ENV_VARS}" "${IMAGE_VARS}" -``` - -### 1. Plan foundations with all producers disabled - -Create or reference encryption keys, buckets, queues, dead-letter queues, lease storage, -database networking, and the Kubernetes control plane. Keep snapshot publication, -signal consumption, dispatch, and recurring scans disabled. - -Run: - -```bash -terraform -chdir="${TF_ROOT}" init -terraform -chdir="${TF_ROOT}" plan -out="${PLAN_FILE}" -terraform -chdir="${TF_ROOT}" show "${PLAN_FILE}" -``` - -Reject any plan that replaces an existing Config recorder, trail, VPC, database, -cluster, or shared policy unexpectedly. - -### 2. Build, scan, and publish immutable images - -Build each component from the reviewed commit, run its unit and contract tests, scan the -image, generate provenance where supported, and publish by digest. Record the digest; -do not deploy a mutable tag such as `latest`. - -Lambda images must remain single-architecture image manifests. The repository helper -therefore emits attached BuildKit attestations only for the EKS operator/scanner images; -release CI should retain Lambda SBOM and provenance as separate artifacts. - -Terraform creates repositories but never builds images. Populate every required -`image_digests` entry with an immutable `sha256:` digest before the runtime stage. - -Set the same application or example root used for the foundation apply. First run the -side-effect-free validation path: - -```bash -export TF_ROOT="terraform/aws/examples/created-vpc" -terraform/aws/scripts/build-images.sh --dry-run "${TF_ROOT}" arm64 -``` - -Dry run validates the central root, clean source gate, tooling, applied repository and -deployment-state outputs, -repository URL shape, all seven Dockerfile/context mappings, and the migration -checksum. It does not call AWS, log in, build, push, or print a digest block. Use -`PORTSCANNER_ALLOW_DIRTY=true` only when locally testing the helper itself, never for a -release build. - -For an ARM deployment, `arm64` maps to `linux/arm64`, -`lambda_architecture = "arm64"`, and `node_ami_type = -"AL2023_ARM_64_STANDARD"` with an ARM instance such as `t4g.medium`. For x86, -`x86_64` maps to `linux/amd64`, -`lambda_architecture = "x86_64"`, and `node_ami_type = -"AL2023_x86_64_STANDARD"` with an x86 instance such as `t3.medium`. Every component in -one deployment must use the same mapping. - -With the reviewed commit checked out cleanly, Trivy installed, and the AWS identity and -Region configured, push all images and capture only the final HCL: - -```bash -SOURCE_REVISION="$(git rev-parse HEAD)" -terraform/aws/scripts/build-images.sh \ - "${TF_ROOT}" arm64 "${SOURCE_REVISION}" \ - >"$HOME/portscanner-image-inputs.tfvars" -``` - -Inventory, generator, parser, processor, migrator, and scanner use the repository root -as their context; operator uses `operator/`. The helper passes `--pull` and the selected -platform, enables attached provenance/SBOM attestations for EKS images when buildx -supports them, authenticates once per exact ECR registry, pushes directly, and verifies -every returned `sha256:` digest. It accepts the application, created-VPC, existing-VPC, and central -multi-account roots at foundation or a valid later stage, so the same path supports -upgrades. It explicitly rejects the member root and rejects malformed stage ordering, -an account/Region mismatch, unsafe roots or tags, `latest`, and partial output. Before -building each component, it checks the architecture-scoped immutable commit tag and -reuses an existing valid digest. Only a successful ECR tagged-image listing that confirms absence permits a -build; lookup errors or malformed existing digests stop the run. A partially completed -seven-image publication can therefore be rerun without attempting to overwrite -immutable tags. Every reused or newly pushed digest must pass a fixable -HIGH/CRITICAL Trivy scan, and Buildx inspection must show exactly the selected Linux -runtime platform (attestation manifests do not count as runtime platforms), before the -helper emits Terraform input. It never runs Terraform apply or enables dispatch. - -The requested build architecture must equal the applied root's -`workload_architecture` output. The emitted input also preserves that applied contract's -`lambda_architecture`, EKS AMI type, and exact node-instance list, so image and runtime -architecture cannot drift silently. - -ECR count-based expiration is disabled. Optional lifecycle configuration expires only -untagged images; Terraform never expires tagged immutable releases. Operators must keep -every digest used by the active release and rollback window, then retire tags/images -through a separately reviewed release-retention process. - -Progress is written to stderr, so stdout can be redirected safely. The destination is -chosen by the operator and must remain private and uncommitted; the helper does not -create a variable file. Review the resulting `image_digests` map and -`migration_checksum` before passing that file to the paused runtime stage. The checksum -covers sorted migration file names and bytes using the deterministic framing documented -in `terraform/aws/README.md`. - -### 3. Apply runtime and IAM wiring, still paused - -Apply service accounts, OIDC trust, least-privilege policies, queues, event targets, and -digest-pinned functions with `deploy_runtime = true`; keep migration, operator install, -event mappings, schedules, and dispatch off. Verify that each workload uses the expected -digest and that no static cloud credential exists in a Secret or image layer. - -### 4. Run compatible migrations, then install the operator - -Back up the database. The `migrate` stage sets `run_migration = true` with a -content-derived `migration_checksum`, passes that checksum to the migrator, requires the -response to verify the same value, and only then sets `install_operator = true`. The -EKS API must be reachable from the approved runner for the Helm install. - -Set `eks_installer_principal_arns` to stable IAM role/user ARNs (never STS session -ARNs). For the default private API, set `eks_api_client_security_group_ids` to the -runner/VPN security groups. A disposable evaluation may instead opt into the restricted -public endpoint described above. Terraform creates EKS access entries with the -cluster-scoped `AmazonEKSClusterAdminPolicy`; Helm uses AWS CLI exec authentication, -not a plan-cached token. Bootstrap creator admin is disabled by default and is not an -installer substitute. The runner credentials must resolve to one of the explicit -principals and have a permitted network path. - -Migrations must be transactional where supported, idempotently recorded, and tested -against both an empty database and the previously released schema. Do not run -destructive contraction in the same release. Remove obsolete columns or constraints -only after all old writers are gone and rollback is no longer required. - -### 5. Enable the one-shot canary pipeline - -Run the `canary` stage. It enables the priority/result queue and DynamoDB stream -consumers needed for one new Target while keeping signal/coverage consumers, recurring -automatic-inventory schedules, and EventBridge inventory hints disabled. The bounded -outbox-replay repair schedule remains enabled so a stream outage cannot lose committed -TargetEvents. Use an explicit account allowlist and a one-address target CIDR allowlist -as defense-in-depth. - -### 6. Invoke one bounded target - -Invoke the snapshot function once with `account_id`, `region`, and the exact canonical -`target_public_ipv4`. The snapshot adapter still performs read-only inventory pagination, -but reconciliation accepts only the matching address and suppresses absence removals for -that scoped invocation. Canary mode rejects missing or unauthorized account/Region -scope and requires a complete inventory response with exactly one matching target before -reconciliation. - -A new Target receives the bounded `fast-full-tcp` profile over ports 1-65535; obtain -approval for that exact profile rather than describing the canary as a narrow port check. -Verify stable Target identity, ownership rereads, source vantage, deadline handling, -evidence upload, UNKNOWN behavior, finding handoff, and cleanup. - -Created-VPC roots output `scanner_egress_public_ips` so a canary can admit only the -scanner's stable NAT address. All central examples expose the external finding queue and -bucket values needed to verify the handoff. - -### 7. Add signals and periodic coverage - -Run the `activate` stage only after the snapshot/canary checks. It sets -`enable_automatic_inventory = true` in addition to the already tested dispatch pipeline, -enabling recurring schedules and filtered signal rules. Confirm that a duplicate or -reordered signal causes only an idempotent reread and that known-door reconciliation -retains reserved capacity. - -To stop directly after the bounded canary, run `pause-canary`; it retains the runtime's -fail-closed canary requirement while forcing mappings, schedules, and signal rules off. -After full activation, use `pause`. Both pause stages inspect the saved plan and reject -migration, image, Helm, or other changes outside dispatch mappings and rules. - -If Helm refresh or the Kubernetes API is unavailable, use the independent AWS-side -brake instead: - -```bash -terraform/aws/scripts/emergency-pause.sh "${TF_ROOT}" -``` - -It reads state outputs without planning, verifies the expected AWS account and Region, -then directly disables every managed Lambda event-source mapping and EventBridge rule. -It is idempotent and does not contact EKS. This stops new dispatch but does not terminate -scanner Jobs already running; use the separately authorized Kubernetes or AWS -network/node-group stop procedure for those Jobs. - -### 8. Expand in reviewed increments - -Add snapshot Regions, regional forwarding roots, accounts, optional CIDRs, and rates -independently. Do not describe another Region as hot-path covered until its forwarding -root and CloudTrail prerequisite are live. Observe queue age, -ownership-gate verdicts, scan error classes, evidence lag, and cost before each -expansion. - -## Rollback - -Run the normal `pause` stage to disable signal intake and dispatch first. If cluster -reachability prevents the Terraform refresh, run `emergency-pause.sh` first and -reconcile Terraform state later. Preserve inventory and evidence for diagnosis. -Roll workloads back to the prior image digests only while the expanded database schema -remains compatible. Do not reverse a migration that would discard evidence during an -incident. - -After an evaluation, destroy project-created resources with the same Terraform state, -verify queue and object lifecycle cleanup, remove spoke trust, delete temporary -environment variables, and confirm that shared Config, CloudTrail, VPC, and cluster -resources were not changed. - -Buckets retain versions and default to `force_destroy_buckets = false`. A disposable -sandbox that must clean itself up may explicitly set this variable to `true`; retained -or production environments should keep the default and use a reviewed evidence -retention/export procedure before destroy. - -ECR repositories likewise retain images by default. A disposable evaluation may set -`force_delete_repositories = true`; production must keep it false and retire release -images through the reviewed retention process. - -The manual sandbox workflow is intentionally inert until repository variables and an -approval-protected environment are configured. Its cleanup step runs with an -unconditional `always()` guard. diff --git a/docs/configuration.md b/docs/configuration.md deleted file mode 100644 index 9a2d169..0000000 --- a/docs/configuration.md +++ /dev/null @@ -1,288 +0,0 @@ - - -# Configuration - -Configuration has two layers: - -1. Terraform and environment configuration selects cloud resources, trust boundaries, - and feature gates. -2. Versioned policy configuration selects authorized Targets, scan profiles, rates, - deadlines, retries, and retention. - -Secrets, account numbers, role identifiers, Terraform state, variable files, and -kubeconfigs must remain outside the repository. - -## Required safety gates - -Dispatch is permitted only when all gates agree: - -- the source account is in the explicit account allowlist; -- the current provider object is in a supported resource class; -- current ownership revalidation returns `ACTIVE`; -- when a deployment CIDR allowlist is configured, the address is inside it; -- the address is not in a denylist or reserved range; -- the requested profile is approved for that source; -- the hard scanner-Pod quota has capacity and no active Pod owns the same destination; -- the work deadline has not expired; and -- the deployment-wide dispatch feature gate is enabled. - -An empty account allowlist blocks activation. An empty deployment CIDR allowlist adds no -CIDR restriction: the account gate and current ownership revalidation remain mandatory. -CIDR denies are evaluated before allows, and any CIDR parse error fails closed. - -## Logical configuration - -Deployments should expose these groups even if the concrete environment-variable names -differ. - -### Source - -- source identifier and adapter type; -- enabled accounts and Regions; -- snapshot cadence and maximum page duration; -- signal route enabled/disabled; -- adapter read-role reference supplied at deployment time; -- maximum source-data age; -- metadata field allowlist; and -- complete-snapshot timeout and retry budget. - -### Scheduling - -- priority classes for `new_target` (New door), `target_change` and `policy_change` - (Changed door), and `coverage` (Known door); -- fast full-TCP policy for `target_change`, with a short deadline comparable to - `policy_change`; -- explicit `manual` work labeled Manual verification; -- reserved baseline capacity; -- per-class queue age alarms; -- work deadline by profile; -- lease duration and heartbeat; -- maximum delivery attempts; and -- dead-letter destination. - -### Scan policy - -- allowed transports and ports; -- host, CIDR, and profile size limits; -- per-Job Nmap minimum/maximum rates and a deployment-wide active-Pod quota; -- required destination serialization; add independent source token buckets before a - mutually untrusted multi-tenant rollout; -- connect, host, and job timeouts; -- approved scanner arguments; and -- evidence completeness requirements. - -Raw command fragments must not come from a Target, label, signal, or user-controlled -metadata. Select a named profile and construct arguments from typed fields. - -### Data - -- encryption key references; -- queue, object-store, and database endpoints; -- evidence and normalized-state retention; -- dead-letter and quarantine retention; -- log redaction mode; and -- whether raw evidence storage is enabled. - -## First-release AWS mapping - -Terraform exposes the deployment boundary in `terraform/aws/application`: - -- `config_mode` (`create`, `existing`, or `disabled`) and - `existing_config_aggregator_name`; -- `cloudtrail_mode` (`create`, `existing`, or `disabled`) and - `existing_cloudtrail_arn`; -- `create_vpc` plus existing VPC/subnet inputs and the required existing-private-subnet - egress declaration; -- `authorized_account_ids`, `allowed_member_account_ids`, - `member_collector_role_arns`, `allowed_target_cidrs`, and - `denied_target_cidrs`; -- `operator_max_concurrent_reconciles`, `scanner_max_concurrent_pods`, - `scanner_max_jobs`, `scanner_min_rate`, and `scanner_max_rate`; -- `eks_api_client_security_group_ids` and `eks_installer_principal_arns` for the - private Helm runner's network and identity paths, or the evaluation-only - `eks_endpoint_public_access` plus restricted `eks_public_access_cidrs`; -- `snapshot_schedule_expression` (five minutes by default) and - `rescan_schedule_expression` (six hours by default); -- `queue_visibility_timeout_seconds`, 360 seconds by default and required to be at - least six times the Lambda timeout; -- `queue_age_alarm_threshold_seconds` and - `iterator_age_alarm_threshold_seconds` (15 minutes by default); -- optional `alarm_action_arns` and `ok_action_arns`, both empty by default; -- `force_destroy_buckets`, disabled by default and intended only for disposable - sandbox cleanup; -- `force_delete_repositories`, disabled by default and intended only for disposable - ECR cleanup during destroy; -- `ecr_untagged_image_expiration_days`, null by default; no Terraform lifecycle rule - expires tagged release images; -- `image_digests` and `migration_checksum`; and -- staged flags `deploy_runtime`, `run_migration`, `install_operator`, - `enable_event_dispatch`, `enable_automatic_inventory`, and `canary_mode`. - -All scope collections default empty and all activation flags default false. Queue and -stream dispatch can be tested independently while recurring schedules and EventBridge -inventory hints remain off. Event -dispatch still requires nonempty `authorized_account_ids`; the target CIDR sets are -optional defense-in-depth. They are useful when authorization follows fixed Elastic IP -ranges, but are often impractical for dynamic public addresses. - -The EKS public API endpoint defaults off. Enabling it requires at least one canonical -IPv4 prefix, rejects `0.0.0.0/0`, and retains private endpoint access. Use that path only -for a bounded evaluation; production should use private runner or VPN security groups. - -`config_mode = "create"` records only the provider Region and scopes its aggregator to -that explicit Region. A member account in that same source Region must grant -`aws_config_aggregate_authorization` to the exact central account and aggregator Region -before becoming a created-aggregator source. Other Config source Regions require direct -EC2 snapshots or an externally provisioned existing aggregator. Existing aggregators -remain a live prerequisite: Terraform cannot prove their source authorizations or -recorder coverage. - -EventBridge is regional. The application `signal_region` output identifies the only -Region covered by its local rules and central bus. Each member forwarding module -instance covers only its provider Region and requires `cloudtrail_mode = "create"` or -`"existing"` before forwarding API-call events. Snapshot reconciliation covers -onboarded Regions that do not have a hot path. - -For an existing VPC, set `existing_private_subnet_egress_mode` to `nat_gateway`, -`transit_gateway`, or `vpc_endpoints`. Route modes are checked against every supplied -private subnet route table. Existing VPC mode requires both `enableDnsSupport` and -`enableDnsHostnames`. Endpoint mode requires an explicit endpoint ID for every AWS -service path needed by inventory, ECR pulls, EKS nodes/Pod Identity, queues, -object/table storage, logs, STS, and Secrets Manager. Terraform verifies VPC ownership, -availability, partition-aware expected service names, interface private DNS, and -S3/DynamoDB gateway route-table associations. AWS China service names are mapped -individually because the required set mixes `com.amazonaws` and `cn.com.amazonaws`. -Supply one attached security group for every interface endpoint; Terraform adds TCP/443 -ingress from its Lambda and EKS workload security groups. The operator must still verify -that endpoint policies permit every required API action. Endpoint-only mode is valid -only while dispatch is paused; enabling scanner dispatch requires validated NAT/TGW -public egress. NAT mode requires an exact set of public NAT IDs matching every private -default route. TGW mode requires an explicit reviewed public-egress acknowledgement -that includes active/non-blackhole default routes because Terraform cannot inspect the -TGW's downstream route and attachment topology. - -Storage queue and dead-letter alarms are created regardless of action configuration. -Lambda error, throttle, and inventory-outbox iterator-age alarms are created only when -`deploy_runtime` creates the functions. Action values must be existing CloudWatch- -compatible action ARNs; the ARN-shape checks are partition-neutral and do not constrain -an account ID. Terraform does not create SNS, email, or paging resources. Integrate a -team-owned SNS/Pager path externally, then pass its action ARN if notifications are -wanted. The `alarm_names` and `alarm_arns` outputs contain no secrets. - -Terraform passes target CIDRs to the generator and through EKS/Helm as deployment -policy. The generator applies them before Scanner creation; the operator validates the -same canonical IPv4 prefixes at startup and adds `SCANNER_ALLOWED_CIDRS` or -`SCANNER_DENIED_CIDRS` to scanner Jobs only for nonempty lists. CIDRs are not fields in -a `Scanner` resource or user event. Use the created-VPC, -existing-VPC, central multi-account, and member-account examples only as synthetic plan -fixtures; application examples pass no alarm actions, dispatch remains disabled by -default, and both CIDR sets may be empty. - -The inventory runtime validates: - -- `SNAPSHOT_BACKEND` as `config` or `ec2`; -- `CONFIG_AGGREGATOR_NAME` for Config snapshots; -- `AWS_ACCOUNT_ID` and `AWS_REGIONS` for direct EC2 snapshots; -- `DISCOVERY_ROLE_ARN_TEMPLATE` and `DISCOVERY_EXTERNAL_ID` for protected hub reads; -- `ALLOWED_TAG_KEYS` as a subset of the shared AWS metadata contract; and -- bounded `SIGNAL_DEDUPE_SECONDS`, `OUTBOX_TTL_SECONDS`, - `RESCAN_BUCKET_SECONDS`, `MAX_SIGNAL_PORT_RANGES`, and `MAX_SIGNAL_PORTS`. - -The generator separately validates queue/object, idempotency table, cluster, namespace, -lease, event-size, and audit-retention settings. Do not bypass those typed settings with -free-form scanner flags. - -## Example policy - -The following networks are documentation-only and non-routable. They demonstrate -the logical policy shape; this YAML is not consumed directly by the first-release -runtime and these networks are not scan targets. The `cidrs` list illustrates an -optional deployment constraint, not a substitute for the mandatory account and current -ownership gates. - -```yaml -version: 1 -dispatch_enabled: false - -authorization: - accounts: - - "${ACCOUNT_ID}" - cidrs: - - "192.0.2.0/28" - - "198.51.100.16/28" - denied_cidrs: - - "203.0.113.128/25" - require_current_ownership: true - -sources: - - id: "aws-primary" - adapter: "aws" - regions: - - "${AWS_REGION}" - snapshot_interval: "30m" - signals_enabled: false - max_source_age: "15m" - metadata_allowlist: - - "environment" - - "service" - - "owner_ref" - -profiles: - priority: - transports: - tcp: [22, 80, 443] - deadline: "10m" - job_timeout: "5m" - max_attempts: 2 - reconciliation: - transports: - tcp: [22, 80, 443] - deadline: "2h" - job_timeout: "15m" - max_attempts: 2 - -rates: - max_concurrent_jobs: 2 - max_packets_per_second: 100 - per_destination_packets_per_second: 20 - baseline_capacity_percent: 25 - -retention: - raw_evidence_days: 30 - normalized_state_days: 180 - dead_letter_days: 14 -``` - -IPv6 documentation may use only `2001:db8::/32`; IPv6 scanning is outside the first -release scope. - -## Feature gates - -Keep independently reversible gates for: - -- snapshot collection; -- snapshot publication; -- signal intake; -- work creation; -- ownership revalidation; -- dispatch; -- result ingestion; -- finding emission; and -- recurring reconciliation. - -No gate may bypass ownership or authorization checks. A canary gate narrows scope; it -does not weaken safety policy. - -## Environment-specific values - -Use repository or protected-environment variables for non-secret deployment references -and a cloud secret service for credentials. CI examples use names such as -`AWS_SANDBOX_ROLE_ARN`, `AWS_SANDBOX_REGION`, and `AWS_TERRAFORM_ROOT`; they intentionally -contain no live values. - -Validate configuration in CI and again at process startup. Log a configuration digest -and safe feature-gate summary, but never render secret values, full provider payloads, -account numbers, or unredacted tags. diff --git a/docs/costs.md b/docs/costs.md deleted file mode 100644 index df8b768..0000000 --- a/docs/costs.md +++ /dev/null @@ -1,137 +0,0 @@ - - -# Costs - -Portscanner has no universal cost estimate. Inventory volume, snapshot cadence, change -rate, scan profile, network path, evidence retention, database choice, and cluster -utilization dominate the result. Use current provider pricing for the selected Regions. - -## Cost drivers - -### Inventory and signals - -- provider inventory/configuration recording and queries; -- API request and pagination volume; -- trail or event-bus ingestion, archive, and delivery; -- cross-account or cross-Region event forwarding; and -- queue requests and dead-letter retention. - -Reusing an existing Config or CloudTrail setup may reduce duplicate collection, but it -can also add charges to a shared service. Confirm ownership and marginal cost. Do not -create a duplicate organization trail by default. - -### Scheduling and orchestration - -- serverless invocations and duration; -- lease/idempotency reads and writes; -- standard-resolution CloudWatch metric alarms; -- Kubernetes control plane, nodes, autoscaling floor, and logs; and -- container registry storage and vulnerability scanning. - -Priority capacity should share a bounded pool with periodic reconciliation. Permanent -overprovisioning for a rare burst is usually more expensive than a measured queue-age -budget. - -The application creates 12 storage alarms (six primary queues and six dead-letter -queues). Deploying the ten Lambda functions adds 20 error/throttle alarms and one -outbox iterator-age alarm, for 33 total. These are standard-resolution metric alarms -and are usually a small low-single-digit monthly charge at common regional prices, but -use current CloudWatch pricing for the deployment Region. Externally managed SNS, -paging, and notification delivery can add separate charges. - -### Network verification - -A rough probe-volume model is: - -```text -probes per cycle - = targets - × selected ports - × transports - × attempts/retries -``` - -Actual packets are higher because discovery, handshakes, service detection, scripts, -timeouts, and retransmission differ by profile. Data-transfer charges depend on scanner -placement and return traffic. Full-port, UDP, service-detection, and script profiles -must be estimated separately. - -### Data - -- raw evidence object count, size, versions, and retention; -- database compute, storage, I/O, backups, and replicas; -- logs, metrics, traces, and search/index retention; -- encryption-key requests; and -- cross-Region replication or disaster recovery. - -Raw scanner output and verbose logs can outgrow normalized state. Apply retention, -compression, and log redaction deliberately; do not discard evidence required for -correct UNKNOWN or Finding lifecycle handling. - -## Estimation worksheet - -For each source, measure or estimate: - -1. supported resources per complete snapshot; -2. pages and API calls per snapshot; -3. snapshots per day; -4. relevant signals per day and duplicate ratio; -5. new, changed, and known Targets per day; -6. ports, attempts, expected duration, and evidence size by profile; -7. peak and average concurrent Jobs; -8. queue, object, database, and log retention; and -9. cross-account/Region data paths. - -Model baseline and priority traffic independently: - -```text -daily work - = periodic known-door work - + deduplicated new/changed-door work - + bounded retries -``` - -Then apply current service prices and a safety margin. Validate the model with a -low-rate authorized canary before increasing scope. - -## Single-account versus hub/spoke - -Single-account deployment minimizes trust and forwarding resources but duplicates -control planes if each account operates independently. - -Hub/spoke deployment can consolidate Kubernetes, database, and evidence services. It -adds assume-role calls, event forwarding, centralized egress, larger failure domains, -and possible cross-account/Region transfer. A hub is not automatically cheaper; compare -both models at expected scale. - -## Cost controls - -- dispatch defaults off and authorization lists default empty; -- cap accounts, Regions, Targets, ports, retries, packet rates, and concurrent Jobs; -- reserve baseline capacity without creating an unbounded priority fleet; -- coalesce signals by Target and generation; -- expire stale work before resource creation; -- autoscale workers with explicit minimum and maximum; -- use object lifecycle and database retention; -- sample high-volume debug logs and exclude raw evidence from logs; -- set provider budgets and anomaly alerts outside this public repository; and -- tag project-owned resources with a non-sensitive cost-allocation label. - -## Cost alarms - -Alert on: - -- inventory request or event-delivery spikes; -- queue redrive loops; -- scan attempts per effective Target; -- worker-hours with no accepted evidence; -- object version growth; -- database I/O/storage deviation; -- log ingestion deviation; and -- resources remaining after sandbox cleanup. - -Cost pressure must not be handled by treating missing evidence as closed. Reduce cadence -or scope explicitly and expose the resulting UNKNOWN/coverage gap. diff --git a/docs/data-handling.md b/docs/data-handling.md deleted file mode 100644 index 68243bd..0000000 --- a/docs/data-handling.md +++ /dev/null @@ -1,142 +0,0 @@ - - -# Data handling - -Inventory and scan evidence can reveal attack-surface details even when no credential is -present. Treat all live deployment data as confidential operational security data. - -## Data classes - -### Source inventory - -May include account and Region references, provider resource IDs, public addresses, -network relationships, labels, and ownership context. Adapters must normalize only -fields required for stable identity, authorization, prioritization, and revalidation. - -### Work and audit records - -Include Target identity, generation, source timestamps, profile, deadline, ownership -verdict, dispatch outcome, and trace identifiers. Keep enough to explain why a scan did -or did not occur without copying full provider payloads. - -### Raw evidence - -May include addresses, ports, banners, certificates, software versions, protocol -responses, and scanner diagnostics. Raw evidence is immutable and access-restricted. -Do not place it in logs, CI artifacts, issues, or public test fixtures. - -### Normalized Exposure and Finding state - -Contains reachability observations, declared coverage, confidence/outcome, policy -context, and lifecycle history. This data can be as sensitive as raw evidence. - -## Metadata allowlist - -Provider metadata is deny-by-default. Each adapter declares a versioned allowlist of -normalized keys. A reasonable initial set is: - -- a non-personal environment classification; -- a service reference that contains no hostname or account identifier; -- an opaque owner reference intended for machine routing; and -- policy labels with documented bounded values. - -Do not ingest arbitrary tags, descriptions, user data, names, email addresses, ticket -bodies, deployment payloads, secrets, or full provider API responses. - -For each allowed field: - -- cap key and value length; -- normalize encoding; -- reject control characters; -- define whether the value may enter logs or Findings; -- test known secret/PII forms; and -- document the retention purpose. - -Redact rather than hash low-entropy identifiers. A hash of an account number, address, -or short name can often be reversed and can still permit unwanted correlation. - -## Storage controls - -- Encrypt queues, objects, databases, backups, and Terraform state. -- Use separate least-privilege read/write identities for source, scanner, parser, - finding, migration, and backup paths. -- Prefer immutable object keys and versioning for raw evidence. -- Block public object access and require transport encryption. -- Keep database and administrative endpoints off the public Internet. -- Enable access logging with the same redaction policy. -- Test restoration into an isolated environment with dispatch disabled. - -## Logging - -Logs should use opaque Target and trace identifiers. Avoid: - -- live account numbers and role/resource identifiers; -- IP addresses and CIDRs; -- provider payloads and tags; -- scanner command output, banners, or certificates; -- credentials, tokens, signed URLs, database strings, and request headers; and -- personal names, contact details, or local filesystem paths. - -Use bounded error classes such as `source_throttled`, `ownership_stale`, -`evidence_incomplete`, and `result_schema_invalid`. Put detailed evidence in the -restricted evidence store. - -## Retention and deletion - -Define retention separately for: - -- accepted and failed snapshots; -- signals and dead letters; -- work/lease audit records; -- raw evidence; -- normalized Exposures; -- Finding history; -- logs and metrics; and -- backups. - -Retention should meet the investigation purpose without becoming indefinite by default. -Deletion must account for object versions, replicas, snapshots, dead-letter queues, -search indexes, and backups. A Target removal does not automatically prove that all -historical evidence should be deleted; follow the operator's approved policy. - -## Public fixtures and documentation - -Public repository content must use only: - -- RFC 5737 IPv4 documentation ranges; -- RFC 3849 IPv6 documentation addresses; -- explicitly synthetic identifiers; -- generic account/role variables; and -- invented metadata that cannot identify a person or environment. - -Do not copy production events, scanner output, dashboards, measurements, screenshots, -slide assets, names, local paths, state, variable files, keys, or kubeconfigs. - -`tools/sanitize.py` checks tracked files by default and can include untracked working -tree files with `--working-tree`. Its policy rejects internal terms, live-looking cloud -identifiers, personal data, non-documentation public addresses, sensitive file types, -all binary files by default, and symlinks that escape the repository. An intentionally -distributable binary requires a security-reviewed allowlist entry containing both its -exact repository-relative path and lowercase SHA-256; there are currently no such -entries. A changed path or byte fails closed. Git-ignored local caches remain outside -working-tree path selection rather than being scanned or allowlisted. The sanitizer is -a release gate, not a substitute for review. - -Narrow sanitizer exceptions are limited to legal notice files, the public project URL, -and explicitly listed synthetic fixture values. Changes to the policy or its exceptions -require security-focused review. - -## Incident handling - -If sensitive data reaches the repository: - -1. stop publication and disable affected automation; -2. treat exposed credentials or signed material as compromised and rotate/revoke them; -3. use the hosting provider's private security process; -4. remove the data from the working tree and, when required, repository history; -5. invalidate caches and artifacts; -6. run both secret scanning and the publication sanitizer; and -7. document the control failure without reproducing the sensitive value. diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index d1213c8..0000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,343 +0,0 @@ - - -# Getting started - -This path proves the repository locally, deploys one paused AWS evaluation, verifies one -explicitly authorized canary, and then pauses scanning before any broader integration. - -The AWS evaluation is not free or one-click. Its foundation creates a VPC, NAT gateway, -EKS control plane and node, Aurora PostgreSQL, queues, buckets, tables, repositories, -logs, and alarms. Use a dedicated sandbox account, review current AWS -pricing and quotas, and destroy the evaluation promptly. Never point it at an address -you do not own or have written authorization to test. - -## 1. Prove the checkout locally - -Required for the full local path: - -- Python 3.12 and [`uv`](https://docs.astral.sh/uv/); -- Go from `operator/go.mod`; -- Terraform 1.7.4; -- `kubectl` with Kustomize support, Helm 3, and kubeconform 0.8; and -- Docker for the isolated PostgreSQL integration tests. - -Run: - -```bash -git clone https://github.com/Roblox/portscanner.git -cd portscanner - -make sync -make check -make test-go -make test-go-envtest -make vet-go -make test-postgresql -make terraform-script-tests -make terraform-validate -make kubernetes-validate -``` - -These checks use synthetic data and do not scan a network target. The PostgreSQL target -starts a digest-pinned local PostgreSQL 16 container, applies every migration, exercises -the parser/database path, and removes the container. - -## 2. Prepare one AWS evaluation - -You need: - -- a dedicated AWS account or an approved sandbox boundary; -- a short-lived deployment identity with permission to create the documented resources; -- one stable IAM role or user ARN for EKS installation (not an STS session ARN); -- one public IPv4 address of the Terraform runner for restricted EKS API access; -- one public IPv4 canary attached to an in-use EC2 network interface in the authorized - account and Region; and -- written authorization for the account, canary `/32`, scan window, and source egress. - -The image publication step additionally requires Git, the AWS CLI, Docker buildx, `jq`, -Python 3, `tar`, and Trivy. - -Copy the guarded evaluation values: - -```bash -export TF_ROOT="$PWD/terraform/aws/examples/created-vpc" -export TF_VARS="$TF_ROOT/evaluation.tfvars" - -cp "$TF_ROOT/evaluation.tfvars.example" "$TF_VARS" -chmod 600 "$TF_VARS" -``` - -Replace every `REPLACE_*` value. Confirm the account before continuing: - -```bash -export AWS_REGION="us-east-1" -export AWS_ACCOUNT_ID="$( - aws sts get-caller-identity \ - --region "$AWS_REGION" \ - --query Account \ - --output text -)" -export PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID="$AWS_ACCOUNT_ID" -export PORTSCANNER_EXPECTED_AWS_REGION="$AWS_REGION" - -test "$AWS_ACCOUNT_ID" != "123456789012" -aws sts get-caller-identity --region "$AWS_REGION" -terraform version -``` - -The staged helper refuses to plan if the live account differs from -`PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID`, and it forces that account plus -`PORTSCANNER_EXPECTED_AWS_REGION` into the reviewed Terraform plan. Keep the same values -in `evaluation.tfvars`. - -The example deliberately: - -- uses direct EC2 snapshots instead of creating the account's singleton AWS Config - recorder; -- disables CloudTrail and local API-call hints for the one-shot test; -- limits scanning to one authorized account and one public `/32`; -- enables the EKS public endpoint only for the runner's restricted IPv4 CIDR while - retaining private endpoint access; and -- permits bucket and ECR cleanup because it is intended only for disposable evaluation - data and images. - -`0.0.0.0/0` is rejected for EKS API access. For production, disable the public endpoint -and supply approved private runner or VPN security groups instead. - -Production can set `cloudtrail_mode = "create"` for a project-owned multi-Region -management trail or `"existing"` with an approved `existing_cloudtrail_arn`. Do not -enable API-call hints until that account-level choice has been reviewed. - -## 3. Bootstrap remote state - -Create the state bucket and lock table once. Keep the bootstrap's local state secure: - -```bash -terraform -chdir=terraform/aws/state-bootstrap init -terraform -chdir=terraform/aws/state-bootstrap apply \ - -var="aws_region=$AWS_REGION" \ - -var="expected_deployment_account_id=$AWS_ACCOUNT_ID" \ - -var='name_prefix=portscanner-eval' -``` - -Write the non-secret backend settings outside the repository and add a unique state key: - -```bash -export TF_BACKEND_CONFIG="$HOME/portscanner-eval-backend.hcl" - -terraform -chdir=terraform/aws/state-bootstrap \ - output -json backend_configuration | - jq -r 'to_entries[] | "\(.key) = \(.value | tojson)"' \ - >"$TF_BACKEND_CONFIG" -printf '%s\n' 'key = "environments/portscanner-eval.tfstate"' \ - >>"$TF_BACKEND_CONFIG" -chmod 600 "$TF_BACKEND_CONFIG" -``` - -Do not put credentials in the backend file or reuse its key for another root. - -## 4. Apply the paused foundation - -The deployment helper saves a plan, forces all runtime/dispatch switches off, and asks -you to type the stage name before applying: - -```bash -terraform/aws/scripts/deploy.sh \ - "$TF_ROOT" foundation "$TF_VARS" -``` - -Read the plan. Reject unexpected changes to shared Config, CloudTrail, networking, -databases, clusters, or policies. AWS charges begin at this stage, but no scanner work -is dispatched. - -Confirm the gate: - -```bash -terraform -chdir="$TF_ROOT" output -json deployment_state -``` - -Every value must be `false`. - -## 5. Publish immutable images - -First validate the image map without logging in, building, or pushing: - -```bash -terraform/aws/scripts/build-images.sh --dry-run "$TF_ROOT" arm64 -``` - -Then build the reviewed clean commit, scan every image, push immutable tags, and retain -the resulting digests in a private file: - -```bash -export IMAGE_VARS="$HOME/portscanner-eval-images.tfvars" -export SOURCE_REVISION="$(git rev-parse HEAD)" - -(umask 077 - terraform/aws/scripts/build-images.sh \ - "$TF_ROOT" arm64 "$SOURCE_REVISION" >"$IMAGE_VARS") -``` - -The default Terraform architecture is ARM64. For x86, set the foundation to -`lambda_architecture = "x86_64"`, `node_ami_type = "AL2023_x86_64_STANDARD"`, and an -x86-compatible `node_instance_types` value such as `["t3.medium"]`, then invoke the -builder with `x86_64`. The helper rejects a mismatch with the applied foundation and -emits those matching runtime values with the image digests. - -## 6. Install the paused runtime - -The helper accepts both the environment values and the generated image values: - -```bash -terraform/aws/scripts/deploy.sh \ - "$TF_ROOT" runtime "$TF_VARS" "$IMAGE_VARS" - -terraform/aws/scripts/deploy.sh \ - "$TF_ROOT" migrate "$TF_VARS" "$IMAGE_VARS" -``` - -The runtime stage creates digest-pinned functions with schedules and queue mappings -paused. The migrate stage applies checksum-locked migrations, creates the application -database credential, and installs the operator. Dispatch remains off. - -Verify that the runner can reach the EKS endpoint, the node is ready, the operator is -healthy, and every deployed image uses the recorded digest. If the runner's public -address changes, update `eks_public_access_cidrs` before retrying. - -## 7. Verify exactly one canary - -Before activation: - -1. Confirm `allowed_target_cidrs` still contains only the authorized canary `/32`. -2. Confirm the canary's EC2 network interface is in the authorized account and Region, - is in use, and still owns that public address. -3. Read the scanner egress address: - - ```bash - terraform -chdir="$TF_ROOT" output -json scanner_egress_public_ips - ``` - -4. Permit the agreed canary service port from that egress address in the canary security - group, and run a disposable service on that port. -5. Reconfirm the test window and owner. - -The first-release `new_target` path runs a bounded full TCP scan over ports 1-65535, even -when only one canary service is expected to be open. The authorization and maintenance -window must cover that exact behavior. The evaluation file limits the operator to one -concurrent reconciliation, hard-caps active scanner Pods at one and scanner Job objects -at four, and passes 100-250 probes/second to the scanner. - -Enable only the priority/result queue and stream pipeline. The durable outbox-replay -repair schedule is enabled; automatic snapshot/rescan/processor schedules, EventBridge -inventory hints, signal consumption, and coverage consumption remain disabled: - -```bash -terraform/aws/scripts/deploy.sh \ - "$TF_ROOT" canary "$TF_VARS" "$IMAGE_VARS" -``` - -Invoke one exact public IPv4 in one account and Region. This still reads the Region's EC2 -inventory, but only the matching address is reconciled and no absence removals run: - -```bash -export CANARY_IP="REPLACE_WITH_AUTHORIZED_PUBLIC_IPV4" -export SNAPSHOT_FUNCTION_ARN="$( - terraform -chdir="$TF_ROOT" output -json function_arns | - jq -r '.snapshot' -)" - -aws lambda invoke \ - --region "$AWS_REGION" \ - --function-name "$SNAPSHOT_FUNCTION_ARN" \ - --cli-binary-format raw-in-base64-out \ - --payload "$( - jq -nc \ - --arg account_id "$AWS_ACCOUNT_ID" \ - --arg region "$AWS_REGION" \ - --arg target "$CANARY_IP" \ - '{ - account_id: $account_id, - region: $region, - target_public_ipv4: $target - }' - )" \ - "${TMPDIR:-/tmp}/portscanner-canary-response.json" -jq . "${TMPDIR:-/tmp}/portscanner-canary-response.json" -``` - -The generator rereads current ownership immediately before dispatch, the scanner verifies -the public edge, and an observed open service produces an immutable finding handoff. Do -not omit `account_id`, `region`, or `target_public_ipv4`: canary mode rejects an -out-of-scope request and requires the address to resolve to exactly one inventory target -in a complete response before writing state. - -Inspect the external finding queue and object by following -[Integrating findings](integrating-findings.md). Validate the object against -`schemas/finding.schema.json` or the shared semantic validator before accepting it. - -As soon as the canary is proven, pause the mappings while schedules and signal rules -remain disabled, retaining the runtime and evidence: - -```bash -terraform/aws/scripts/deploy.sh \ - "$TF_ROOT" pause-canary "$TF_VARS" "$IMAGE_VARS" -``` - -The pause helper applies only saved plans containing dispatch mapping/rule updates. It -rejects migration, image, Helm, and other infrastructure changes. - -If the cluster API cannot be reached, disable the AWS producers and mappings without a -Terraform plan or Helm refresh: - -```bash -terraform/aws/scripts/emergency-pause.sh "$TF_ROOT" -``` - -This does not terminate an already running scanner Job; follow the authorized -Kubernetes or AWS network/node-group stop procedure for that separate step. - -Do not broaden the account, Region, or CIDR scope until queue age, ownership verdicts, -scan outcomes, finding delivery, cost, and cleanup have all been reviewed. - -## 8. Integrate, expand, or remove - -Keep Portscanner's immutable finding bucket and external SQS queue as the boundary to -your ticketing, SIEM, paging, or data platform. The integration should be idempotent by -`event.event_key`, tolerate duplicate S3 notifications, and delete an SQS message only -after the referenced object has been fetched and validated. - -For production, create a separate reviewed variable set: disable public EKS API access, -use private runner/VPN security groups, disable forced bucket/repository deletion, -enable database protection and final snapshots, enable DynamoDB PITR, add resilient -NAT/nodes/database capacity, and set policy-appropriate retention and alarms. - -For a disposable evaluation, pause first and retain or export any required evidence. -Then create and review a destroy plan while keeping the migrated configuration visible -to Terraform: - -```bash -export DESTROY_PLAN="$HOME/portscanner-eval-destroy.tfplan" - -terraform -chdir="$TF_ROOT" plan -destroy \ - -var-file="$TF_VARS" \ - -var-file="$IMAGE_VARS" \ - -var=deploy_runtime=true \ - -var=run_migration=true \ - -var=install_operator=true \ - -var=enable_event_dispatch=false \ - -var=enable_automatic_inventory=false \ - -var=canary_mode=false \ - -out="$DESTROY_PLAN" - -terraform -chdir="$TF_ROOT" show "$DESTROY_PLAN" -terraform -chdir="$TF_ROOT" apply "$DESTROY_PLAN" -rm -f "$DESTROY_PLAN" -``` - -Confirm the application resources are gone. The state bootstrap intentionally has -`prevent_destroy` and requires a separate retirement decision. Production removal must -instead follow the retention and destruction procedure in -[AWS deployment](aws-deployment.md). diff --git a/docs/integrating-findings.md b/docs/integrating-findings.md deleted file mode 100644 index 6096c5f..0000000 --- a/docs/integrating-findings.md +++ /dev/null @@ -1,137 +0,0 @@ - - -# Integrating findings - -Portscanner publishes immutable finding JSON to S3 and sends standard S3 object-created -notifications to an external SQS queue. No in-stack consumer reads that queue. This is -the supported boundary for ticketing, SIEM, paging, data-lake, or custom workflow -integration. - -The SQS message is a pointer, not the finding itself: - -```text -SQS message - -> S3 event notification - -> findings/... JSON object - -> Finding contract -``` - -## Discover the handoff - -Central Terraform examples expose the values a consumer needs: - -```bash -terraform -chdir="$TF_ROOT" output -raw external_finding_queue_url -terraform -chdir="$TF_ROOT" output -raw external_finding_queue_arn -terraform -chdir="$TF_ROOT" output -raw external_finding_bucket_name -terraform -chdir="$TF_ROOT" output -raw external_finding_bucket_arn -``` - -Manage the consumer identity outside this stack. Its identity policy normally needs: - -- `sqs:ReceiveMessage`, `sqs:DeleteMessage`, `sqs:ChangeMessageVisibility`, and - `sqs:GetQueueAttributes` on the exact finding queue ARN; and -- `s3:GetObject` and `s3:GetObjectVersion` on - `/findings/*`. - -Do not grant list, write, or delete access unless the downstream design separately -requires and reviews it. - -## Consume safely - -For each message: - -1. Long-poll SQS and keep the receipt handle private. -2. Parse the message body as an AWS S3 event notification. Ignore the one-time - `s3:TestEvent` after verifying its source. -3. Require `eventSource == "aws:s3"`, the exact configured bucket, an - `ObjectCreated` event, and a key beneath `findings/`. -4. URL-decode the object key with form semantics (`unquote_plus` in Python). -5. Fetch the exact object version when the notification includes a version ID. -6. Bound the accepted object size, hash the bytes, and compare the digest with the - object's `payload-sha256` metadata. -7. Parse JSON and validate it with the shared semantic validator: - - ```python - import json - - from portscanner_contracts import validate_finding - - finding = validate_finding(json.loads(object_bytes)) - ``` - - `schemas/finding.schema.json` is the language-neutral structural contract. Schema - validation alone does not replace the semantic checks implemented by - `validate_finding`. -8. Commit the downstream side effect idempotently. -9. Delete the SQS message only after every referenced finding object in that message - has been accepted or safely recorded. - -On a transient failure, leave the message for retry and extend visibility when -processing may exceed the queue timeout. On malformed, unauthorized, or -contract-incompatible data, preserve diagnostics without logging sensitive payloads, -then allow the configured redrive policy to isolate the message. - -S3 notifications and standard SQS delivery are at least once and are not globally -ordered. A notification can be duplicated, delayed, or delivered with other records. -Never use queue receive order as finding state order. - -## Event and snapshot documents - -`kind == "finding_event"` represents a lifecycle transition: - -- `opened`; -- `reopened`; -- `updated`; or -- `resolved`. - -Use `event.event_key` as the immutable idempotency key. Apply transitions according to -`event.occurred_at` and the finding's monotonic `version`, not queue order. - -`kind == "current_finding"` is a periodic repair snapshot. It deliberately has no -`event` member. Upsert it by `finding.fingerprint` and `finding.version`; repeated -snapshots of the same state are expected and should not create duplicate tickets. - -Both kinds use `schema_version == "1.0"`. Reject unknown major versions until the -consumer has explicitly added compatibility. A representative transition is available -at `examples/finding.json`. - -## Minimal canary inspection - -During an evaluation, inspect without deleting: - -```bash -QUEUE_URL="$( - terraform -chdir="$TF_ROOT" output -raw external_finding_queue_url -)" - -aws sqs receive-message \ - --queue-url "$QUEUE_URL" \ - --wait-time-seconds 20 \ - --max-number-of-messages 1 \ - --attribute-names All \ - --message-attribute-names All -``` - -The result should contain an S3 notification whose bucket matches -`external_finding_bucket_name` and whose key starts with `findings/events/` or -`findings/current/`. Fetch and validate that object before calling the canary complete. -The command above intentionally leaves the message in the queue. - -## Production handoff checklist - -- Pin the consumer to a reviewed contract version and test the repository examples. -- Restrict IAM to the exact queue and finding prefix. -- Validate bucket, key, object version, metadata hash, JSON, and semantic contract. -- Make ticket/SIEM writes idempotent before deleting the message. -- Handle both transition events and current-state repair snapshots. -- Monitor queue age, receive count, DLQ depth, object-fetch failures, and contract - rejection counts. -- Redrive only after the consumer defect or compatibility gap is fixed. -- Keep raw finding payloads out of broadly accessible logs. - -To add a new cloud inventory source rather than a finding consumer, follow -[Adding inventory or signal sources](adding-sources.md). diff --git a/docs/multi-account.md b/docs/multi-account.md deleted file mode 100644 index 4a151b1..0000000 --- a/docs/multi-account.md +++ /dev/null @@ -1,158 +0,0 @@ - - -# Multi-account deployment - -Single-account operation is the default. Multi-account operation is optional and should -be adopted only when centralized ownership, authorization, incident response, and -network egress are already defined. - -## Patterns - -### Independent accounts - -Each account runs its own inventory, scheduling, scanner, evidence, and state. This has -the smallest cross-account trust boundary and clearest cost attribution, but duplicates -infrastructure and policy. - -### Hub with assumed read roles - -The hub runs inventory, scheduling, scanning, evidence, and state. Each spoke exposes a -read role for: - -- supported inventory list/get operations; -- the specific current-state get required for ownership revalidation; and -- optional read-only access to an existing inventory aggregator. - -The role does not grant network mutation, security-group changes, pass-role, -credential creation, or scan dispatch. The hub stores a role reference in protected -deployment configuration, not in the public repository. - -### Hub with event forwarding - -Spokes filter supported provider events and forward them to a hub event bus or queue. -The hub authenticates the source and extracts a bounded `SignalHint`/resource locator. -This can reduce signal delay, but it does not replace: - -- complete periodic snapshots; -- a current provider-state reread; -- dispatch-time ownership revalidation; or -- account and CIDR authorization gates. - -A forwarded event is a hint. Its mutable fields never become authoritative Target -metadata or scanner arguments. - -The AWS member module forwards only its provider Region. Its EC2 API-call pattern emits -nothing without a member or organization management CloudTrail, so active forwarding -requires `cloudtrail_mode = "create"` or an explicit `existing_cloudtrail_arn`. A -multi-Region trail supplies CloudTrail history, but EventBridge default buses and rules -are regional. Deploy a uniquely named member forwarding root in every Region requiring -the hot path, all targeting the central regional bus; use snapshots elsewhere. - -### Hybrid - -Spokes forward signals while the hub assumes a read role for snapshots and rereads. -This is the preferred centralized pattern when change acceleration is required. - -## Trust design - -For each spoke: - -- trust only the dedicated hub source identity; -- constrain OIDC subject, principal, external condition, partition, and Region where - supported; -- grant only named inventory list/get actions; -- restrict resources when the provider API supports effective resource scoping; -- require short sessions and log every assume-role operation; -- prevent role chaining into unrelated identities; and -- keep scanner workers unable to assume spoke roles. - -Separate the source-reader identity from the Terraform deployment identity. Event-bus -resource policy should accept only approved spokes and only the expected event shape. -Use a per-spoke source identifier that is not a live account number in logs and metrics. - -## Account registry - -Maintain the live registry in protected operator configuration. Each entry should -contain: - -- opaque source ID; -- account reference supplied at deployment time; -- enabled Regions and resource classes; -- snapshot adapter and optional signal route; -- read-role reference; -- authorized CIDRs and exclusions; -- profile and rate ceilings; -- authorization/change-record reference; -- metadata allowlist version; and -- enabled, paused, or decommissioning state. - -Repository examples use `${SPOKE_ACCOUNT_ID}` and `${SPOKE_READ_ROLE}` placeholders. -Never commit account numbers, role ARNs, organization IDs, or external-condition values. -The Terraform examples expose these values as variables with synthetic defaults. Put -live account maps, names, Regions, ARNs, and external IDs only in ignored private -variable files. - -## Source isolation - -- Namespace snapshot cursors, deterministic IDs, queues, leases, and object prefixes by - opaque source ID. -- Include source ID in every Target and result correlation key. -- Prevent one source's snapshot from removing another source's Targets. -- Before mutually untrusted or large-scale multi-source rollout, add per-source API, - queue, dispatch, and packet budgets; the first release provides a deployment-wide Pod - quota and destination serialization, not an independent per-source packet bucket. -- Quarantine malformed or unauthorized spoke events without blocking other sources. -- Ensure database authorization and queries cannot cross source boundaries accidentally. - -Stable Target identity should include provider, source boundary, Region or location, -provider resource identity, and address binding as defined by the adapter contract. -Display names and mutable tags are not identity. - -## Onboarding a spoke - -1. Record explicit scanning authorization, provider-policy review, accounts, Regions, - CIDRs, profiles, rates, and owner. -2. Choose snapshot-only, assume-role, event-forwarding, or hybrid mode. -3. Review existing Config and CloudTrail resources. For Config aggregation, create the - source authorization in each exact member Region for the central account and - aggregator Region before adding that source to an aggregator configured for that - Region. The application create mode includes only its own provider Region; use direct - snapshots or an externally provisioned existing aggregator for others. For event - forwarding, choose member `cloudtrail_mode = "create"` or reference and live-verify - a member/organization multi-Region management trail. -4. Deploy the spoke read role and one optional filtered event target per signal Region, - with unique state/name and no dispatch grant. -5. Add the protected hub registry entry with dispatch disabled. -6. Run a complete snapshot and validate pagination, completeness, metadata redaction, - identity, and generation. -7. Exercise `ACTIVE`, `STALE`, `MOVED`, `INACTIVE`, and `UNKNOWN` ownership paths. -8. Enable one narrow authorized canary at the lowest rate. -9. Confirm evidence isolation, idempotent replay, cost attribution, and cleanup. -10. Enable periodic reconciliation, then optional signals. - -## Failure behavior - -- A spoke API failure pauses only that source and yields UNKNOWN ownership. -- An incomplete snapshot cannot remove Targets. -- An event-forwarding outage does not stop periodic snapshots. -- Hub queue pressure preserves reserved baseline capacity and per-source fairness. -- Expired work is discarded before role assumption or Job creation. -- If a spoke is removed from authorization, dispatch stops before queued work is - reconciled. - -No cross-source fallback is allowed: the hub must not scan a Target under another -spoke's authorization because an API or role is unavailable. - -## Offboarding - -1. Disable dispatch and signals for the source. -2. Expire or remove queued work and verify no active Job remains. -3. Take a final complete snapshot if policy requires it, then mark Targets inactive. -4. Remove event-bus permissions and the spoke read-role trust. -5. Remove protected registry entries and source-specific secrets. -6. Apply evidence retention/deletion policy to queues, objects, state, logs, and backups. -7. Confirm shared Config, CloudTrail, VPC, and account services are unchanged. -8. Revoke the authorization record and review residual cost. diff --git a/docs/operations.md b/docs/operations.md deleted file mode 100644 index b795311..0000000 --- a/docs/operations.md +++ /dev/null @@ -1,240 +0,0 @@ - - -# Operations - -Operate Portscanner as a safety-sensitive distributed system. Success is not merely -“Jobs completed”; it is complete inventory, current ownership, declared scan coverage, -durable evidence, and correctly preserved UNKNOWN state. - -## Health signals - -Monitor by source, account alias, Region, profile, and outcome without logging raw -account numbers or addresses. - -### Discovery - -- last complete snapshot time and duration; -- pages, Targets, upserts, unchanged Targets, and removals per snapshot; -- incomplete/aborted snapshot count; -- source freshness and throttling; -- duplicate and out-of-order signals; -- signal-to-current-state-reread latency; and -- known Targets not present in a recently accepted complete snapshot. - -### Priority and ownership - -- queue depth and oldest age by priority; -- baseline capacity actually served; -- expired work and lease contention; -- idempotency conflicts; -- ownership verdicts (`ACTIVE`, `STALE`, `MOVED`, `INACTIVE`, `UNKNOWN`); -- time between ownership reread and dispatch; and -- work suppressed by account, CIDR, profile, or rate gates. - -### Verification and evidence - -- Jobs pending, running, succeeded, failed, timed out, and evicted; -- packet/concurrency budget utilization; -- result envelopes missing, duplicated, malformed, or quarantined; -- declared coverage versus observed output; -- evidence upload and parse lag; and -- Exposure transitions, UNKNOWN age, and Findings by lifecycle state. - -Do not publish a p95 objective until the deployment has a defined clock, scope, -exclusions, sufficient samples, and an alert policy. Provider freshness, queue time, -scan time, and parse time should be reported separately. - -## Provisioned CloudWatch alarms - -The AWS application creates alarms even when no notification actions are configured. -Storage always creates: - -- `ApproximateAgeOfOldestMessage` for each primary SQS queue. The maximum age must - reach `queue_age_alarm_threshold_seconds` (15 minutes by default) in two consecutive - five-minute periods. -- `ApproximateNumberOfMessagesVisible` for each dead-letter queue. Any visible message - in a one-minute period alarms. - -When `deploy_runtime` is true, every Lambda function also gets an `Errors` alarm and a -`Throttles` alarm for one or more events in a five-minute period. The dedicated -inventory outbox function gets an `IteratorAge` alarm, using the stable Lambda -`FunctionName` dimension for its DynamoDB Streams mapping. Its maximum iterator age -must reach `iterator_age_alarm_threshold_seconds` (15 minutes by default) in two -consecutive five-minute periods. AWS reports that metric in milliseconds; Terraform's -input remains seconds. - -Committed outbox rows remain pending without TTL until S3 publication and both SQS sends -succeed. Delivery changes the sparse-index entity and starts a seven-day TTL. A -five-minute repair schedule queries the pending index whenever dispatch is enabled, so -records older than DynamoDB Streams' 24-hour retention are still delivered. Treat -repeated outbox Lambda errors as a durable poison-row or downstream outage, not as -permission to delete the row. - -All of these alarms treat missing data as not breaching. This avoids false alarms for -idle queues, disabled event sources, and functions with no invocations, but operators -must separately detect missing expected schedules or traffic. Use the application -`alarm_names` and `alarm_arns` outputs to inventory the resulting alarms. - -### Attach notifications externally - -The stack deliberately creates no SNS topic, email subscription, or organization- -specific paging integration. Pass existing action ARNs through `alarm_action_arns` and, -optionally, `ok_action_arns`. For example, an operator can supply the ARN of an -externally managed SNS topic whose subscriptions route to the team's paging system. -Empty lists leave every alarm active and visible without sending notifications. - -Grant and test any required publish/invoke permissions in the external integration. -Route recovery notifications only when they are actionable; an `OK` transition does -not prove that queued work was processed successfully. - -Finding delivery is a separate S3/SQS handoff. Follow -[Integrating findings](integrating-findings.md) for its IAM, validation, idempotency, and -message-deletion requirements. - -### Interpret and extend coverage - -- Primary queue age means a producer/consumer imbalance, paused event source, failed - downstream service, or deliberately unconsumed external handoff needs investigation. -- A dead-letter message means retries were exhausted or EventBridge delivery failed; - inspect and preserve the message before redriving it. Deterministic generator, - result-parser, and target-projector contract rejections deliberately fail their SQS - item until this redrive policy quarantines it. -- Lambda errors indicate failed invocations. Throttles indicate reserved concurrency or - account concurrency prevented work from starting. -- Outbox iterator age means DynamoDB stream records are not being drained promptly, - even if invocation errors are absent. - -Lambda metrics do not observe scanner Kubernetes Job failures after generator work is -accepted. The SQS age alarms cover queue backlog before consumption, but do not prove -that a scanner Job completed or that evidence arrived. Add environment-specific -Kubernetes/Container Insights dashboards, Prometheus alerts, or log metric filters for -Jobs failed, timed out, or stuck, and correlate them with queue age and missing result -envelopes. - -## Normal operating checks - -Daily: - -1. Confirm each enabled source has a recent complete snapshot. -2. Confirm known-door reconciliation is advancing while priority work is present. -3. Review dead-letter queues, quarantine, UNKNOWN age, and ownership-gate failures. -4. Compare dispatched work with immutable result envelopes and parsed records. -5. Check rate, cost, and Kubernetes capacity guardrails. - -After every deployment: - -1. Record deployed image digests and migration version. -2. Verify feature gates and authorization policy digest. -3. Run a non-network contract check, then one approved canary. -4. Confirm duplicate canary delivery is idempotent. -5. Confirm a deliberately expired item and stale generation do not dispatch. -6. Verify cleanup and retention behavior. - -Tagged ECR releases have no Terraform count-expiration policy. Treat retention of active -and rollback digests as an operator-owned release invariant. If optional untagged-image -expiration is enabled, confirm no deployed digest has lost its release tag before -retiring images. - -## Runbooks - -### Snapshot is late or incomplete - -1. Disable removal application for the affected source; never infer deletion from a - partial page walk. -2. Check provider throttling, pagination cursor handling, role access, Region selection, - and source-data freshness. -3. Retry the full snapshot with bounded backoff. -4. Accept removals only after one complete, validated snapshot commits atomically. -5. Reconcile queued work against the newly accepted generation. - -### Signal backlog grows - -Signals only accelerate work. Preserve snapshot and periodic queues, reduce signal -intake if necessary, and coalesce by stable Target identity. For each surviving hint, -reread current state before scheduling. Do not replay stale signal payloads as inventory. -If a Region produces no API-call hints, verify that Region has its own enabled -EventBridge forwarding rule and a live member/organization management CloudTrail; a -multi-Region trail does not make another Region's EventBridge rule global. - -### Ownership UNKNOWN increases - -Pause dispatch for the affected source. Check provider API availability, credentials, -throttling, eventual-consistency windows, and adapter parsing. Retry within the work -deadline. If current ownership cannot be proved, leave work suppressed and Exposure -state unchanged. - -### Scanner failures increase - -Pause the affected profile, not the inventory path. Check image digest, Kubernetes -events, resource limits, network egress, DNS, destination connection limits, and timeout -classes. A failed or incomplete scan remains UNKNOWN and must not resolve Findings. - -### Parser or database is unavailable - -Stop result deletion and preserve the object/queue source of truth. Increase visibility -timeout if needed, avoid parallel replay beyond database capacity, and restore parsing -with the same idempotency keys. Confirm that replay creates no duplicate Exposure or -Finding transitions. - -Keep every Lambda-consumed SQS queue visibility timeout at least six times the Lambda -timeout; the Terraform default is 360 seconds for a 60-second function. - -### Authorization boundary changes - -Disable dispatch first. Update the external authorization record, account/CIDR gates, -and provider policy together. Reconcile queued work and discard anything outside the -new boundary. Re-enable only after an ownership-gated canary. - -### Cluster-independent dispatch brake - -If an ordinary `pause` cannot refresh Helm because EKS is unreachable, run -`terraform/aws/scripts/emergency-pause.sh `. It validates STS and the -state's account/Region, then disables exact AWS mappings and rules without a Terraform -plan or Kubernetes connection. It does not stop active Jobs; follow the separately -approved cluster or network/node-group stop procedure and reconcile with a normal pause -plan after access is restored. - -## Backup and recovery - -- Version object evidence and apply lifecycle retention deliberately. -- Back up relational state before migrations and test point-in-time recovery. -- Keep Terraform state encrypted, locked, access-logged, and outside the repository. -- Treat queue retention and dead-letter retention as part of the recovery-point - objective. -- Preserve pending outbox rows; they intentionally have no TTL before successful - delivery and are the repair source when stream records expire. -- Restore into an isolated environment with dispatch disabled. -- Reconcile provider snapshots after restore; do not blindly replay old scan work. - -## Upgrades - -Follow the staged order in [aws-deployment.md](aws-deployment.md): foundation, tested -image digests, disabled runtime wiring, expand-only migration/operator installation, -bounded canary, then wider activation. Keep old readers and writers schema-compatible -during rollback. - -The image helper accepts only central roots exposing `repository_urls`, -`deployment_state`, and `workload_architecture`, rejects a requested build-architecture -mismatch, and supports valid foundation, paused-runtime, migrated, -operator-installed, active-canary, paused-canary, dispatch-only, and fully active states -so it can publish upgrade images. It explicitly rejects the member root. The -migrate/install runner needs AWS CLI exec authentication, an explicit EKS installer -access entry, and network reachability through either the private EKS endpoint or the -explicitly restricted public endpoint configuration. - -Generated manifests must be regenerated by the documented command and checked for drift -in CI. Never hand-edit a generated artifact without updating its source. - -## Decommission and cleanup - -1. Disable signals, scheduling, and dispatch. -2. Drain or explicitly discard queues and record the decision. -3. Export required audit/evidence data, then apply retention and deletion policy. -4. Destroy project-owned infrastructure from its Terraform state. -5. Remove hub/spoke trust, event-bus permissions, OIDC subjects, registry images, DNS, - and Kubernetes service accounts. -6. Verify shared Config, CloudTrail, VPC, cluster, and database resources remain intact. -7. Revoke the scanning authorization and document completion. diff --git a/docs/scanning-safety.md b/docs/scanning-safety.md deleted file mode 100644 index 7bc6ad0..0000000 --- a/docs/scanning-safety.md +++ /dev/null @@ -1,171 +0,0 @@ - - -# Scanning safety - -Network scanning can disrupt services, trigger abuse controls, violate provider policy, -or reach an address that changed owners. This project does not grant permission to scan. - -## Non-negotiable rules - -1. Scan only assets you own or for which you have explicit written authorization. -2. Follow the cloud provider's acceptable-use, penetration-testing, and notification - policies. -3. Never scan third parties, the general Internet, documentation ranges, vendor - endpoints, shared services, or an address merely because it appeared in an old - event. -4. Gate dispatch by an explicit authorized account; an empty account scope means no - access. -5. Revalidate current provider ownership immediately before dispatch. -6. Treat deployment CIDR allowlisting as optional defense-in-depth. Apply CIDR denies - first; an empty allowlist adds no CIDR restriction. -7. Set conservative packet, concurrency, destination, and time limits. -8. Keep an emergency dispatch-off control that does not depend on a healthy queue or - cluster rollout. -9. Clean up temporary trust, events, Jobs, data, and infrastructure after a test. - -The RFC 5737 ranges `192.0.2.0/24`, `198.51.100.0/24`, and `203.0.113.0/24`, and the RFC -3849 prefix `2001:db8::/32`, are documentation examples only. They are non-routable and -must not be used as canary destinations. - -## Authorization record - -Before enabling dispatch, record: - -- approving owner and change record reference; -- exact accounts, Regions, resource types, and any optional CIDR constraints; -- excluded resources and shared-service boundaries; -- approved source vantage and egress identity; -- transports, ports, scanner features, and maximum Target size; -- global and per-destination packet/concurrency rates; -- start/end time and recurring cadence; -- monitoring and escalation path; -- provider-policy review; and -- stop and cleanup procedure. - -Do not put personal names, email addresses, live account numbers, role identifiers, or -real CIDRs in this public repository. Keep the authorization record in the operator's -approved private system. - -## Layered scope enforcement - -Scope checks must occur when a Target enters inventory, when work is created, and again -at dispatch. The effective set is an intersection with explicit exclusions, never a -union: - -```text -supported source -− deployment CIDR denylist -∩ authorized account -∩ authorized resource type -∩ current provider ownership -∩ optional deployment CIDR allowlist (identity when empty) -∩ approved scan profile -∩ available scanner-Pod quota and destination serialization -``` - -The scanner evaluates denies before the optional allowlist. A CIDR allowlist can provide -a useful second boundary for fixed Elastic IP ranges, but dynamic public addresses make -it impractical in many deployments. Empty CIDR lists do not weaken the mandatory -account and current-ownership gates. Inclusion in a CIDR never authorizes scanning a -third party. - -Reject hostnames, address families, ranges, profiles, or free-form arguments that policy -does not explicitly support. Do not resolve a hostname at the worker and scan whatever -it returns. - -## Ownership and recycled addresses - -Public addresses can be released and reassigned quickly. A discovery-time match is not -enough. - -Immediately before dispatch, reread the resource by stable provider identity and verify: - -- the resource still exists in the expected account and Region; -- the address is still attached to that resource; -- the observed generation is the queued generation; -- the resource remains in an allowed class and, when configured, CIDR; and -- the verdict is unambiguous and fresh. - -Only `ACTIVE` may dispatch. `STALE`, `MOVED`, `INACTIVE`, and `UNKNOWN` suppress the -scan. Ambiguous current state maps to `UNKNOWN`; a retry must repeat the reread. - -## Rate and disruption controls - -- Start with conservative `scanner_min_rate`, `scanner_max_rate`, and - `scanner_max_concurrent_pods` values; bound queued/retained Job objects with - `scanner_max_jobs`. -- Treat `scanner_max_concurrent_pods × scanner_max_rate` as the deployment's configured - aggregate upper bound. Required destination-hash anti-affinity serializes active Pods - for one public IPv4 destination. -- Limit Targets and ports per Job. -- Set connect, host, process, and Job deadlines. -- Reserve capacity for periodic reconciliation but cap priority bursts. -- Schedule around sensitive windows and honor destination-owner requests immediately. -- Watch packet loss, destination health, firewall/IDS events, provider abuse notices, - queue age, and scanner retries during expansion. -- Do not assume a cloud instance type's nominal bandwidth is a safe scan rate. - -Service detection, scripts, operating-system detection, UDP, and full-port profiles can -be materially more disruptive than a narrow TCP connect check. Enable each as a -separate reviewed profile. - -## Outside-vantage requirement - -The worker should test the public path that an untrusted external client would use. It -must not quietly fall back to a private route, cluster-local service, provider metadata -endpoint, or internal DNS answer. Record and monitor the worker's egress identity. - -Outside vantage does not relax ownership requirements. Scanning from another provider, -Region, or account can introduce additional acceptable-use and data-transfer rules. - -## Canary procedure - -1. Keep dispatch globally disabled. -2. Configure one authorized account, one resource, one low-impact profile, and the - minimum rate. Add a narrow CIDR allowlist when the resource uses a fixed authorized - range. -3. Verify the current-state ownership response manually through an approved private - process. -4. Enable one work item with a short deadline. -5. Observe dispatch, packet rate, destination health, evidence, parsing, and cleanup. -6. Replay the same work and confirm idempotency. -7. Change or expire the generation and confirm dispatch is suppressed. -8. Disable dispatch and verify that no Job remains. - -Never use an unrelated public host as a “known open” test. - -## Emergency stop - -The operator must be able to: - -1. disable new dispatch; -2. suspend signal and recurring schedules; -3. delete or stop active Jobs; -4. block scanner egress at the network boundary; -5. retain evidence and audit records; and -6. notify the authorization owner and provider if required. - -Practice this procedure before broad activation. - -`terraform/aws/scripts/emergency-pause.sh ` performs steps 1 and 2 -directly through the AWS APIs after validating the expected account, Region, and exact -controls from Terraform state. It does not plan or refresh Helm and therefore does not -need a reachable Kubernetes API. It intentionally does not terminate Jobs already -running: use an authorized Kubernetes path for step 3, or an independently reviewed AWS -network/node-group control for steps 3 and 4 when the cluster API is unavailable. - -## Cleanup - -After a test or decommission: - -- stop schedules and drain or discard queued work; -- remove temporary account/CIDR grants; -- remove spoke trust and event forwarding; -- delete test Jobs, namespaces, images, object prefixes, and databases according to - retention policy; -- destroy project-created infrastructure from the correct Terraform state; -- verify shared Config, CloudTrail, VPC, and Kubernetes resources were preserved; and -- review billing and provider notices for delayed effects. diff --git a/docs/security-model.md b/docs/security-model.md deleted file mode 100644 index 079d228..0000000 --- a/docs/security-model.md +++ /dev/null @@ -1,152 +0,0 @@ - - -# Security model - -Portscanner can emit network traffic, read cloud inventory, and retain security -evidence. Compromise can therefore cause unauthorized scanning, data disclosure, or -incorrect Findings. Deploy it with a smaller trust boundary than the assets it observes. - -## Protected assets - -- scanning authorization and account/CIDR policy; -- provider read roles and OIDC trust; -- Target ownership and generation state; -- queues, leases, and idempotency records; -- scanner profiles and rate limits; -- raw evidence, normalized Exposures, and Findings; -- database credentials and encryption keys; -- container images, manifests, migrations, and Terraform state; and -- audit records that correlate source, dispatch, evidence, and action. - -## Trust boundaries - -1. **Provider boundary:** source adapters read provider APIs. Provider payloads are - untrusted input even when authenticated. -2. **Signal boundary:** events can be delayed, duplicated, reordered, forged through a - compromised producer, or incomplete. -3. **Scheduling boundary:** normalized work crosses from cloud data into a system - capable of network access. -4. **Kubernetes boundary:** the controller creates Jobs; scanner pods hold network-egress - authority and evidence-write credentials. -5. **Evidence boundary:** result objects and scanner output are untrusted parser input. -6. **Finding boundary:** normalized state can trigger notifications or downstream - operator action. -7. **CI/deployment boundary:** workflows can publish images and change infrastructure - when explicitly trusted. - -## Security invariants - -- Account, CIDR, resource-type, profile, and rate gates are deny-by-default. -- Only assets with explicit authorization may be scanned. -- Provider policy terms and acceptable-use constraints override project configuration. -- Current ownership is revalidated immediately before every dispatch. -- Scanner workers cannot choose or expand Target scope. -- Source metadata cannot become command-line fragments. -- A signal cannot establish ownership, delete a Target, or assert an Exposure. -- Failed or incomplete work produces UNKNOWN, never a closed result. -- Evidence and state writes are idempotent and generation-scoped. -- Deployment credentials are short-lived; no static cloud key is committed or stored in - an image. - -## Threats and controls - -### Recycled address or stale queue item - -An address may move to another resource or customer after discovery. Stable Target -identity, monotonic generations, short work deadlines, and a current-state ownership -reread suppress stale dispatch. - -### Forged or misleading signal - -A signal is treated as a hint. The adapter uses only its locator and source identity to -request current provider state. Snapshot reconciliation repairs missed signals. - -### Command or argument injection - -Targets and metadata are parsed into typed values. The scheduler selects a named, -reviewed profile. The worker invokes the scanner without a shell and without accepting -free-form provider or user arguments. - -### Scope expansion - -The effective destination set is the intersection of current provider ownership, -account allowlist, CIDR allowlist, supported resource classes, and profile limits. -Normalization rejects hostnames, ranges, wildcards, or address families not enabled by -the release. - -### Excessive traffic - -Every scanner Job receives reviewed minimum/maximum Nmap rates. A PriorityClass-scoped -ResourceQuota hard-caps active scanner Pods, so aggregate configured throughput is at -most `scanner_max_concurrent_pods × scanner_max_rate`; required cross-namespace -anti-affinity allows only one active Pod for an opaque hash of a public destination. A -second object-count quota caps active, pending, and retained scanner Jobs at -`scanner_max_jobs`. The operator cache, RBAC, generator, Jobs, and both quotas are -restricted to one dedicated release namespace so another namespace cannot multiply the -limit. -Deadlines, timeouts, the exact-one evaluation gate, and an AWS-only emergency -dispatch-off switch further bound impact. The first release does not claim an -independent per-source packet token bucket. - -### Malicious scanner output - -Parsers impose file, XML/JSON depth, field length, count, and decompression limits; do -not resolve external entities; validate schema versions; and quarantine malformed -evidence. Database writes use parameters and least-privilege identities. - -### Queue replay or duplicate delivery - -Deterministic event, work, dispatch, and result keys make processing idempotent. Leases -have bounded duration and owner tokens. A result cannot update another Target -generation. - -### Metadata or evidence disclosure - -Adapters emit only allowlisted metadata. Logs redact addresses, account identifiers, -provider resource IDs, tags, and raw evidence. Storage uses encryption, narrow roles, -private network paths where appropriate, retention, and access logging. - -### Compromised scanner pod - -Scanner pods use a dedicated service account with no provider inventory access, -read-only root filesystem where supported, all Linux capabilities dropped, resource -limits, egress policy, and write-only -evidence access scoped to an immutable prefix. Never mount deployment credentials or a -kubeconfig. - -### Supply-chain compromise - -CI runs tests, generated-drift checks, static analysis, secret and publication -sanitizers, IaC validation, image scans, and license checks. Releases use immutable -digests and protected environments. Third-party actions and dependencies should be -pinned and updated through reviewed automation. - -## IAM separation - -Use distinct identities for: - -- snapshot listing; -- one-resource ownership rereads; -- signal publication; -- queue consumption and lease updates; -- Kubernetes Job creation; -- evidence upload; -- parsing and state updates; -- Finding publication; -- migrations; and -- Terraform deployment. - -Do not combine scanner network-egress authority with provider inventory reads. In hub/spoke -mode, each spoke role restricts trusted principal, external conditions where supported, -actions, Regions, and resources. Event forwarding does not grant dispatch authority. - -## Residual risks - -Outside-vantage scanning can still trigger provider or destination defenses, incur -egress cost, and affect fragile services. Provider inventory can be eventually -consistent. Port reachability does not establish vulnerability, and non-observation -within one profile does not establish safety. Operators remain responsible for -authorization, policy, monitoring, and incident response. diff --git a/docs/testing.md b/docs/testing.md deleted file mode 100644 index 8f0ecc0..0000000 --- a/docs/testing.md +++ /dev/null @@ -1,276 +0,0 @@ - - -# Testing - -Tests must prove safety and failure semantics, not only the successful scan path. All -repository fixtures must be synthetic and use only RFC 5737/3849 addresses. - -## Local publication checks - -The sanitizer has no third-party Python dependency: - -```bash -python3 -m unittest discover -s tools/tests -p 'test_*.py' -v -python3 tools/sanitize.py --working-tree -``` - -Run configured repository checks before opening a pull request: - -```bash -pre-commit run --all-files -``` - -The default sanitizer scans Git-tracked files. `--working-tree` additionally includes -untracked, non-ignored files so a new document or fixture is checked before staging. - -## Unit tests - -Install the locked workspace and run the full local Python, contract, schema, -generated-file, and license suite: - -```bash -make sync -make check -``` - -Run the Go operator independently: - -```bash -make test-go -make test-go-envtest -make vet-go -``` - -Run the real migration and parser database paths against an isolated, digest-pinned -PostgreSQL 16 container: - -```bash -make test-postgresql -``` - -The helper binds PostgreSQL only to an ephemeral localhost port and removes the -container on exit. Core CI runs the same target. - -Build both migrator distribution formats and verify a clean wheel installation embeds -the byte-identical canonical SQL payload: - -```bash -make test-packaging -``` - -`make ci` additionally renders Kubernetes and runs every Terraform validation -root. `terraform init -backend=false` avoids backend access, but it may download -providers and is not an offline operation. A missing Terraform executable fails -the target by default. A developer intentionally omitting this check may set -`PORTSCANNER_SKIP_TERRAFORM_VALIDATE=true`; CI must not set that escape hatch. - -`make containers` builds all seven local images from temporary `git archive` -contexts containing committed `HEAD` only. It is kept separate because image -builds and vulnerability scans are substantially slower. The target is -local-only: it does not log in or push. - -Adapter tests must cover pagination, deterministic identity, monotonic generation, -duplicate and reordered input, complete and incomplete snapshots, removal rules, -metadata filtering, signal rereads, ownership verdicts, throttling, malformed payloads, -and idempotent retries. - -Scheduler, worker, and parser tests must cover expired work, stale ownership, denied -scope, rate exhaustion, timeout, partial output, malformed evidence, declared coverage, -UNKNOWN preservation, duplicate results, and generation isolation. - -## Contract and schema tests - -Versioned contracts require: - -- schema validity checks; -- one minimal and one representative synthetic fixture; -- rejection fixtures for unknown versions and malformed required fields; -- backward-compatibility tests for every supported reader/writer overlap; -- canonical serialization tests for deterministic identifiers; and -- boundary tests for field length, item count, address family, and timestamp ordering. - -The public JSON Schema is a structural interoperability contract. Schema validation -alone does not prove global IP-address policy, canonical/non-overlapping port ranges, -cross-field ordering, or deterministic identifiers. Every producer must also run the -shared Python semantic validator before publishing: `parse_target_event`, -`validate_scan_result`, or `validate_finding`, as appropriate. Consumers must fail -closed through the same model validation rather than treating JSON Schema success as -semantic acceptance. - -Examples should use addresses such as `192.0.2.10`, `198.51.100.20`, `203.0.113.30`, or -`2001:db8::10` and non-live identifiers such as `resource-synthetic-a`. - -## Runtime dependency lock - -All six Python runtime images (five Lambda functions plus the Nmap worker) install -third-party dependencies from checked-in, hash-verified exports generated from -`uv.lock`. Regenerate and check them with: - -```bash -uv run --frozen python tools/export_runtime_requirements.py -uv run --frozen python tools/export_runtime_requirements.py --check -``` - -Image builds install those exports with pip hash checking and force replacement before -installing first-party wheels with dependency resolution disabled. The Debian Python -base uses the locked AWS Lambda Runtime Interface Client and boto3 versions rather than -an image-bundled SDK. Container CI also verifies the frozen requirements, repository -legal files under `/licenses`, and MIT license metadata in each Lambda component wheel. -The shared `requirements-build.txt` pins and hashes Hatchling/Setuptools; component -wheels are built with `--no-build-isolation`. - -## Generated drift - -When source types, CRDs, manifests, or templates change, run the repository's generation -targets and require a clean diff: - -```bash -make check-generated -make -C operator generate manifests -git diff --exit-code -``` - -Run commands from the component directory documented by its Makefile. Generated output -must be reproducible and reviewed together with its source. - -## Database migrations - -Every migration is tested twice: - -1. **Clean database:** apply all migrations from an empty supported database. -2. **Upgrade database:** restore the prior released schema with synthetic rows, apply - new migrations, and run old/new reader compatibility checks. - -Also test transaction rollback, repeated migration invocation, index/lock duration on a -representative synthetic volume, and forward recovery after a worker is interrupted. -Destructive contraction belongs to a later release. - -## Terraform - -CI runs: - -```bash -terraform fmt -check -recursive -make terraform-validate -tflint -``` - -The canonical validation helper runs backend-disabled init/validate for each root and -every committed `*.tftest.hcl` contract. Validation uses no live account and must not -require a variable file. Provider installation can require network access even with -backend initialization disabled. Plan tests use mocked or sandbox-only values and assert -that dispatch defaults off, one-shot canary and automatic inventory gates remain -separate, allowlists default empty, public storage is blocked, and existing -Config/CloudTrail/VPC resources are not replaced in reference mode. - -Never upload a plan containing live identifiers as a public CI artifact. - -## Kubernetes and Helm - -For every Kustomize root and Helm chart: - -- build or template with synthetic values; -- render both supported Kubernetes 1.35 and 1.36 versions and include chart CRDs; -- run lint; -- validate rendered objects with kubeconform; -- reject missing namespaces, mutable images, privileged defaults, broad RBAC, mounted - cloud keys, writable root filesystems without justification, and absent resource - limits; and -- verify dispatch is disabled in default and example values. - -CI exports every versioned custom-resource schema from the generated CRD and supplies it -to kubeconform without `-ignore-missing-schemas`. Custom-resource objects therefore fail -when their schema is absent or invalid. The CRD object itself is explicitly skipped by -kubeconform because the upstream built-in registry omits that type; operator envtest -installs the same CRD into a real API server and fails if the definition is rejected. - -## Containers - -Build every Dockerfile from a committed, tracked-only context, then: - -- build natively on both AMD64 and ARM64 GitHub runners; -- resolve every Terraform Lambda handler command inside its final image; -- start the operator binary with its argument parser; -- run component tests before publishing; -- inspect the final user, entrypoint, capabilities, and included files; -- reject secrets and package-manager caches; -- scan the image with Trivy; -- publish only immutable digests from protected release automation; and -- exercise startup with dispatch disabled. - -Check the AWS publication helper without AWS credentials, a registry login, or image -builds: - -```bash -bash -n terraform/aws/scripts/deploy.sh -bash -n terraform/aws/scripts/build-images.sh -bash -n terraform/aws/scripts/tests/deploy-test.sh -bash -n terraform/aws/scripts/tests/build-images-test.sh -terraform/aws/scripts/tests/deploy-test.sh -terraform/aws/scripts/tests/build-images-test.sh -``` - -The lightweight tests supply synthetic Terraform state/outputs and fake AWS/Docker -commands. They verify ordered multi-file staged plans, expected account/Region injection, -identity mismatch rejection, canary/activation/pause semantics, all seven -component/platform mappings, empty dry-run stdout, no image-helper dry-run AWS calls, and -rejection of unsafe arguments, outside roots, `latest`, and incomplete foundation -outputs. - -The Terraform-owned publication helper must preserve its clean-source gate and assemble -every Docker context from the selected committed source revision. Any explicit -dirty-worktree mode is for non-publishing local diagnosis only and must not authenticate, -push, or produce a release digest. - -If ShellCheck is installed, also run: - -```bash -shellcheck \ - tools/test_postgresql.sh \ - tools/test_migrator_package.sh \ - terraform/aws/scripts/deploy.sh \ - terraform/aws/scripts/build-images.sh \ - terraform/aws/scripts/tests/deploy-test.sh \ - terraform/aws/scripts/tests/build-images-test.sh -``` - -Network integration tests must use an isolated, explicitly authorized target owned by -the test operator. Public CI never scans a live host. - -## Integration tests - -A local integration stack should test: - -1. complete snapshot acceptance; -2. new/changed/known classification; -3. duplicate signal coalescing and current-state reread; -4. ownership `ACTIVE` dispatch; -5. stale/moved/inactive/UNKNOWN suppression; -6. result upload and parse; -7. Exposure update only for declared complete coverage; -8. Finding lifecycle; -9. replay idempotency; and -10. dead-letter and cleanup paths. - -Use fake provider clients and a fake scanner by default. A cloud sandbox test is manual, -approval-protected, disabled unless configured, and must always run cleanup. - -## CI map - -- core CI: Python, clean package payloads, PostgreSQL integration, Go/Python boundary, - operator envtest, schemas, and generated drift; -- Terraform: formatting, backend-disabled initialization/validation, contract tests, and - TFLint; -- Kubernetes: Kustomize, Helm/CRD rendering for 1.35 and 1.36, and kubeconform; -- containers: native AMD64/ARM64 clean builds, Lambda handler/operator startup checks, - and Trivy; -- CodeQL: supported source languages; -- supply chain: license/REUSE, gitleaks, pre-commit, and sanitizer; and -- AWS sandbox: manual OIDC workflow with environment approval and unconditional cleanup. - -The CodeQL workflow and `security-events: write` permission remain enabled. On a -repository without GitHub Advanced Security entitlement, result upload fails externally; -do not disable CodeQL or weaken its analysis to mask that entitlement failure. diff --git a/generator/.gitignore b/generator/.gitignore deleted file mode 100644 index f9dd8f0..0000000 --- a/generator/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -__pycache__/ -*.py[cod] -.pytest_cache/ -.ruff_cache/ -.venv/ -build/ -dist/ diff --git a/generator/Dockerfile b/generator/Dockerfile index e690088..f241490 100644 --- a/generator/Dockerfile +++ b/generator/Dockerfile @@ -26,7 +26,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps \ && rm -rf /opt/build RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ USER 65532:65532 ENTRYPOINT ["/usr/local/bin/python", "-m", "awslambdaric"] diff --git a/generator/README.md b/generator/README.md index 912d496..7fcc836 100644 --- a/generator/README.md +++ b/generator/README.md @@ -1,49 +1,48 @@ -# portscanner-generator + -`portscanner-generator` is a Python 3.12 AWS Lambda package that turns one -immutable `TargetEvent` object into at most one Kubernetes `Scanner` resource. -It validates the SQS/S3 source, checks event freshness, revalidates inventory -ownership immediately before dispatch, cancels older generations, and uses a -DynamoDB claim state machine for retry-safe idempotency. +# Portscanner generator -## Required environment +The generator is a Python 3.12 AWS Lambda that turns one immutable +`TargetEvent` into at most one namespaced Kubernetes `Scanner` resource. It is +an internal deployment component; users configure it through +[`terraform/aws`](../terraform/aws/README.md), not by invoking it directly. -- `TARGET_EVENT_BUCKET` (or `S3_BUCKET`) -- `TARGET_EVENT_PREFIX` (or `S3_PREFIX`) -- `IDEMPOTENCY_TABLE` (or `DYNAMODB_TABLE` / `DISPATCH_TABLE_NAME`) -- `INVENTORY_TABLE` (or `TARGET_TABLE_NAME`) when constructing the bundled - inventory ownership validator -- `EKS_CLUSTER_NAME` -- `K8S_NAMESPACE` -- `AWS_REGION` (or `AWS_DEFAULT_REGION`) +## Dispatch boundary -`K8S_API_GROUP`, when set, must be `scanning.portscanner.io`. -`K8S_API_VERSION`, when set, must be `v1alpha1`. +For each SQS/S3 event pointer, the generator: -The DynamoDB table partition key defaults to `dispatch_id` and can be changed with -`DYNAMODB_PARTITION_KEY`. Enable DynamoDB TTL on the `expires_at` attribute. +1. validates the configured bucket/prefix, object integrity, contract, and + freshness; +2. rejects targets outside the authorized account and deny-before-allow CIDR + policy; +3. acquires a DynamoDB idempotency claim and compares the current inventory + generation; +4. rereads provider ownership and scan-relevant policy immediately before + dispatch; +5. cancels superseded Scanner resources; and +6. creates one deterministic Scanner resource only while the target is + `ACTIVE` and the dispatch deadline is still valid. -The runtime prefers the shared `portscanner_contracts.parse_target_event()` -entrypoint so the contract package owns the complete event union, including -removals. It falls back to `TargetEvent.model_validate()` (or `from_dict()`) for -older released packages and isolated tests. When a contract does not carry a -separate trace identifier, its opaque event ID is propagated as the trace ID. -The inventory boundary supports both `validate_event(event)` and -`revalidate(target, generation)`. +`STALE`, `MOVED`, `INACTIVE`, ambiguous, or out-of-scope targets do not scan. +Retries may repeat reads but cannot create a second effective dispatch. -## Tests +The runtime receives exact bucket, table, EKS, namespace, account, Region, and +CIDR settings from Terraform. It uses the standard AWS credential chain and +EKS authentication; no kubeconfig or static cloud credential is embedded in +the image. -From the repository root, install the locked workspace and run the package tests. They -use only in-memory fakes: +## Development + +From the repository root: ```sh -uv sync --frozen --all-packages --group dev uv run --package portscanner-generator pytest generator/tests +docker build --file generator/Dockerfile --tag portscanner-generator:local . ``` -Build the Lambda image from the repository root so the shared contract and -inventory packages are included: - -```sh -docker build -f generator/Dockerfile . -``` +The repository root is the required image context because the generator +packages the shared contracts and inventory ownership adapter. Tests use +in-memory fakes and never contact a live cluster or target. diff --git a/generator/pyproject.toml b/generator/pyproject.toml index 5d76b28..5b3d64f 100644 --- a/generator/pyproject.toml +++ b/generator/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "portscanner-generator" -version = "0.1.0" +version = "1.0.0" description = "Freshness-gated TargetEvent dispatcher for Scanner resources" requires-python = ">=3.12" license = "MIT" @@ -16,8 +16,8 @@ dependencies = [ "awslambdaric>=4,<5", "boto3>=1.35", "kubernetes", - "portscanner-contracts>=0.1.0", - "portscanner-inventory>=0.1.0", + "portscanner-contracts>=1.0.0", + "portscanner-inventory>=1.0.0", ] [project.optional-dependencies] diff --git a/generator/requirements-runtime.txt b/generator/requirements-runtime.txt index 9942228..fca3d1d 100644 --- a/generator/requirements-runtime.txt +++ b/generator/requirements-runtime.txt @@ -101,17 +101,29 @@ attrs==26.1.0 \ awslambdaric==4.0.2 \ --hash=sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5 \ --hash=sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948 \ + --hash=sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e \ + --hash=sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f \ --hash=sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378 \ + --hash=sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594 \ + --hash=sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd \ + --hash=sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e \ + --hash=sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f \ --hash=sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb \ + --hash=sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e \ --hash=sha256:92cceacf2e37d455dd1c90a771b0b518f336971dace4cb771a5e43f9a595e867 \ --hash=sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318 \ --hash=sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005 \ + --hash=sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c \ + --hash=sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5 \ --hash=sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35 \ --hash=sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5 \ + --hash=sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c \ + --hash=sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68 \ --hash=sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372 \ --hash=sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6 \ --hash=sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3 \ - --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 + --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 \ + --hash=sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693 boto3==1.43.65 \ --hash=sha256:a8217fb68cae3f8a7575eef395383b68b846d92d29b0e8f6e948e6e9e08dcc3f \ --hash=sha256:f2331154aee1ae97ece48077d77f41d3bd5ea39eb4e3037030448b58695a3a79 diff --git a/generator/src/portscanner_generator/config.py b/generator/src/portscanner_generator/config.py index 6170bde..12d43e8 100644 --- a/generator/src/portscanner_generator/config.py +++ b/generator/src/portscanner_generator/config.py @@ -21,6 +21,7 @@ _TABLE_RE = re.compile(r"^[A-Za-z0-9_.-]{3,255}$") _CLUSTER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,99}$") _ATTRIBUTE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,254}$") +_ACCOUNT_RE = re.compile(r"^[0-9]{12}$") _REGION_RE = re.compile(r"^[a-z]{2}(?:-[a-z0-9]+)+-\d+$") @@ -79,6 +80,17 @@ def _cidrs(environment: Mapping[str, str], name: str) -> tuple[IPv4Network, ...] return tuple(sorted(networks, key=lambda item: (int(item.network_address), item.prefixlen))) +def _accounts(environment: Mapping[str, str]) -> tuple[str, ...]: + values = { + value.strip() + for value in environment.get("AUTHORIZED_ACCOUNT_IDS", "").split(",") + if value.strip() + } + if any(not _ACCOUNT_RE.fullmatch(value) for value in values): + raise ConfigurationError("AUTHORIZED_ACCOUNT_IDS must contain 12-digit account IDs") + return tuple(sorted(values)) + + @dataclass(frozen=True) class GeneratorConfig: """Complete, validated deployment configuration.""" @@ -99,6 +111,7 @@ class GeneratorConfig: max_event_bytes: int = 65_536 allowed_target_cidrs: tuple[IPv4Network, ...] = () denied_target_cidrs: tuple[IPv4Network, ...] = () + authorized_account_ids: tuple[str, ...] = () def __post_init__(self) -> None: bucket = self.event_bucket.strip() @@ -160,6 +173,8 @@ def __post_init__(self) -> None: for network in networks ): raise ConfigurationError(f"{name} must contain canonical IPv4 networks") + if any(not _ACCOUNT_RE.fullmatch(value) for value in self.authorized_account_ids): + raise ConfigurationError("AUTHORIZED_ACCOUNT_IDS must contain 12-digit account IDs") def target_allowed(self, address: str) -> bool: try: @@ -172,6 +187,9 @@ def target_allowed(self, address: str) -> bool: parsed in network for network in self.allowed_target_cidrs ) + def account_allowed(self, account_id: str) -> bool: + return account_id in self.authorized_account_ids + @classmethod def from_environment( cls, @@ -179,15 +197,9 @@ def from_environment( ) -> GeneratorConfig: env = environment if environment is not None else os.environ return cls( - event_bucket=_aliased(env, "TARGET_EVENT_BUCKET", "S3_BUCKET") or "", - event_prefix=_aliased(env, "TARGET_EVENT_PREFIX", "S3_PREFIX") or "", - table_name=_aliased( - env, - "IDEMPOTENCY_TABLE", - "DYNAMODB_TABLE", - "DISPATCH_TABLE_NAME", - ) - or "", + event_bucket=_aliased(env, "TARGET_EVENT_BUCKET") or "", + event_prefix=_aliased(env, "TARGET_EVENT_PREFIX") or "", + table_name=_aliased(env, "IDEMPOTENCY_TABLE") or "", cluster_name=_aliased(env, "EKS_CLUSTER_NAME") or "", namespace=_aliased(env, "K8S_NAMESPACE") or "", aws_region=_aliased( @@ -199,7 +211,6 @@ def from_environment( api_group=_aliased( env, "K8S_API_GROUP", - "K8S_CRD_GROUP", required=False, ) or SCANNER_API_GROUP, @@ -208,7 +219,6 @@ def from_environment( inventory_table_name=_aliased( env, "INVENTORY_TABLE", - "TARGET_TABLE_NAME", required=False, ), partition_key=env.get( @@ -224,4 +234,5 @@ def from_environment( max_event_bytes=_positive_int(env, "MAX_EVENT_BYTES", 65_536), allowed_target_cidrs=_cidrs(env, "ALLOWED_TARGET_CIDRS"), denied_target_cidrs=_cidrs(env, "DENIED_TARGET_CIDRS"), + authorized_account_ids=_accounts(env), ) diff --git a/generator/src/portscanner_generator/event_reader.py b/generator/src/portscanner_generator/event_reader.py index 6b733ba..ffd0cca 100644 --- a/generator/src/portscanner_generator/event_reader.py +++ b/generator/src/portscanner_generator/event_reader.py @@ -2,7 +2,6 @@ from __future__ import annotations -import importlib import json import re from collections.abc import Callable, Mapping @@ -10,6 +9,8 @@ from typing import Any from urllib.parse import unquote_plus +from portscanner_contracts import parse_target_event + from .config import GeneratorConfig _INVALID_PERCENT_ENCODING = re.compile(r"%(?![0-9A-Fa-f]{2})") @@ -165,31 +166,8 @@ def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def parse_shared_contract(document: Mapping[str, Any]) -> Any: """Parse all fields through the sibling shared contract package.""" - module = importlib.import_module("portscanner_contracts") - shared_parser = getattr(module, "parse_target_event", None) - if callable(shared_parser): - try: - return shared_parser(document) - except (KeyError, TypeError, ValueError) as error: - raise MalformedTargetEvent("TargetEvent failed shared contract validation") from error - - # Compatibility for released contract packages and isolated tests that - # predate parse_target_event(). New contract packages own the complete - # event union, including removal tombstones. - target_event_type = getattr(module, "TargetEvent", None) - if target_event_type is None: - models = importlib.import_module("portscanner_contracts.models") - target_event_type = getattr(models, "TargetEvent", None) - if target_event_type is None: - raise RuntimeError("portscanner_contracts does not export TargetEvent") - - parser = getattr(target_event_type, "from_dict", None) - if parser is None: - parser = getattr(target_event_type, "model_validate", None) - if not callable(parser): - raise RuntimeError("portscanner_contracts.TargetEvent has no supported parser") try: - return parser(document) + return parse_target_event(document) except (KeyError, TypeError, ValueError) as error: raise MalformedTargetEvent("TargetEvent failed shared contract validation") from error diff --git a/generator/src/portscanner_generator/handler.py b/generator/src/portscanner_generator/handler.py index 79ed409..9ebecdc 100644 --- a/generator/src/portscanner_generator/handler.py +++ b/generator/src/portscanner_generator/handler.py @@ -164,6 +164,18 @@ def _reason(event: Any) -> str: return _enum_text(event.scan.reason) +def _target_scope(event: Any) -> tuple[str, str]: + target = event.target + account_id = getattr(target, "scope_id", None) + address = getattr(target, "public_address", None) + if account_id is None: + context = getattr(event, "aws_context", None) + account_id = getattr(context, "account_id", None) + if not isinstance(account_id, str) or not isinstance(address, str): + raise ValueError("target event has no account/address scope") + return account_id, address + + def _retry_claim( services: GeneratorServices, claim: EventClaim, @@ -275,6 +287,29 @@ def process_sqs_record( is_removal = _event_type(event) == "target.removed" deadline_at = None if is_removal else _event_time(event.scan.deadline_at, "scan.deadline_at") not_after = None if is_removal else _event_time(event.scan.not_after, "scan.not_after") + account_id, public_address = _target_scope(event) + + if not services.config.account_allowed(account_id) or not services.config.target_allowed( + public_address + ): + _log( + logging.WARNING, + "event_rejected", + event_hash=event_hash, + trace_hash=trace_hash, + target_hash=opaque_target_hash, + generation=event.target.generation, + verdict="scope-denied", + outcome="scope_denied", + cancelled=0, + ) + return RecordOutcome( + "scope_denied", + event_hash, + trace_hash, + "scope-denied", + 0, + ) try: claim_result = services.claim_store.acquire( @@ -319,47 +354,6 @@ def process_sqs_record( trace_hash=trace_hash, ) - if not is_removal and not services.config.target_allowed(str(event.target.public_address)): - try: - exact_deleted = cancel_event_scanner( - services.scanner_client(), - event_id=event.event_id, - ) - services.claim_store.complete( - claim, - state=ClaimState.CANCELLED, - now=services.clock(), - verdict="scope-denied", - scanner_name=scanner_name(event.event_id), - cancelled_count=int(exact_deleted), - ) - except Exception as error: - _retry_claim(services, claim, error_code="scope_denial_failed") - raise RetryableRecordError( - "scope_denial_failed", - event_hash=event_hash, - trace_hash=trace_hash, - verdict="scope-denied", - ) from error - _log( - logging.WARNING, - "event_rejected", - event_hash=event_hash, - trace_hash=trace_hash, - target_hash=opaque_target_hash, - generation=event.target.generation, - verdict="scope-denied", - outcome="scope_denied", - cancelled=int(exact_deleted), - ) - return RecordOutcome( - "scope_denied", - event_hash, - trace_hash, - "scope-denied", - int(exact_deleted), - ) - try: check = revalidate_event(services.ownership_service, event) except Exception as error: diff --git a/generator/src/portscanner_generator/revalidation.py b/generator/src/portscanner_generator/revalidation.py index 618a899..9ec9039 100644 --- a/generator/src/portscanner_generator/revalidation.py +++ b/generator/src/portscanner_generator/revalidation.py @@ -156,7 +156,7 @@ def ownership_service_from_environment( from portscanner_inventory.aws.session import AwsClientFactory from portscanner_inventory.state import DynamoStateStore - table_name = state_table or env.get("INVENTORY_TABLE") or env.get("TARGET_TABLE_NAME") + table_name = state_table or env.get("INVENTORY_TABLE") if not table_name: raise RuntimeError("inventory ownership state table is not configured") if session is None: @@ -175,8 +175,20 @@ def ownership_service_from_environment( {value.strip() for value in env.get("ALLOWED_TAG_KEYS", "").split(",") if value.strip()} ) ) + allowed_interface_types = tuple( + sorted( + { + value.strip() + for value in env.get("ALLOWED_ENI_INTERFACE_TYPES", "").split(",") + if value.strip() + } + ) + ) return OwnershipValidator( state, factory, allowed_tag_keys=allowed_tags, + allowed_interface_types=allowed_interface_types, + required_tag_key=env.get("REQUIRED_TARGET_TAG_KEY") or None, + required_tag_value=env.get("REQUIRED_TARGET_TAG_VALUE") or None, ) diff --git a/generator/src/portscanner_generator/scanner_resource.py b/generator/src/portscanner_generator/scanner_resource.py index 8adff6d..a2427fc 100644 --- a/generator/src/portscanner_generator/scanner_resource.py +++ b/generator/src/portscanner_generator/scanner_resource.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from datetime import UTC, datetime from typing import Any @@ -89,21 +89,10 @@ def _port_range(value: Any) -> tuple[int, int]: return start, end -def _port_selection( - event: Any, - fallback_ports: Sequence[int] | None, -) -> tuple[list[int], list[dict[str, int]]]: +def _port_selection(event: Any) -> tuple[list[int], list[dict[str, int]]]: contract_ranges = getattr(event.scan, "tcp_port_ranges", None) if contract_ranges is None: - if fallback_ports is None: - raise ValueError("scan directive has no explicit TCP ports") - fallback_selection = sorted(set(fallback_ports)) - if any( - isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535 - for port in fallback_selection - ): - raise ValueError("ports must contain TCP port numbers") - return fallback_selection, [] + raise ValueError("scan directive has no explicit TCP ports") ports: list[int] = [] ranges: list[dict[str, int]] = [] @@ -122,7 +111,6 @@ def build_scanner_resource( event: Any, *, namespace: str, - ports: Sequence[int] | None = None, api_group: str = SCANNER_API_GROUP, api_version: str = SCANNER_API_VERSION, ) -> dict[str, Any]: @@ -132,10 +120,7 @@ def build_scanner_resource( if isinstance(target, (list, tuple, set, frozenset)): raise ValueError("a TargetEvent must contain exactly one target") target_id = _required_text(target.target_id, "target.target_id") - address_value = getattr(target, "public_address", None) - if address_value is None: - address_value = getattr(target, "address", None) - address = _required_text(address_value, "target.public_address") + address = _required_text(getattr(target, "public_address", None), "target.public_address") provider_value = getattr(target, "provider", None) provider = _required_text( getattr(provider_value, "value", provider_value), @@ -162,7 +147,7 @@ def build_scanner_resource( profile = _required_text(event.scan.profile, "scan.profile") if reason not in REASON_STATUS: raise ValueError("scan.reason has no readable status mapping") - explicit_ports, explicit_ranges = _port_selection(event, ports) + explicit_ports, explicit_ranges = _port_selection(event) priority = event.scan.priority if isinstance(priority, bool) or not isinstance(priority, int): raise ValueError("scan.priority must be an integer") diff --git a/generator/tests/test_dispatch.py b/generator/tests/test_dispatch.py index cb37719..4289ec3 100644 --- a/generator/tests/test_dispatch.py +++ b/generator/tests/test_dispatch.py @@ -42,6 +42,7 @@ def config() -> GeneratorConfig: cluster_name="scanner-cluster", namespace="scanner-system", aws_region="us-east-1", + authorized_account_ids=("123456789012",), ) @@ -83,6 +84,7 @@ def test_generator_denies_out_of_scope_target_before_ownership_or_creation(self) namespace="scanner-system", aws_region="us-east-1", allowed_target_cidrs=(IPv4Network("198.51.100.0/24"),), + authorized_account_ids=("123456789012",), ) response = lambda_handler( @@ -100,9 +102,31 @@ def test_generator_denies_out_of_scope_target_before_ownership_or_creation(self) self.assertEqual(response, {"batchItemFailures": []}) self.assertEqual(ownership.calls, []) self.assertEqual(scanner.created, []) - self.assertEqual(scanner.deleted, [exact_name]) - self.assertEqual(claims.completed[0]["state"], ClaimState.CANCELLED) - self.assertEqual(claims.completed[0]["verdict"], "scope-denied") + self.assertEqual(scanner.deleted, []) + self.assertEqual(claims.acquired, []) + self.assertEqual(claims.completed, []) + + def test_generator_denies_unauthorized_account_before_claim_or_target_apis(self) -> None: + document = event_document() + target = document["target"] + assert isinstance(target, dict) + target["scope_id"] = "222222222222" + scanner = FakeScannerClient() + claims = FakeClaimStore() + ownership = FakeOwnershipService("ACTIVE") + + response = lambda_handler( + {"Records": [sqs_record(document)]}, + None, + services=services_for(document, ownership, claims, scanner), + ) + + self.assertEqual(response, {"batchItemFailures": []}) + self.assertEqual(claims.acquired, []) + self.assertEqual(claims.completed, []) + self.assertEqual(ownership.calls, []) + self.assertEqual(scanner.created, []) + self.assertEqual(scanner.deleted, []) def test_active_exact_generation_dispatches_after_cancelling_older(self) -> None: document = event_document() diff --git a/generator/tests/test_event_reader.py b/generator/tests/test_event_reader.py index 2ea703a..4301b77 100644 --- a/generator/tests/test_event_reader.py +++ b/generator/tests/test_event_reader.py @@ -1,8 +1,6 @@ from __future__ import annotations import json -import sys -import types import unittest from datetime import timedelta @@ -12,7 +10,6 @@ MutableS3Object, UnexpectedS3Object, decode_sqs_s3_record, - parse_shared_contract, read_target_event, ) from portscanner_generator.handler import GeneratorServices, lambda_handler @@ -38,6 +35,7 @@ def config(**overrides: object) -> GeneratorConfig: "cluster_name": "scanner-cluster", "namespace": "scanner-system", "aws_region": "us-east-1", + "authorized_account_ids": ("123456789012",), } values.update(overrides) return GeneratorConfig(**values) # type: ignore[arg-type] @@ -119,83 +117,6 @@ def test_requires_immutable_object_identity(self) -> None: config(), ) - def test_shared_contract_receives_complete_document(self) -> None: - captured: list[dict[str, object]] = [] - - class SharedTargetEvent: - @classmethod - def from_dict(cls, value: dict[str, object]) -> object: - captured.append(value) - return object() - - module = types.ModuleType("portscanner_contracts") - module.TargetEvent = SharedTargetEvent # type: ignore[attr-defined] - document = event_document() - original = sys.modules.get("portscanner_contracts") - sys.modules["portscanner_contracts"] = module - try: - parse_shared_contract(document) - finally: - if original is None: - sys.modules.pop("portscanner_contracts", None) - else: - sys.modules["portscanner_contracts"] = original - - self.assertEqual(captured, [document]) - self.assertIn("provider_metadata", captured[0]) - self.assertIn("policy", captured[0]) - - def test_shared_parse_target_event_is_preferred_when_available(self) -> None: - captured: list[dict[str, object]] = [] - sentinel = object() - - module = types.ModuleType("portscanner_contracts") - - def parse_target_event(value: dict[str, object]) -> object: - captured.append(value) - return sentinel - - class LegacyTargetEvent: - @classmethod - def model_validate(cls, _: object) -> object: - raise AssertionError("legacy parser must not be used") - - module.parse_target_event = parse_target_event # type: ignore[attr-defined] - module.TargetEvent = LegacyTargetEvent # type: ignore[attr-defined] - document = event_document() - original = sys.modules.get("portscanner_contracts") - sys.modules["portscanner_contracts"] = module - try: - parsed = parse_shared_contract(document) - finally: - if original is None: - sys.modules.pop("portscanner_contracts", None) - else: - sys.modules["portscanner_contracts"] = original - - self.assertIs(parsed, sentinel) - self.assertEqual(captured, [document]) - - def test_removal_tombstone_is_owned_by_shared_event_parser(self) -> None: - document = {"event_type": "target.removed", "event_id": "removal-event-hash"} - expected = types.SimpleNamespace( - event_type="target.removed", - event_id="removal-event-hash", - ) - module = types.ModuleType("portscanner_contracts") - module.parse_target_event = lambda value: expected # type: ignore[attr-defined] - original = sys.modules.get("portscanner_contracts") - sys.modules["portscanner_contracts"] = module - try: - event = parse_shared_contract(document) - finally: - if original is None: - sys.modules.pop("portscanner_contracts", None) - else: - sys.modules["portscanner_contracts"] = original - - self.assertIs(event, expected) - def test_wrong_source_and_malformed_json_are_redriven(self) -> None: malformed = sqs_record("{not-json") wrong_source = sqs_record( @@ -300,6 +221,7 @@ def test_configuration_parses_canonical_cidrs_and_applies_denies_first(self) -> "EKS_CLUSTER_NAME": "scanner-cluster", "K8S_NAMESPACE": "scanner-system", "AWS_REGION": "us-east-1", + "AUTHORIZED_ACCOUNT_IDS": "222222222222,123456789012", "ALLOWED_TARGET_CIDRS": "203.0.113.0/24", "DENIED_TARGET_CIDRS": "203.0.113.10/32", } @@ -308,11 +230,18 @@ def test_configuration_parses_canonical_cidrs_and_applies_denies_first(self) -> self.assertTrue(settings.target_allowed("203.0.113.11")) self.assertFalse(settings.target_allowed("203.0.113.10")) self.assertFalse(settings.target_allowed("198.51.100.1")) + self.assertTrue(settings.account_allowed("123456789012")) + self.assertFalse(settings.account_allowed("000000000000")) environment["ALLOWED_TARGET_CIDRS"] = "203.0.113.10/24" with self.assertRaisesRegex(ConfigurationError, "canonical IPv4"): GeneratorConfig.from_environment(environment) + environment["ALLOWED_TARGET_CIDRS"] = "203.0.113.0/24" + environment["AUTHORIZED_ACCOUNT_IDS"] = "not-an-account" + with self.assertRaisesRegex(ConfigurationError, "12-digit"): + GeneratorConfig.from_environment(environment) + def test_not_after_boundary_includes_later_time(self) -> None: document = event_document(not_after=NOW - timedelta(microseconds=1)) s3 = FakeS3(json.dumps(document).encode()) diff --git a/generator/tests/test_scanner_resource.py b/generator/tests/test_scanner_resource.py index 5ca65d7..3093ed1 100644 --- a/generator/tests/test_scanner_resource.py +++ b/generator/tests/test_scanner_resource.py @@ -116,6 +116,22 @@ def test_reason_annotations_match_talk_labels(self) -> None: label, ) + def test_managed_canary_targeted_port_is_preserved_exactly(self) -> None: + document = event_document(reason="new_target") + scan = document["scan"] + assert isinstance(scan, dict) + scan["profile"] = "targeted-tcp" + scan["tcp_port_ranges"] = [{"start": 18080, "end": 18080}] + + body = build_scanner_resource( + parse_test_event(document), + namespace="scanner-system", + ) + + self.assertEqual(body["spec"]["profile"], "targeted-tcp") + self.assertEqual(body["spec"]["ports"], [18080]) + self.assertEqual(body["spec"]["ranges"], []) + def test_labels_and_name_never_contain_raw_target_data(self) -> None: event = parse_test_event(event_document()) body = build_scanner_resource( @@ -126,7 +142,7 @@ def test_labels_and_name_never_contain_raw_target_data(self) -> None: serialized_labels = " ".join(f"{key}={value}" for key, value in labels.items()) self.assertNotIn(event.target.target_id, serialized_labels) - self.assertNotIn(event.target.address, serialized_labels) + self.assertNotIn(event.target.public_address, serialized_labels) self.assertEqual( labels[TARGET_HASH_LABEL], target_hash(event.target.target_id), diff --git a/inventory/Dockerfile b/inventory/Dockerfile index c43cacf..ea799f5 100644 --- a/inventory/Dockerfile +++ b/inventory/Dockerfile @@ -24,7 +24,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps \ && rm -rf /opt/build RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ USER 65532:65532 ENTRYPOINT ["/usr/local/bin/python", "-m", "awslambdaric"] diff --git a/inventory/README.md b/inventory/README.md new file mode 100644 index 0000000..e7561c7 --- /dev/null +++ b/inventory/README.md @@ -0,0 +1,79 @@ + + +# Portscanner inventory + +Inventory establishes which public address the deployment is authorized to +scan. It is the source of target identity, monotonic generation, current +ownership, and bounded scan policy; scanner output never creates inventory. + +## AWS model + +The first release supports public IPv4 addresses attached to supported AWS +EC2 network interfaces. A target identity is stable across observations while +its address, security-group policy, lifecycle state, and generation may +change. + +Authoritative snapshots use either direct EC2 reads or an explicitly +configured AWS Config aggregator. Optional EventBridge and CloudTrail signals +are acceleration hints: they trigger a current-state reread but are not trusted +as inventory evidence by themselves. + +Snapshot reconciliation writes target state and a transactional outbox record +in one DynamoDB transaction. The outbox publishes an immutable `TargetEvent` +to S3 and the appropriate queues. DynamoDB Streams is the fast path; a bounded +repair schedule retries pending outbox rows. + +## Dispatch safety + +Before a scan is created, inventory and generator checks enforce: + +- an explicitly authorized account and Region; +- deny-before-allow IPv4 CIDR policy; +- a supported resource class and optional target-selection tags; +- an active target at the expected generation; and +- an immediate provider reread of the address binding and scan-relevant + security-group state. + +`STALE`, `MOVED`, `INACTIVE`, ambiguous, incomplete, or out-of-scope state +must not dispatch. Provider addresses can be recycled, so a queued event alone +never proves present ownership. + +Complete snapshots may reconcile absence. Scoped canary reads and partial or +failed snapshots must not remove targets they did not observe. + +## Default evaluation + +The canonical AWS deployment creates one isolated managed canary and invokes a +single scoped snapshot. Only that target receives the evaluation's +`targeted-tcp` directive. Periodic account inventory and signal processing +remain disabled until the operator edits the same environment file and +explicitly activates an authorized account, CIDR, and tag scope. + +## Adding an inventory source + +New providers are not enabled by adding arbitrary scanner targets. A source +adapter must provide: + +1. Stable provider target identity and monotonic generation semantics. +2. Complete-snapshot boundaries and explicit partial/failure behavior. +3. Current ownership revalidation immediately before dispatch. +4. Normalized lifecycle, address, location, and minimized context. +5. Provider-policy, account/project, CIDR, pagination, and rate bounds. +6. Deterministic synthetic fixtures for create, change, removal, stale work, + pagination, retries, and recycled-address suppression. + +Signals must resolve to a provider reread and cannot emit removals from an +unverified event payload. Keep provider SDK objects and credentials outside +the public contracts and finding exports. + +Run the focused package checks from the repository root: + +```sh +uv run --package portscanner-inventory pytest inventory/tests +``` + +Contract, Terraform, generator, and end-to-end boundary tests are also +required for any source that can cause network traffic. diff --git a/inventory/pyproject.toml b/inventory/pyproject.toml index c75b5e5..039e32f 100644 --- a/inventory/pyproject.toml +++ b/inventory/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "portscanner-inventory" -version = "0.1.0" +version = "1.0.0" description = "AWS public-IPv4 inventory and change discovery" requires-python = ">=3.12" license = "MIT" @@ -15,7 +15,7 @@ classifiers = [ dependencies = [ "awslambdaric>=4,<5", "boto3>=1.35", - "portscanner-contracts>=0.1.0", + "portscanner-contracts>=1.0.0", ] [project.optional-dependencies] diff --git a/inventory/requirements-runtime.txt b/inventory/requirements-runtime.txt index 4b4a419..937fd84 100644 --- a/inventory/requirements-runtime.txt +++ b/inventory/requirements-runtime.txt @@ -8,17 +8,29 @@ annotated-types==0.8.0 \ awslambdaric==4.0.2 \ --hash=sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5 \ --hash=sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948 \ + --hash=sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e \ + --hash=sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f \ --hash=sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378 \ + --hash=sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594 \ + --hash=sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd \ + --hash=sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e \ + --hash=sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f \ --hash=sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb \ + --hash=sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e \ --hash=sha256:92cceacf2e37d455dd1c90a771b0b518f336971dace4cb771a5e43f9a595e867 \ --hash=sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318 \ --hash=sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005 \ + --hash=sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c \ + --hash=sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5 \ --hash=sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35 \ --hash=sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5 \ + --hash=sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c \ + --hash=sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68 \ --hash=sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372 \ --hash=sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6 \ --hash=sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3 \ - --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 + --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 \ + --hash=sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693 boto3==1.43.65 \ --hash=sha256:a8217fb68cae3f8a7575eef395383b68b846d92d29b0e8f6e948e6e9e08dcc3f \ --hash=sha256:f2331154aee1ae97ece48077d77f41d3bd5ea39eb4e3037030448b58695a3a79 diff --git a/inventory/src/portscanner_inventory/aws/config_snapshot.py b/inventory/src/portscanner_inventory/aws/config_snapshot.py index ea3a659..3ef0596 100644 --- a/inventory/src/portscanner_inventory/aws/config_snapshot.py +++ b/inventory/src/portscanner_inventory/aws/config_snapshot.py @@ -27,6 +27,7 @@ resourceId, resourceType, configurationItemCaptureTime, + tags, configuration WHERE resourceType = 'AWS::EC2::NetworkInterface' @@ -39,6 +40,7 @@ resourceId, resourceType, configurationItemCaptureTime, + tags, configuration WHERE resourceType = 'AWS::EC2::SecurityGroup' @@ -93,21 +95,32 @@ def __init__( aggregator_name: str, scope: SnapshotScope | None = None, allowed_tag_keys: Sequence[str] = (), + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, page_size: int = 100, + max_pages: int = 1000, ) -> None: self._client = client self._aggregator_name = aggregator_name self._scope = scope or SnapshotScope(source="aws-config", name=aggregator_name) self._allowed_tag_keys = tuple(allowed_tag_keys) + self._allowed_interface_types = tuple(allowed_interface_types) + self._required_tag_key = required_tag_key + self._required_tag_value = required_tag_value self._page_size = page_size + self._max_pages = max_pages if self._scope.source != "aws-config": raise ValueError("Config backend requires an aws-config scope") if not 1 <= page_size <= 100: raise ValueError("Config page_size must be within 1..100") + if not 1 <= max_pages <= 10_000: + raise ValueError("Config max_pages must be within 1..10000") def fetch_records(self, expression: str) -> _Fetch: records: list[Mapping[str, Any]] = [] token: str | None = None + seen_tokens: set[str] = set() pages = 0 malformed = 0 while True: @@ -141,7 +154,23 @@ def fetch_records(self, expression: str) -> _Fetch: next_token = response.get("NextToken") if not next_token: return _Fetch(tuple(records), pages, malformed) - token = str(next_token) + next_value = str(next_token) + if next_value in seen_tokens: + return _Fetch( + tuple(records), + pages, + malformed, + "pagination-token-repeated", + ) + if pages >= self._max_pages: + return _Fetch( + tuple(records), + pages, + malformed, + "pagination-page-limit", + ) + seen_tokens.add(next_value) + token = next_value def collect(self) -> SnapshotBatch: group_fetch = self.fetch_records(SECURITY_GROUP_QUERY) @@ -224,6 +253,9 @@ def collect(self) -> SnapshotBatch: observed_at=composite_observed_at, eni_observed_at=eni_captured_at, security_group_observed_at=group_captures, + allowed_interface_types=self._allowed_interface_types, + required_tag_key=self._required_tag_key, + required_tag_value=self._required_tag_value, ): existing = normalized.get(target.target_id) if existing is None or ( diff --git a/inventory/src/portscanner_inventory/aws/ec2_snapshot.py b/inventory/src/portscanner_inventory/aws/ec2_snapshot.py index f91da70..e34f06b 100644 --- a/inventory/src/portscanner_inventory/aws/ec2_snapshot.py +++ b/inventory/src/portscanner_inventory/aws/ec2_snapshot.py @@ -34,18 +34,29 @@ def __init__( account_id: str, region: str, allowed_tag_keys: Sequence[str] = (), + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, page_size: int = 500, + max_pages: int = 1000, ) -> None: self._client = client self._scope = SnapshotScope(source="ec2", account_id=account_id, region=region) self._allowed_tag_keys = tuple(allowed_tag_keys) + self._allowed_interface_types = tuple(allowed_interface_types) + self._required_tag_key = required_tag_key + self._required_tag_value = required_tag_value self._page_size = page_size + self._max_pages = max_pages if not 5 <= page_size <= 1000: raise ValueError("EC2 page size must be within 5..1000") + if not 1 <= max_pages <= 10_000: + raise ValueError("EC2 max_pages must be within 1..10000") def _fetch(self, operation: str, result_key: str) -> _PageResult: values: list[Mapping[str, Any]] = [] token: str | None = None + seen_tokens: set[str] = set() pages = 0 while True: request: dict[str, Any] = {"MaxResults": self._page_size} @@ -62,7 +73,13 @@ def _fetch(self, operation: str, result_key: str) -> _PageResult: next_token = response.get("NextToken") if not next_token: return _PageResult(tuple(values), pages) - token = str(next_token) + next_value = str(next_token) + if next_value in seen_tokens: + return _PageResult(tuple(values), pages, "pagination-token-repeated") + if pages >= self._max_pages: + return _PageResult(tuple(values), pages, "pagination-page-limit") + seen_tokens.add(next_value) + token = next_value def collect(self) -> SnapshotBatch: group_fetch = self._fetch("describe_security_groups", "SecurityGroups") @@ -110,6 +127,9 @@ def collect(self) -> SnapshotBatch: security_groups=security_groups, allowed_tag_keys=self._allowed_tag_keys, instance_state=instance_states.get(instance_id or ""), + allowed_interface_types=self._allowed_interface_types, + required_tag_key=self._required_tag_key, + required_tag_value=self._required_tag_value, ): normalized[target.target_id] = target except (KeyError, TypeError, ValueError): diff --git a/inventory/src/portscanner_inventory/aws/normalize.py b/inventory/src/portscanner_inventory/aws/normalize.py index 57f4a22..dc7cc6d 100644 --- a/inventory/src/portscanner_inventory/aws/normalize.py +++ b/inventory/src/portscanner_inventory/aws/normalize.py @@ -171,7 +171,7 @@ def _validate_private_ip(value: Any) -> str: return str(ipaddress.IPv4Address(str(value))) -def _tags(value: Mapping[str, Any], allowlist: frozenset[str]) -> tuple[tuple[str, str], ...]: +def _tag_values(value: Mapping[str, Any]) -> dict[str, str]: raw = _get(value, "TagSet", "tagSet", "Tags", "tags", default=()) or () if isinstance(raw, Mapping) and isinstance( _get(raw, "items", "Items"), @@ -192,7 +192,7 @@ def _tags(value: Mapping[str, Any], allowlist: frozenset[str]) -> tuple[tuple[st ) sanitized: dict[str, str] = {} for key, item_value in pairs: - if not isinstance(key, str) or key not in allowlist or not isinstance(item_value, str): + if not isinstance(key, str) or not isinstance(item_value, str): continue if ( not item_value.strip() @@ -201,9 +201,39 @@ def _tags(value: Mapping[str, Any], allowlist: frozenset[str]) -> tuple[tuple[st ): continue sanitized[key] = item_value + return sanitized + + +def _tags(value: Mapping[str, Any], allowlist: frozenset[str]) -> tuple[tuple[str, str], ...]: + sanitized = { + key: item_value for key, item_value in _tag_values(value).items() if key in allowlist + } return tuple(sorted(sanitized.items())) +def network_interface_supported( + value: Mapping[str, Any], + *, + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, +) -> bool: + """Apply an optional ENI-class and opt-in tag gate before normalization.""" + + configuration = _as_mapping(_get(value, "configuration", "Configuration", default=value)) + interface_type = str(_get(configuration, "InterfaceType", "interfaceType", default="")).lower() + if allowed_interface_types and interface_type not in set(allowed_interface_types): + return False + if required_tag_key is None and required_tag_value is None: + return True + if required_tag_key is None or required_tag_value is None: + raise ValueError("required target tag key and value must be configured together") + tags = _tag_values(configuration) + if configuration is not value: + tags.update(_tag_values(value)) + return tags.get(required_tag_key) == required_tag_value + + def _group_ids(value: Mapping[str, Any]) -> tuple[str, ...]: raw_groups = _get(value, "Groups", "groups", "groupSet", default=()) or () if isinstance(raw_groups, Mapping): @@ -331,6 +361,7 @@ class NormalizedTarget: association_id: str | None = None tags: tuple[tuple[str, str], ...] = () candidate_ports: CandidatePorts = field(default_factory=CandidatePorts.full) + managed_canary: bool = False source_event_name: str | None = None source_event_id: str | None = None source_request_id: str | None = None @@ -350,6 +381,8 @@ def __post_init__(self) -> None: raise ValueError("security_group_ids must be sorted and unique") if tuple(sorted(self.tags)) != self.tags: raise ValueError("tags must be sorted") + if not isinstance(self.managed_canary, bool): + raise ValueError("managed_canary must be boolean") if not re.fullmatch(r"[0-9a-f]{64}", self.policy_fingerprint): raise ValueError("invalid policy fingerprint") if self.observed_at is not None: @@ -496,6 +529,7 @@ def to_state_dict(self) -> dict[str, Any]: "allocation_id": self.allocation_id, "association_id": self.association_id, "tags": dict(self.tags), + "managed_canary": self.managed_canary, "observed_at": ( self.observed_at.isoformat().replace("+00:00", "Z") if self.observed_at is not None @@ -553,6 +587,7 @@ def from_state_dict(cls, value: Mapping[str, Any]) -> NormalizedTarget: allocation_id=str(value["allocation_id"]) if value.get("allocation_id") else None, association_id=str(value["association_id"]) if value.get("association_id") else None, tags=tuple(sorted((str(key), str(item)) for key, item in tags.items())), + managed_canary=bool(value.get("managed_canary", False)), observed_at=observed_at, eni_observed_at=eni_observed_at, security_group_observed_at=group_observations, @@ -570,8 +605,18 @@ def normalize_network_interface( observed_at: datetime | None = None, eni_observed_at: datetime | None = None, security_group_observed_at: tuple[tuple[str, datetime], ...] = (), + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, ) -> tuple[NormalizedTarget, ...]: configuration = _as_mapping(_get(value, "configuration", "Configuration", default=value)) + if not network_interface_supported( + value, + allowed_interface_types=allowed_interface_types, + required_tag_key=required_tag_key, + required_tag_value=required_tag_value, + ): + return () eni_id = str( _get( configuration, @@ -592,6 +637,15 @@ def normalize_network_interface( attachment_id = _get(attachment, "AttachmentId", "attachmentId") lifecycle = _lifecycle(configuration, instance_state=instance_state) tags = _tags(configuration, frozenset(allowed_tag_keys)) + if configuration is not value: + tags = tuple( + sorted( + { + **dict(tags), + **dict(_tags(value, frozenset(allowed_tag_keys))), + }.items() + ) + ) return tuple( NormalizedTarget( account_id=account_id, diff --git a/inventory/src/portscanner_inventory/aws/ownership.py b/inventory/src/portscanner_inventory/aws/ownership.py index 89f08b4..74e9f51 100644 --- a/inventory/src/portscanner_inventory/aws/ownership.py +++ b/inventory/src/portscanner_inventory/aws/ownership.py @@ -27,10 +27,16 @@ def __init__( client_factory: Any, *, allowed_tag_keys: Sequence[str] = (), + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, ) -> None: self._state = state self._factory = client_factory self._allowed_tag_keys = tuple(allowed_tag_keys) + self._allowed_interface_types = tuple(allowed_interface_types) + self._required_tag_key = required_tag_key + self._required_tag_value = required_tag_value def _client(self, target: NormalizedTarget) -> Any: if hasattr(self._factory, "client"): @@ -183,6 +189,9 @@ def validate(self, target_id: str, generation: int) -> OwnershipCheck: security_groups=groups, allowed_tag_keys=self._allowed_tag_keys, instance_state=instance_state, + allowed_interface_types=self._allowed_interface_types, + required_tag_key=self._required_tag_key, + required_tag_value=self._required_tag_value, ) except Exception as error: return OwnershipCheck(OwnershipVerdict.UNKNOWN, reason=aws_error_code(error)) diff --git a/inventory/src/portscanner_inventory/aws/resolve.py b/inventory/src/portscanner_inventory/aws/resolve.py index ca02b00..60ec2e6 100644 --- a/inventory/src/portscanner_inventory/aws/resolve.py +++ b/inventory/src/portscanner_inventory/aws/resolve.py @@ -30,11 +30,21 @@ def __init__( client_factory: Any, *, allowed_tag_keys: Sequence[str] = (), + allowed_interface_types: Sequence[str] = (), + required_tag_key: str | None = None, + required_tag_value: str | None = None, page_size: int = 500, + max_pages: int = 1000, ) -> None: self._factory = client_factory self._allowed_tag_keys = tuple(allowed_tag_keys) + self._allowed_interface_types = tuple(allowed_interface_types) + self._required_tag_key = required_tag_key + self._required_tag_value = required_tag_value self._page_size = page_size + self._max_pages = max_pages + if not 1 <= max_pages <= 10_000: + raise ValueError("EC2 resolver max_pages must be within 1..10000") def _client(self, hint: SignalHint) -> Any: if hasattr(self._factory, "client"): @@ -53,6 +63,8 @@ def _paginate_enis( ) -> tuple[Mapping[str, Any], ...]: values: list[Mapping[str, Any]] = [] token: str | None = None + seen_tokens: set[str] = set() + pages = 0 while True: request: dict[str, Any] = { "Filters": list(filters), @@ -61,13 +73,20 @@ def _paginate_enis( if token: request["NextToken"] = token response = client.describe_network_interfaces(**request) + pages += 1 values.extend( item for item in response.get("NetworkInterfaces", ()) if isinstance(item, Mapping) ) next_token = response.get("NextToken") if not next_token: return tuple(values) - token = str(next_token) + next_value = str(next_token) + if next_value in seen_tokens: + raise ValueError("EC2 pagination token repeated") + if pages >= self._max_pages: + raise ValueError("EC2 pagination page limit exceeded") + seen_tokens.add(next_value) + token = next_value def _read_eni_ids( self, @@ -267,6 +286,9 @@ def resolve(self, hint: SignalHint) -> Resolution: security_groups=groups, allowed_tag_keys=self._allowed_tag_keys, instance_state=instance_states.get(instance_id or ""), + allowed_interface_types=self._allowed_interface_types, + required_tag_key=self._required_tag_key, + required_tag_value=self._required_tag_value, ): signaled = target.with_signal( event_name=hint.event_name, diff --git a/inventory/src/portscanner_inventory/config.py b/inventory/src/portscanner_inventory/config.py index bdbaad0..2bc0658 100644 --- a/inventory/src/portscanner_inventory/config.py +++ b/inventory/src/portscanner_inventory/config.py @@ -5,10 +5,14 @@ import re from collections.abc import Mapping from dataclasses import dataclass +from ipaddress import AddressValueError, IPv4Address, IPv4Network +from typing import Any from portscanner_contracts import AWS_TAG_KEYS _ACCOUNT_RE = re.compile(r"^[0-9]{12}$") +_ENI_RE = re.compile(r"^eni-[0-9a-fA-F]+$") +_REGION_RE = re.compile(r"^[a-z]{2}(?:-[a-z0-9]+)+-[0-9]+$") _TAG_RE = re.compile(r"^[A-Za-z0-9_.:/=+\-@]{1,128}$") @@ -36,6 +40,96 @@ def _boolean(env: Mapping[str, str], name: str, default: bool = False) -> bool: return raw.lower() == "true" +def _cidrs(env: Mapping[str, str], name: str) -> tuple[IPv4Network, ...]: + networks: set[IPv4Network] = set() + for raw in env.get(name, "").split(","): + value = raw.strip() + if not value: + continue + try: + network = IPv4Network(value, strict=True) + except ValueError as error: + raise ValueError(f"{name} must contain canonical IPv4 networks") from error + if str(network) != value: + raise ValueError(f"{name} must contain canonical IPv4 networks") + networks.add(network) + return tuple(sorted(networks, key=lambda item: (int(item.network_address), item.prefixlen))) + + +@dataclass(frozen=True, slots=True) +class ManagedCanary: + account_id: str + region: str + network_interface_id: str + private_ip: str + public_ip: str + tag_key: str + tag_value: str + tcp_port: int + + def __post_init__(self) -> None: + if not _ACCOUNT_RE.fullmatch(self.account_id): + raise ValueError("managed canary account must contain 12 digits") + if not _REGION_RE.fullmatch(self.region): + raise ValueError("managed canary region is invalid") + if not _ENI_RE.fullmatch(self.network_interface_id): + raise ValueError("managed canary ENI is invalid") + try: + private = IPv4Address(self.private_ip) + public = IPv4Address(self.public_ip) + except AddressValueError as error: + raise ValueError("managed canary addresses must be canonical IPv4") from error + if str(private) != self.private_ip or str(public) != self.public_ip or not public.is_global: + raise ValueError("managed canary addresses must be canonical public/private IPv4") + if not _TAG_RE.fullmatch(self.tag_key) or not self.tag_value.strip(): + raise ValueError("managed canary tag is invalid") + if not 1 <= self.tcp_port <= 65535: + raise ValueError("managed canary TCP port must be within 1..65535") + + def matches(self, target: Any) -> bool: + return bool( + target.account_id == self.account_id + and target.region == self.region + and target.network_interface_id == self.network_interface_id + and target.private_ip == self.private_ip + and target.public_ip == self.public_ip + and target.tags_dict.get(self.tag_key) == self.tag_value + ) + + +def _managed_canary(env: Mapping[str, str]) -> ManagedCanary | None: + names = ( + "MANAGED_CANARY_ACCOUNT_ID", + "MANAGED_CANARY_REGION", + "MANAGED_CANARY_ENI_ID", + "MANAGED_CANARY_PRIVATE_IP", + "MANAGED_CANARY_PUBLIC_IP", + "MANAGED_CANARY_TAG_KEY", + "MANAGED_CANARY_TAG_VALUE", + "MANAGED_CANARY_TCP_PORT", + ) + values = {name: env.get(name, "").strip() for name in names} + configured = {name for name, value in values.items() if value} + if not configured: + return None + if configured != set(names): + raise ValueError("managed canary runtime configuration must be complete") + try: + port = int(values["MANAGED_CANARY_TCP_PORT"]) + except ValueError as error: + raise ValueError("MANAGED_CANARY_TCP_PORT must be an integer") from error + return ManagedCanary( + account_id=values["MANAGED_CANARY_ACCOUNT_ID"], + region=values["MANAGED_CANARY_REGION"], + network_interface_id=values["MANAGED_CANARY_ENI_ID"], + private_ip=values["MANAGED_CANARY_PRIVATE_IP"], + public_ip=values["MANAGED_CANARY_PUBLIC_IP"], + tag_key=values["MANAGED_CANARY_TAG_KEY"], + tag_value=values["MANAGED_CANARY_TAG_VALUE"], + tcp_port=port, + ) + + @dataclass(frozen=True, slots=True) class Settings: state_table: str @@ -50,9 +144,16 @@ class Settings: authorized_account_ids: tuple[str, ...] = () regions: tuple[str, ...] = () canary_mode: bool = False + managed_canary: ManagedCanary | None = None discovery_role_arn_template: str | None = None external_id: str | None = None allowed_tag_keys: tuple[str, ...] = () + allowed_target_cidrs: tuple[IPv4Network, ...] = () + denied_target_cidrs: tuple[IPv4Network, ...] = () + allowed_eni_interface_types: tuple[str, ...] = () + required_target_tag_key: str | None = None + required_target_tag_value: str | None = None + snapshot_max_pages: int = 1000 signal_dedupe_seconds: int = 86_400 outbox_ttl_seconds: int = 604_800 outbox_replay_batch_size: int = 100 @@ -67,6 +168,8 @@ def __post_init__(self) -> None: raise ValueError("account_id must contain 12 digits") if any(not _ACCOUNT_RE.fullmatch(value) for value in self.authorized_account_ids): raise ValueError("authorized_account_ids must contain 12-digit account IDs") + if any(not _REGION_RE.fullmatch(value) for value in self.regions): + raise ValueError("regions must contain valid AWS regions") if self.discovery_role_arn_template: if "{account_id}" not in self.discovery_role_arn_template: raise ValueError("role ARN template must include {account_id}") @@ -76,6 +179,20 @@ def __post_init__(self) -> None: raise ValueError("invalid tag allowlist entry") if not set(self.allowed_tag_keys).issubset(AWS_TAG_KEYS): raise ValueError("tag allowlist contains a field outside the shared AWS context") + if any( + not re.fullmatch(r"[a-z0-9-]+", value) for value in self.allowed_eni_interface_types + ): + raise ValueError("allowed ENI interface types are invalid") + if (self.required_target_tag_key is None) != (self.required_target_tag_value is None): + raise ValueError("required target tag key and value must be configured together") + if self.required_target_tag_key is not None and ( + not _TAG_RE.fullmatch(self.required_target_tag_key) + or not self.required_target_tag_value + or not self.required_target_tag_value.strip() + ): + raise ValueError("required target tag is invalid") + if not 1 <= self.snapshot_max_pages <= 10_000: + raise ValueError("snapshot_max_pages must be between 1 and 10000") if not 86_400 <= self.outbox_ttl_seconds <= 31_536_000: raise ValueError("outbox_ttl_seconds must be between one day and one year") if not 1 <= self.outbox_replay_batch_size <= 1000: @@ -93,11 +210,23 @@ def validate_snapshot(self) -> None: raise ValueError("account_id and regions are required for direct EC2 snapshots") if not self.authorized_account_ids: raise ValueError("authorized_account_ids is required for snapshots") + if self.canary_mode and self.managed_canary is not None: + if self.managed_canary.account_id not in self.authorized_account_ids: + raise ValueError("managed canary account is outside authorized account scope") + if self.managed_canary.region not in self.regions: + raise ValueError("managed canary region is outside snapshot Region scope") + if not self.target_allowed(self.managed_canary.public_ip): + raise ValueError("managed canary address is outside configured CIDR scope") def validate_state(self) -> None: if not self.state_table: raise ValueError("state_table is required") + def validate_signals(self) -> None: + self.validate_state() + if not self.authorized_account_ids: + raise ValueError("authorized_account_ids is required for signals") + def validate_outbox(self) -> None: self.validate_state() if not self.event_bucket: @@ -111,6 +240,20 @@ def validate_outbox(self) -> None: if missing: raise ValueError(f"{', '.join(missing)} required for outbox") + def account_authorized(self, account_id: str) -> bool: + return account_id in self.authorized_account_ids + + def target_allowed(self, address: str) -> bool: + try: + parsed = IPv4Address(address) + except AddressValueError: + return False + if any(parsed in network for network in self.denied_target_cidrs): + return False + return not self.allowed_target_cidrs or any( + parsed in network for network in self.allowed_target_cidrs + ) + @classmethod def from_env(cls, env: Mapping[str, str]) -> Settings: return cls( @@ -126,9 +269,16 @@ def from_env(cls, env: Mapping[str, str]) -> Settings: authorized_account_ids=_csv(env.get("AUTHORIZED_ACCOUNT_IDS")), regions=_csv(env.get("AWS_REGIONS")), canary_mode=_boolean(env, "CANARY_MODE"), + managed_canary=_managed_canary(env), discovery_role_arn_template=env.get("DISCOVERY_ROLE_ARN_TEMPLATE"), external_id=env.get("DISCOVERY_EXTERNAL_ID"), allowed_tag_keys=_csv(env.get("ALLOWED_TAG_KEYS")), + allowed_target_cidrs=_cidrs(env, "ALLOWED_TARGET_CIDRS"), + denied_target_cidrs=_cidrs(env, "DENIED_TARGET_CIDRS"), + allowed_eni_interface_types=_csv(env.get("ALLOWED_ENI_INTERFACE_TYPES")), + required_target_tag_key=env.get("REQUIRED_TARGET_TAG_KEY") or None, + required_target_tag_value=env.get("REQUIRED_TARGET_TAG_VALUE") or None, + snapshot_max_pages=_positive_int(env, "SNAPSHOT_MAX_PAGES", 1000), signal_dedupe_seconds=_positive_int(env, "SIGNAL_DEDUPE_SECONDS", 86_400), outbox_ttl_seconds=_positive_int(env, "OUTBOX_TTL_SECONDS", 604_800), outbox_replay_batch_size=_positive_int(env, "OUTBOX_REPLAY_BATCH_SIZE", 100), diff --git a/inventory/src/portscanner_inventory/events.py b/inventory/src/portscanner_inventory/events.py index edc31fe..6b1eb50 100644 --- a/inventory/src/portscanner_inventory/events.py +++ b/inventory/src/portscanner_inventory/events.py @@ -225,9 +225,14 @@ def build_target_event( not_after = collected_at + COVERAGE_EXECUTION_WINDOW event_type = TargetEventType.RESCAN_REQUESTED elif reason in {ScanReason.NEW_TARGET, ScanReason.MANUAL}: - candidate = CandidatePorts.full() - ranges = (FULL_TCP_PORT_RANGE,) - profile = ScanProfile.FAST_FULL_TCP + managed_targeted = ( + reason is ScanReason.NEW_TARGET + and current.managed_canary + and not current.candidate_ports.full_tcp + ) + candidate = current.candidate_ports if managed_targeted else CandidatePorts.full() + ranges = _ranges(candidate) + profile = ScanProfile.TARGETED_TCP if managed_targeted else ScanProfile.FAST_FULL_TCP priority = 500 if reason is ScanReason.MANUAL else 100 deadline = collected_at + ( MANUAL_DISPATCH_WINDOW if reason is ScanReason.MANUAL else PRIORITY_DISPATCH_WINDOW diff --git a/inventory/src/portscanner_inventory/handlers/outbox.py b/inventory/src/portscanner_inventory/handlers/outbox.py index 2e21c20..61a3125 100644 --- a/inventory/src/portscanner_inventory/handlers/outbox.py +++ b/inventory/src/portscanner_inventory/handlers/outbox.py @@ -27,7 +27,6 @@ _CONFLICT_CODES = {"ConditionalRequestConflict", "PreconditionFailed"} _OUTBOX_INDEX_NAME = "entity-event-index" _DELIVERED_ENTITY = "outbox-delivered" -_LEGACY_DELIVERY_TTL_SECONDS = 604_800 def _attribute(image: Mapping[str, Any], name: str) -> str | None: @@ -46,8 +45,6 @@ def _number_attribute(image: Mapping[str, Any], name: str) -> int | None: def _delivery_retention_seconds(image: Mapping[str, Any]) -> int: - if "delivery_ttl_seconds" not in image: - return _LEGACY_DELIVERY_TTL_SECONDS retention_seconds = _number_attribute(image, "delivery_ttl_seconds") if retention_seconds is None or not 86_400 <= retention_seconds <= 31_536_000: raise ValueError("outbox delivery retention is invalid") @@ -234,9 +231,7 @@ def dispatch_record( if not isinstance(image, Mapping) or _attribute(image, "entity") != "outbox": return False if dynamodb_client is not None: - # Validate delivery-tracking identity before any S3/SQS side effect. Legacy - # pending rows did not carry delivery_ttl_seconds and use the original - # seven-day retention when finalized. + # Validate delivery-tracking identity before any S3/SQS side effect. pk = _attribute(image, "pk") sk = _attribute(image, "sk") if not pk or not sk: diff --git a/inventory/src/portscanner_inventory/handlers/signals.py b/inventory/src/portscanner_inventory/handlers/signals.py index f1dda80..279e367 100644 --- a/inventory/src/portscanner_inventory/handlers/signals.py +++ b/inventory/src/portscanner_inventory/handlers/signals.py @@ -5,7 +5,7 @@ import json import logging import os -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime from typing import Any @@ -60,6 +60,8 @@ def process_signal( dedupe_seconds: int, max_port_ranges: int = 32, max_ports: int = 8_192, + authorized_account_ids: Sequence[str] = (), + target_allowed: Callable[[str], bool] | None = None, ) -> dict[str, Any]: hint = parse_signal( event, @@ -68,14 +70,26 @@ def process_signal( ) if hint is None: return {"status": "ignored", "events": 0} - if not state.claim_signal(hint.event_id, now=now, ttl_seconds=dedupe_seconds): - return {"status": "duplicate", "events": 0} - + if authorized_account_ids and hint.account_id not in authorized_account_ids: + raise ValueError("signal account is outside the configured authorized account scope") + if target_allowed is not None and any( + not target_allowed(address) for address in hint.public_ips + ): + raise ValueError("signal address is outside the configured CIDR scope") + + claimed = False try: - prior_candidates = state.find_signal_candidates(hint) resolution = resolver.resolve(hint) if resolution.status is ResolutionStatus.UNKNOWN: raise SignalResolutionError("EC2 signal resolution is unknown") + if target_allowed is not None and any( + not target_allowed(target.public_ip) for target in resolution.targets + ): + raise ValueError("resolved signal target is outside the configured CIDR scope") + if not state.claim_signal(hint.event_id, now=now, ttl_seconds=dedupe_seconds): + return {"status": "duplicate", "events": 0} + claimed = True + prior_candidates = state.find_signal_candidates(hint) changed = 0 resolved_ids: set[str] = set() @@ -94,6 +108,8 @@ def process_signal( source = _hint_source(hint, now) for candidate in prior_candidates: + if target_allowed is not None and not target_allowed(candidate.target.public_ip): + continue if candidate.target_id in resolved_ids: continue check = ownership.validate(candidate.target_id, candidate.generation) @@ -126,7 +142,8 @@ def process_signal( "targets": len(resolution.targets), } except Exception: - state.release_signal(hint.event_id) + if claimed: + state.release_signal(hint.event_id) raise @@ -134,7 +151,7 @@ class _Runtime: def __init__(self, settings: Settings) -> None: import boto3 - settings.validate_state() + settings.validate_signals() session = boto3.Session() state = DynamoStateStore( session.client("dynamodb"), @@ -152,11 +169,18 @@ def __init__(self, settings: Settings) -> None: self.resolver = Ec2Resolver( factory, allowed_tag_keys=settings.allowed_tag_keys, + allowed_interface_types=settings.allowed_eni_interface_types, + required_tag_key=settings.required_target_tag_key, + required_tag_value=settings.required_target_tag_value, + max_pages=settings.snapshot_max_pages, ) self.ownership = OwnershipValidator( state, factory, allowed_tag_keys=settings.allowed_tag_keys, + allowed_interface_types=settings.allowed_eni_interface_types, + required_tag_key=settings.required_target_tag_key, + required_tag_value=settings.required_target_tag_value, ) def process(self, event: Mapping[str, Any]) -> dict[str, Any]: @@ -169,6 +193,8 @@ def process(self, event: Mapping[str, Any]) -> dict[str, Any]: dedupe_seconds=self.settings.signal_dedupe_seconds, max_port_ranges=self.settings.max_signal_port_ranges, max_ports=self.settings.max_signal_ports, + authorized_account_ids=self.settings.authorized_account_ids, + target_allowed=self.settings.target_allowed, ) diff --git a/inventory/src/portscanner_inventory/handlers/snapshot.py b/inventory/src/portscanner_inventory/handlers/snapshot.py index 2ec2bb4..9642e5c 100644 --- a/inventory/src/portscanner_inventory/handlers/snapshot.py +++ b/inventory/src/portscanner_inventory/handlers/snapshot.py @@ -6,6 +6,7 @@ import logging import os from collections.abc import Callable, Mapping +from dataclasses import replace from datetime import datetime from ipaddress import AddressValueError, IPv4Address from typing import Any @@ -15,13 +16,14 @@ from portscanner_inventory.aws.ownership import OwnershipValidator from portscanner_inventory.aws.session import AwsClientFactory from portscanner_inventory.base import ( + CandidatePorts, OwnershipVerdict, SnapshotScope, partial_batch, structured_log, utc_now, ) -from portscanner_inventory.config import Settings +from portscanner_inventory.config import ManagedCanary, Settings from portscanner_inventory.events import snapshot_source from portscanner_inventory.state import DynamoStateStore, ReconcileAction @@ -36,6 +38,8 @@ def reconcile_snapshot( now: datetime, target_public_ipv4: str | None = None, require_exact_target: bool = False, + target_allowed: Callable[[str], bool] | None = None, + managed_canary: ManagedCanary | None = None, ) -> dict[str, Any]: """Apply observed targets, then remove only directly revalidated absences.""" @@ -49,10 +53,23 @@ def reconcile_snapshot( selected = tuple( target for target in batch.targets - if target_public_ipv4 is None or target.public_ip == target_public_ipv4 + if (target_public_ipv4 is None or target.public_ip == target_public_ipv4) + and (managed_canary is None or managed_canary.matches(target)) + and (target_allowed is None or target_allowed(target.public_ip)) ) if require_exact_target and len(selected) != 1: - raise ValueError("canary snapshot must resolve target_public_ipv4 to exactly one target") + raise ValueError("canary snapshot must resolve the configured target to exactly one target") + if managed_canary is not None: + selected = tuple( + replace( + target, + candidate_ports=CandidatePorts( + ranges=((managed_canary.tcp_port, managed_canary.tcp_port),) + ), + managed_canary=True, + ) + for target in selected + ) observed_ids: set[str] = set() revalidated = 0 @@ -97,6 +114,8 @@ def reconcile_snapshot( if batch.complete and target_public_ipv4 is None: for current in state.list_current(batch.scope): + if target_allowed is not None and not target_allowed(current.target.public_ip): + continue if current.target_id in observed_ids: continue revalidated += 1 @@ -157,14 +176,12 @@ def _requested_account_id(request: Mapping[str, Any], settings: Settings) -> str def _requested_region( request: Mapping[str, Any], settings: Settings, - *, - field: str = "region", ) -> str | None: - value = request.get(field) + value = request.get("region") if value is None: return None if not isinstance(value, str) or value not in settings.regions: - raise ValueError(f"{field} is outside the configured snapshot Region scope") + raise ValueError("region is outside the configured snapshot Region scope") return value @@ -190,44 +207,67 @@ def __init__(self, settings: Settings) -> None: self.state, self.factory, allowed_tag_keys=settings.allowed_tag_keys, + allowed_interface_types=settings.allowed_eni_interface_types, + required_tag_key=settings.required_target_tag_key, + required_tag_value=settings.required_target_tag_value, ) def process(self, request: Mapping[str, Any]) -> list[dict[str, Any]]: now = utc_now() - target_public_ipv4 = _requested_target_public_ipv4(request) - account_id = _requested_account_id(request, self.settings) - requested_region = _requested_region(request, self.settings) - if self.settings.canary_mode and ( - target_public_ipv4 is None or "account_id" not in request or requested_region is None - ): - raise ValueError("canary snapshot requires account_id, region, and target_public_ipv4") + operation = request.get("operation") + managed_canary: ManagedCanary | None = None + target_public_ipv4: str | None + account_id: str + requested_region: str | None + if operation == "managed-canary": + if not self.settings.canary_mode or self.settings.managed_canary is None: + raise ValueError("managed-canary operation is not enabled") + forbidden = {"account_id", "region", "target_public_ipv4"} + if forbidden.intersection(request): + raise ValueError("managed-canary operation does not accept target overrides") + managed_canary = self.settings.managed_canary + target_public_ipv4 = managed_canary.public_ip + account_id = managed_canary.account_id + requested_region = managed_canary.region + else: + if operation is not None: + raise ValueError("unsupported snapshot operation") + if self.settings.canary_mode and self.settings.managed_canary is not None: + raise ValueError("managed canary snapshot requires operation=managed-canary") + target_public_ipv4 = _requested_target_public_ipv4(request) + account_id = _requested_account_id(request, self.settings) + requested_region = _requested_region(request, self.settings) + if self.settings.canary_mode and ( + target_public_ipv4 is None + or "account_id" not in request + or requested_region is None + ): + raise ValueError( + "canary snapshot requires account_id, region, and target_public_ipv4" + ) + if not self.settings.account_authorized(account_id): + raise ValueError("account_id is outside the configured authorized account scope") + if target_public_ipv4 is not None and not self.settings.target_allowed(target_public_ipv4): + raise ValueError("target_public_ipv4 is outside the configured CIDR scope") backend: ConfigSnapshotBackend | Ec2SnapshotBackend if self.settings.snapshot_backend == "config": client_region = os.environ.get("AWS_REGION") or "us-east-1" - legacy_target_region = _requested_region( - request, - self.settings, - field="target_region", - ) - if ( - requested_region is not None - and legacy_target_region is not None - and requested_region != legacy_target_region - ): - raise ValueError("region and target_region must identify the same target Region") - target_region = requested_region or legacy_target_region client = self.session.client("config", region_name=client_region) scope = SnapshotScope( source="aws-config", name=self.settings.config_aggregator_name, account_id=account_id, - region=target_region, + region=requested_region, ) backend = ConfigSnapshotBackend( client, aggregator_name=self.settings.config_aggregator_name or "", scope=scope, allowed_tag_keys=self.settings.allowed_tag_keys, + allowed_interface_types=self.settings.allowed_eni_interface_types, + required_tag_key=self.settings.required_target_tag_key, + required_tag_value=self.settings.required_target_tag_value, + max_pages=self.settings.snapshot_max_pages, ) return [ reconcile_snapshot( @@ -237,6 +277,8 @@ def process(self, request: Mapping[str, Any]) -> list[dict[str, Any]]: now=now, target_public_ipv4=target_public_ipv4, require_exact_target=self.settings.canary_mode, + target_allowed=self.settings.target_allowed, + managed_canary=managed_canary, ) ] @@ -249,6 +291,10 @@ def process(self, request: Mapping[str, Any]) -> list[dict[str, Any]]: account_id=account_id, region=region, allowed_tag_keys=self.settings.allowed_tag_keys, + allowed_interface_types=self.settings.allowed_eni_interface_types, + required_tag_key=self.settings.required_target_tag_key, + required_tag_value=self.settings.required_target_tag_value, + max_pages=self.settings.snapshot_max_pages, ) summaries.append( reconcile_snapshot( @@ -258,6 +304,8 @@ def process(self, request: Mapping[str, Any]) -> list[dict[str, Any]]: now=now, target_public_ipv4=target_public_ipv4, require_exact_target=self.settings.canary_mode, + target_allowed=self.settings.target_allowed, + managed_canary=managed_canary, ) ) return summaries diff --git a/inventory/src/portscanner_inventory/state.py b/inventory/src/portscanner_inventory/state.py index d3a8a30..e6710f6 100644 --- a/inventory/src/portscanner_inventory/state.py +++ b/inventory/src/portscanner_inventory/state.py @@ -138,8 +138,7 @@ def _config_observation_order( return _ObservationOrder.NOT_NEWER -def _observation_version(target: NormalizedTarget, fallback: datetime) -> str: - del fallback +def _observation_version(target: NormalizedTarget) -> str: eni_version, group_versions = _component_versions(target) return deterministic_sha256( "portscanner.inventory.observation-version.v1", @@ -231,7 +230,7 @@ def _state_item( "public_ip": _s(state.target.public_ip), "contract_target_id": _s(state.contract_target_id or ""), "observed_at": _s(_timestamp(observed_at)), - "observation_version": _s(_observation_version(state.target, observed_at)), + "observation_version": _s(_observation_version(state.target)), "updated_at": _s(_timestamp(updated_at)), } @@ -268,22 +267,19 @@ def _decode_state(self, item: Mapping[str, Any]) -> TargetState: ): raise ValueError("invalid target state item") target = NormalizedTarget.from_state_dict(json.loads(target_json)) - raw_observed_at = ( - _attribute(item, "observed_at") - or (_timestamp(target.observed_at) if target.observed_at is not None else None) - or _attribute(item, "updated_at") - ) + raw_observed_at = _attribute(item, "observed_at") + signature = _attribute(item, "signature") if raw_observed_at is None: raise ValueError("target state item has no observation time") + if signature is None: + raise ValueError("target state item has no signature") observed_at = _parse_timestamp(raw_observed_at) - if target.observed_at is None: - target = target.with_observation(observed_at) return TargetState( target_id=pk.removeprefix("TARGET#"), generation=int(generation), status=status, target=target, - signature=_attribute(item, "signature") or target.state_signature, + signature=signature, contract_target_id=_attribute(item, "contract_target_id") or None, observed_at=observed_at, ) @@ -369,10 +365,8 @@ def _write_state_and_outbox( raise ValueError("previous target state has no observation time") condition = ( "#generation = :expected AND #status = :status " - "AND (attribute_not_exists(#observed_at) " - "OR #observed_at = :expected_observed_at) " - "AND (attribute_not_exists(#observation_version) " - "OR #observation_version = :expected_observation_version)" + "AND #observed_at = :expected_observed_at " + "AND #observation_version = :expected_observation_version" ) names = { "#generation": "generation", @@ -384,9 +378,7 @@ def _write_state_and_outbox( ":expected": _n(previous.generation), ":status": _s(previous.status), ":expected_observed_at": _s(_timestamp(previous.observed_at)), - ":expected_observation_version": _s( - _observation_version(previous.target, previous.observed_at) - ), + ":expected_observation_version": _s(_observation_version(previous.target)), } put_state: dict[str, Any] = { "TableName": self._table_name, @@ -424,10 +416,8 @@ def _write_state_only( "Item": self._state_item(state, updated_at=now), "ConditionExpression": ( "#generation = :expected AND #status = :status " - "AND (attribute_not_exists(#observed_at) " - "OR #observed_at = :expected_observed_at) " - "AND (attribute_not_exists(#observation_version) " - "OR #observation_version = :expected_observation_version)" + "AND #observed_at = :expected_observed_at " + "AND #observation_version = :expected_observation_version" ), "ExpressionAttributeNames": { "#generation": "generation", @@ -440,10 +430,7 @@ def _write_state_only( ":status": _s(previous.status), ":expected_observed_at": _s(_timestamp(previous.observed_at)), ":expected_observation_version": _s( - _observation_version( - previous.target, - previous.observed_at, - ) + _observation_version(previous.target) ), }, } diff --git a/inventory/tests/helpers.py b/inventory/tests/helpers.py index f2651fd..6d8fc62 100644 --- a/inventory/tests/helpers.py +++ b/inventory/tests/helpers.py @@ -59,6 +59,7 @@ def eni( group_ids: tuple[str, ...] = (SG_ID,), status: str = "in-use", instance_id: str = "i-bbbbbbbb", + interface_type: str = "interface", tags: list[dict[str, str]] | None = None, secondary: list[tuple[str, str]] | None = None, ) -> dict[str, Any]: @@ -88,6 +89,7 @@ def eni( "PrivateIpAddress": private_ip, "PrivateIpAddresses": private_values, "Status": status, + "InterfaceType": interface_type, "Attachment": { "AttachmentId": "eni-attach-eeeeeeee", "Status": "attached", diff --git a/inventory/tests/test_handlers.py b/inventory/tests/test_handlers.py index 91647fd..e74b420 100644 --- a/inventory/tests/test_handlers.py +++ b/inventory/tests/test_handlers.py @@ -2,7 +2,9 @@ import hashlib import json +from dataclasses import replace from datetime import UTC, datetime, timedelta +from ipaddress import IPv4Address, IPv4Network from typing import Any from urllib.parse import unquote_plus @@ -11,7 +13,7 @@ from portscanner_inventory.aws.signals import parse_signal from portscanner_inventory.base import CandidatePorts, Resolution, SnapshotScope -from portscanner_inventory.config import Settings +from portscanner_inventory.config import ManagedCanary, Settings from portscanner_inventory.events import ( build_removal_event, build_target_event, @@ -97,6 +99,27 @@ def validate(self, *_args): assert duplicate == {"status": "duplicate", "events": 0} +def test_signal_rejects_unauthorized_account_before_api_or_state_side_effects() -> None: + class State: + def claim_signal(self, *_args, **_kwargs): + raise AssertionError("unauthorized signal mutated state") + + class Resolver: + def resolve(self, _hint): + raise AssertionError("unauthorized signal reached EC2") + + with pytest.raises(ValueError, match="authorized account scope"): + process_signal( + _signal_event(), + State(), + Resolver(), + object(), + now=NOW, + dedupe_seconds=60, + authorized_account_ids=("222222222222",), + ) + + def test_signal_lambda_returns_partial_sqs_batch_failures() -> None: records = [ { @@ -220,6 +243,37 @@ def _removal_event() -> TargetRemoval: ) +def test_only_trusted_managed_new_target_uses_targeted_tcp() -> None: + scope = SnapshotScope(source="ec2", account_id=ACCOUNT_ID, region=REGION) + managed = replace( + normalized_target(tags=[{"Key": "service", "Value": "managed-canary"}]), + candidate_ports=CandidatePorts(ranges=((18080, 18080),)), + managed_canary=True, + ) + managed_event = build_target_event( + managed, + 1, + source=snapshot_source(scope, NOW), + collected_at=NOW, + ) + ordinary_event = build_target_event( + replace(managed, managed_canary=False), + 1, + source=snapshot_source(scope, NOW), + collected_at=NOW, + ) + + assert managed_event.scan.profile.value == "targeted-tcp" + assert [(item.start, item.end) for item in managed_event.scan.tcp_port_ranges] == [ + (18080, 18080) + ] + assert managed_event.aws_context.candidate_tcp_port_ranges == ( + managed_event.scan.tcp_port_ranges + ) + assert ordinary_event.scan.profile.value == "fast-full-tcp" + assert [(item.start, item.end) for item in ordinary_event.scan.tcp_port_ranges] == [(1, 65535)] + + def _dispatch(record: dict[str, Any], s3: S3, sqs: SQS) -> bool: return dispatch_record( record, @@ -346,43 +400,6 @@ def update_item(self, **request: Any) -> dict[str, Any]: assert int(update["ExpressionAttributeValues"][":expires"]["N"]) > int(NOW.timestamp()) -def test_outbox_legacy_pending_row_uses_safe_retention_before_sending() -> None: - event = _work_event(ScanReason.NEW_TARGET) - record = _outbox_record(event) - image = record["dynamodb"]["NewImage"] - del image["delivery_ttl_seconds"] - image["expires_at"] = {"N": str(int(NOW.timestamp()) + 60)} - - class Dynamo: - def __init__(self) -> None: - self.updates: list[dict[str, Any]] = [] - - def update_item(self, **request: Any) -> dict[str, Any]: - self.updates.append(request) - return {} - - dynamodb = Dynamo() - s3 = S3() - sqs = SQS() - assert dispatch_record( - record, - s3, - sqs, - bucket="events", - priority_queue_url=PRIORITY_QUEUE_URL, - coverage_queue_url=COVERAGE_QUEUE_URL, - target_event_queue_url=TARGET_EVENT_QUEUE_URL, - dynamodb_client=dynamodb, - table_name="inventory", - delivered_at=NOW, - ) - - assert len(sqs.requests) == 2 - assert dynamodb.updates[0]["ExpressionAttributeValues"][":expires"] == { - "N": str(int(NOW.timestamp()) + 604_800) - } - - def test_outbox_invalid_tracking_fails_before_s3_or_sqs_side_effects() -> None: event = _work_event(ScanReason.NEW_TARGET) record = _outbox_record(event) @@ -527,6 +544,27 @@ def test_snapshot_settings_parse_and_validate_runtime_boundaries() -> None: ) member_only.validate_snapshot() + managed_outside_normal_scope = Settings( + state_table="inventory", + event_bucket="events", + snapshot_backend="ec2", + account_id="123456789012", + authorized_account_ids=("123456789012",), + regions=("us-east-1",), + managed_canary=ManagedCanary( + account_id="123456789012", + region="us-east-1", + network_interface_id="eni-aaaaaaaa", + private_ip="10.0.0.10", + public_ip=str(IPv4Address(0x08080808)), + tag_key="service", + tag_value="managed-canary", + tcp_port=18080, + ), + allowed_target_cidrs=(IPv4Network("198.51.100.1/32"),), + ) + managed_outside_normal_scope.validate_snapshot() + with pytest.raises(ValueError, match="must be true or false"): Settings.from_env( { diff --git a/inventory/tests/test_snapshot_handler.py b/inventory/tests/test_snapshot_handler.py index ba509e3..760d8d1 100644 --- a/inventory/tests/test_snapshot_handler.py +++ b/inventory/tests/test_snapshot_handler.py @@ -5,6 +5,7 @@ from ipaddress import IPv4Address import pytest +from portscanner_contracts import parse_target_event from portscanner_inventory.base import ( OwnershipCheck, @@ -13,7 +14,7 @@ SnapshotBatch, SnapshotScope, ) -from portscanner_inventory.config import Settings +from portscanner_inventory.config import ManagedCanary, Settings from portscanner_inventory.events import snapshot_source from portscanner_inventory.handlers.snapshot import ( _requested_account_id, @@ -89,6 +90,22 @@ def test_snapshot_request_must_remain_inside_configured_account_and_regions() -> _requested_region({"region": "us-west-2"}, settings) +def test_unauthorized_snapshot_account_is_rejected_before_aws_or_state_access() -> None: + settings = Settings( + state_table="inventory", + event_bucket="events", + snapshot_backend="ec2", + account_id="123456789012", + authorized_account_ids=("123456789012",), + regions=("us-east-1",), + ) + runtime = _Runtime.__new__(_Runtime) + runtime.settings = settings + + with pytest.raises(ValueError, match="authorized account scope"): + runtime.process({"account_id": "222222222222"}) + + def test_canary_rejects_present_but_null_region_before_backend_access() -> None: settings = Settings( state_table="inventory", @@ -112,6 +129,116 @@ def test_canary_rejects_present_but_null_region_before_backend_access() -> None: ) +def test_managed_canary_rejects_caller_target_overrides_before_backend_access() -> None: + canary = ManagedCanary( + account_id="123456789012", + region="us-east-1", + network_interface_id="eni-aaaaaaaa", + private_ip="10.0.0.10", + public_ip=_global_ipv4(), + tag_key="service", + tag_value="managed-canary", + tcp_port=18080, + ) + settings = Settings( + state_table="inventory", + event_bucket="events", + snapshot_backend="ec2", + account_id="123456789012", + authorized_account_ids=("123456789012",), + regions=("us-east-1",), + canary_mode=True, + managed_canary=canary, + allowed_target_cidrs=(), + ) + runtime = _Runtime.__new__(_Runtime) + runtime.settings = settings + + with pytest.raises(ValueError, match="does not accept target overrides"): + runtime.process( + { + "operation": "managed-canary", + "target_public_ipv4": "198.51.100.1", + } + ) + + +def test_managed_canary_snapshot_writes_only_exact_targeted_tcp_event() -> None: + public_ip = _global_ipv4() + canary = ManagedCanary( + account_id="123456789012", + region="us-east-1", + network_interface_id="eni-aaaaaaaa", + private_ip="10.0.0.10", + public_ip=public_ip, + tag_key="service", + tag_value="managed-canary", + tcp_port=18080, + ) + target = normalized_target( + public_ip=public_ip, + tags=[{"Key": "service", "Value": "managed-canary"}], + ) + outside = replace( + normalized_target(public_ip="203.0.113.30"), + network_interface_id="eni-bbbbbbbb", + private_ip="10.0.1.10", + ) + dynamo = FakeDynamo() + store = DynamoStateStore(dynamo, "inventory") + + summary = reconcile_snapshot( + Backend( + SnapshotBatch( + scope=SCOPE, + targets=(target, outside), + completion=ScopeCompletion.COMPLETE, + pages=1, + ) + ), + store, + Ownership(OwnershipVerdict.ACTIVE), + now=NOW, + target_public_ipv4=public_ip, + require_exact_target=True, + target_allowed=lambda address: address == public_ip, + managed_canary=canary, + ) + + outbox = next(item for (pk, _sk), item in dynamo.items.items() if pk.startswith("OUTBOX#")) + event = parse_target_event(outbox["event_json"]["S"]) + assert summary["targets"] == 1 + assert event.scan.reason.value == "new_target" + assert event.scan.profile.value == "targeted-tcp" + assert [(item.start, item.end) for item in event.scan.tcp_port_ranges] == [(18080, 18080)] + assert store.get(outside.target_id) is None + + +def test_snapshot_cidr_scope_filters_before_state_mutation() -> None: + dynamo = FakeDynamo() + store = DynamoStateStore(dynamo, "inventory") + target = normalized_target(public_ip=_global_ipv4()) + + summary = reconcile_snapshot( + Backend( + SnapshotBatch( + scope=SCOPE, + targets=(target,), + completion=ScopeCompletion.COMPLETE, + pages=1, + ) + ), + store, + Ownership(OwnershipVerdict.ACTIVE), + now=NOW, + target_allowed=lambda _address: False, + ) + + assert summary["targets"] == 0 + assert dynamo.transactions == 0 + assert store.list_current(SCOPE) == () + + def test_target_scoped_snapshot_reconciles_only_canary_without_removals() -> None: store = DynamoStateStore(FakeDynamo(), "inventory") old = _seed(store) diff --git a/inventory/tests/test_snapshots.py b/inventory/tests/test_snapshots.py index 89c21ec..ca4ea5b 100644 --- a/inventory/tests/test_snapshots.py +++ b/inventory/tests/test_snapshots.py @@ -9,6 +9,7 @@ ConfigSnapshotBackend, ) from portscanner_inventory.aws.ec2_snapshot import Ec2SnapshotBackend +from portscanner_inventory.aws.normalize import normalize_network_interface from portscanner_inventory.base import ScopeCompletion, SnapshotScope from .helpers import ACCOUNT_ID, ENI_ID, REGION, AwsError, eni, security_group @@ -250,3 +251,96 @@ def test_scope_completion_is_authoritative_only_when_explicit() -> None: scope = SnapshotScope(source="ec2", account_id=ACCOUNT_ID, region=REGION) assert scope.includes(ACCOUNT_ID, REGION) assert not scope.includes(ACCOUNT_ID, "us-west-2") + + +def test_snapshot_pagination_rejects_repeated_tokens_and_page_limits() -> None: + class RepeatedEc2: + def describe_security_groups(self, **_request: Any) -> dict[str, Any]: + return {"SecurityGroups": [], "NextToken": "repeat"} + + ec2_fetch = Ec2SnapshotBackend( + RepeatedEc2(), + account_id=ACCOUNT_ID, + region=REGION, + )._fetch("describe_security_groups", "SecurityGroups") + assert ec2_fetch.failure_code == "pagination-token-repeated" + assert ec2_fetch.pages == 2 + + class EndlessConfig: + def select_aggregate_resource_config(self, **request: Any) -> dict[str, Any]: + token = request.get("NextToken") + return {"Results": [], "NextToken": "next-2" if token else "next-1"} + + config_fetch = ConfigSnapshotBackend( + EndlessConfig(), + aggregator_name="example-aggregator", + max_pages=2, + ).fetch_records(NETWORK_INTERFACE_QUERY) + assert config_fetch.failure_code == "pagination-page-limit" + assert config_fetch.pages == 2 + + class RepeatedConfig: + def select_aggregate_resource_config(self, **_request: Any) -> dict[str, Any]: + return {"Results": [], "NextToken": "repeat"} + + repeated_config_fetch = ConfigSnapshotBackend( + RepeatedConfig(), + aggregator_name="example-aggregator", + ).fetch_records(NETWORK_INTERFACE_QUERY) + assert repeated_config_fetch.failure_code == "pagination-token-repeated" + assert repeated_config_fetch.pages == 2 + + +def test_eni_class_and_opt_in_gate_is_configurable_without_breaking_disabled_mode() -> None: + interface = eni( + interface_type="interface", + tags=[{"Key": "service", "Value": "managed-canary"}], + ) + groups = {"sg-11111111": security_group()["IpPermissions"]} + + assert ( + len( + normalize_network_interface( + interface, + account_id=ACCOUNT_ID, + region=REGION, + security_groups=groups, + ) + ) + == 1 + ) + assert ( + len( + normalize_network_interface( + interface, + account_id=ACCOUNT_ID, + region=REGION, + security_groups=groups, + allowed_interface_types=("interface",), + required_tag_key="service", + required_tag_value="managed-canary", + ) + ) + == 1 + ) + assert ( + normalize_network_interface( + interface, + account_id=ACCOUNT_ID, + region=REGION, + security_groups=groups, + allowed_interface_types=("efa",), + ) + == () + ) + assert ( + normalize_network_interface( + interface, + account_id=ACCOUNT_ID, + region=REGION, + security_groups=groups, + required_tag_key="service", + required_tag_value="different", + ) + == () + ) diff --git a/inventory/tests/test_state.py b/inventory/tests/test_state.py index 25ac3ca..ab11f34 100644 --- a/inventory/tests/test_state.py +++ b/inventory/tests/test_state.py @@ -479,36 +479,6 @@ def test_newer_idempotent_observation_advances_state_watermark() -> None: assert store.get(target.target_id).observed_at == observed_at -def test_legacy_state_without_observation_is_conditionally_upgraded() -> None: - client = FakeDynamo() - store = DynamoStateStore(client, "inventory") - target = normalized_target() - added = store.reconcile(target, source=_source(), now=NOW) - assert added.state is not None - key = (f"TARGET#{target.target_id}", "STATE") - legacy = client.items[key] - legacy.pop("observed_at") - target_json = json.loads(legacy["target_json"]["S"]) - target_json.pop("observed_at") - legacy["target_json"]["S"] = json.dumps( - target_json, - separators=(",", ":"), - sort_keys=True, - ) - - changed_at = NOW + timedelta(minutes=1) - result = store.reconcile( - normalized_target(public_ip="203.0.113.20"), - source=_source(changed_at), - now=changed_at, - ) - - assert result.action is ReconcileAction.CHANGED - assert result.state is not None - assert result.state.observed_at == changed_at - assert client.items[key]["observed_at"]["S"] == "2026-01-02T03:05:05Z" - - def test_signal_dedupe_uses_ttl_and_can_be_released() -> None: client = FakeDynamo() store = DynamoStateStore(client, "inventory") diff --git a/operator/.dockerignore b/operator/.dockerignore index 6453125..90f9dc4 100644 --- a/operator/.dockerignore +++ b/operator/.dockerignore @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT .git .github diff --git a/operator/.gitignore b/operator/.gitignore index eba385c..7999506 100644 --- a/operator/.gitignore +++ b/operator/.gitignore @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT /bin/ /cover.out diff --git a/operator/Dockerfile b/operator/Dockerfile index 650e533..f7d80e1 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT # syntax=docker/dockerfile:1.7 @@ -21,9 +21,9 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ FROM scratch LABEL org.opencontainers.image.source="https://github.com/Roblox/portscanner" \ - org.opencontainers.image.licenses="Apache-2.0" + org.opencontainers.image.licenses="MIT" COPY --from=builder --chmod=0555 /out/manager /manager COPY --from=builder /workspace/LICENSE /licenses/LICENSE -COPY --from=builder /workspace/NOTICE /licenses/NOTICE +COPY --from=builder /workspace/THIRD_PARTY_NOTICES.md /licenses/THIRD_PARTY_NOTICES.md USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/operator/LICENSE b/operator/LICENSE index 03e56be..f5aec1c 100644 --- a/operator/LICENSE +++ b/operator/LICENSE @@ -1,201 +1,21 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2026 Roblox Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +MIT License + +Copyright (c) 2026 Roblox + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/operator/Makefile b/operator/Makefile index 8fc5321..dc86f29 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -1,10 +1,9 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT SHELL := /usr/bin/env bash GO ?= go -KUBECTL ?= kubectl HELM ?= helm IMG ?= portscanner-operator:latest LOCALBIN ?= $(CURDIR)/bin @@ -48,11 +47,9 @@ generate: $(CONTROLLER_GEN) .PHONY: manifests manifests: $(CONTROLLER_GEN) $(CONTROLLER_GEN) \ - rbac:roleName=portscanner-operator,headerFile=hack/boilerplate.yaml.txt \ crd:headerFile=hack/boilerplate.yaml.txt \ paths="./..." \ - output:crd:artifacts:config=config/crd/bases \ - output:rbac:artifacts:config=config/rbac + output:crd:artifacts:config=config/crd/bases $(CONTROLLER_GEN) \ crd:headerFile=hack/boilerplate.yaml.txt \ paths="./..." \ @@ -60,32 +57,15 @@ manifests: $(CONTROLLER_GEN) .PHONY: verify-manifests verify-manifests: - $(KUBECTL) kustomize config/default >/dev/null $(HELM) lint chart/portscanner $(HELM) template portscanner chart/portscanner \ - --namespace portscanner-system \ + --namespace portscanner-system --include-crds \ --kube-version 1.36.0 >/dev/null .PHONY: docker-build docker-build: docker build --tag $(IMG) . -.PHONY: install -install: - $(KUBECTL) apply -f config/crd/bases/scanning.portscanner.io_scanners.yaml - -.PHONY: uninstall -uninstall: - $(KUBECTL) delete -f config/crd/bases/scanning.portscanner.io_scanners.yaml --ignore-not-found - -.PHONY: deploy -deploy: - $(KUBECTL) apply -k config/default - -.PHONY: undeploy -undeploy: - $(KUBECTL) delete -k config/default --ignore-not-found - $(CONTROLLER_GEN): mkdir -p $(LOCALBIN) GOBIN=$(LOCALBIN) $(GO) install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION) diff --git a/operator/NOTICE b/operator/NOTICE deleted file mode 100644 index 3420036..0000000 --- a/operator/NOTICE +++ /dev/null @@ -1,4 +0,0 @@ -Portscanner Kubernetes Operator -Copyright 2026 Roblox Corporation - -This product is licensed under the Apache License, Version 2.0. diff --git a/operator/PROJECT b/operator/PROJECT index 3b13a97..b2c2488 100644 --- a/operator/PROJECT +++ b/operator/PROJECT @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT domain: portscanner.io layout: diff --git a/operator/README.md b/operator/README.md index 269ca21..ba1a45a 100644 --- a/operator/README.md +++ b/operator/README.md @@ -1,11 +1,11 @@ # Portscanner Kubernetes Operator -This directory is the public, Apache-2.0-licensed Kubernetes operator boundary +This directory is the public, MIT-licensed Kubernetes operator boundary for `github.com/Roblox/portscanner/operator`. It defines the `scanning.portscanner.io/v1alpha1` `Scanner` API and creates one bounded Kubernetes Job for each accepted Scanner resource. @@ -46,8 +46,6 @@ The configured scanner image receives these arguments: - `--target=
` - `--profile=fast-full-tcp|targeted-tcp|deep` - `--tcp-ports=` -- `--scan-mode=fast|targeted|deep` -- `--service-detection` for the `deep` profile `fast-full-tcp` always uses `1-65535`. `targeted-tcp` requires ports or ranges. `deep` uses supplied coverage or explicitly defaults to `1-65535`. Explicit @@ -78,19 +76,10 @@ external destination and exit zero on success or non-zero on failure. ## Install -Build and deploy the Kustomize base: - -```sh -make docker-build IMG=portscanner-operator:your-tag -# Set images and environment-specific arguments in a Kustomize overlay. -kubectl apply -k config/default -``` - -The base's `portscanner-system` namespace is an overridable Kustomize default; -the controller itself never assumes an operator or Scanner namespace. Scanner -Jobs are created in the Scanner resource's namespace. - -Install with Helm: +Helm is the only supported installation surface. The canonical AWS deployment +installs this chart after its database migration succeeds. For standalone +development, supply immutable operator/scanner images and an existing result +bucket: ```sh helm install portscanner ./chart/portscanner \ @@ -121,6 +110,10 @@ single namespace explicitly. The all-zero operator/scanner digests and placehold result bucket in the default values are render-only placeholders and must be replaced before deployment. +The controller itself never assumes a fixed namespace. Scanner Jobs are +created in the Helm release namespace selected by Terraform or the standalone +Helm command. + ## Security Scanner Pods run non-root with RuntimeDefault seccomp, no privilege escalation, diff --git a/operator/THIRD_PARTY_NOTICES.md b/operator/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..72ebf48 --- /dev/null +++ b/operator/THIRD_PARTY_NOTICES.md @@ -0,0 +1,75 @@ + + +# Third-party notices + +This repository depends on or interoperates with the projects listed below. +The links point to upstream licensing material; this inventory does not make an +independent license determination. Before distributing an image or binary, +verify the exact resolved versions and preserve all notices required by those +upstream projects. + +## Scanner runtime + +### Nmap + +The scanner boundary invokes the separately packaged Nmap executable. + +- Project: +- Upstream legal and licensing information: + +- Source distributions: + +Nmap has project-specific licensing terms. Consult the terms shipped with the +exact Nmap release and distribution package rather than inferring them from +this notice. + +## Python runtime and libraries + +### Python + +- Project: +- Upstream licensing information: + +### Pydantic and pydantic-core + +- Projects: and + +- Upstream licensing files: + and + + +### Boto3 and Botocore + +- Projects: and + +- Upstream licensing files: + and + + +### Kubernetes Python client + +- Project: +- Upstream licensing file: + + +### defusedxml + +- Project: +- Upstream licensing file: + + +### Psycopg + +- Project: +- Upstream licensing information: + + +## Build, test, and repository tooling + +Development and build environments may also resolve uv, Hatchling, setuptools, +pytest, Ruff, mypy, jsonschema, and REUSE. Their authoritative notices and +license files are distributed by the corresponding upstream projects and +package artifacts. They are not necessarily included in runtime images. diff --git a/operator/api/v1alpha1/groupversion_info.go b/operator/api/v1alpha1/groupversion_info.go index ba5f6a4..9ff52d3 100644 --- a/operator/api/v1alpha1/groupversion_info.go +++ b/operator/api/v1alpha1/groupversion_info.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT // Package v1alpha1 contains the public Scanner API. // +kubebuilder:object:generate=true diff --git a/operator/api/v1alpha1/scanner_types.go b/operator/api/v1alpha1/scanner_types.go index 62c2503..e5ae5dd 100644 --- a/operator/api/v1alpha1/scanner_types.go +++ b/operator/api/v1alpha1/scanner_types.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package v1alpha1 diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 8560ae3..c520017 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT // Code generated by controller-gen. DO NOT EDIT. diff --git a/operator/chart/portscanner/Chart.yaml b/operator/chart/portscanner/Chart.yaml index bc91347..3c090eb 100644 --- a/operator/chart/portscanner/Chart.yaml +++ b/operator/chart/portscanner/Chart.yaml @@ -1,15 +1,15 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT apiVersion: v2 name: portscanner description: A public one-shot Kubernetes port scanner operator. type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 1.0.0 +appVersion: "1.0.0" kubeVersion: ">=1.35.0-0 <1.37.0-0" home: https://github.com/Roblox/portscanner sources: - https://github.com/Roblox/portscanner annotations: - licenses: Apache-2.0 + licenses: MIT diff --git a/operator/chart/portscanner/crds/scanning.portscanner.io_scanners.yaml b/operator/chart/portscanner/crds/scanning.portscanner.io_scanners.yaml index cd4f64d..d5470b6 100644 --- a/operator/chart/portscanner/crds/scanning.portscanner.io_scanners.yaml +++ b/operator/chart/portscanner/crds/scanning.portscanner.io_scanners.yaml @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/operator/chart/portscanner/templates/_helpers.tpl b/operator/chart/portscanner/templates/_helpers.tpl index 28652a4..ad5494e 100644 --- a/operator/chart/portscanner/templates/_helpers.tpl +++ b/operator/chart/portscanner/templates/_helpers.tpl @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- define "portscanner.name" -}} diff --git a/operator/chart/portscanner/templates/deployment.yaml b/operator/chart/portscanner/templates/deployment.yaml index 0ae58bb..eaae181 100644 --- a/operator/chart/portscanner/templates/deployment.yaml +++ b/operator/chart/portscanner/templates/deployment.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- $scannerNodeSelector := list -}} {{- range $key, $value := .Values.scanner.nodeSelector -}} diff --git a/operator/chart/portscanner/templates/generator-rbac.yaml b/operator/chart/portscanner/templates/generator-rbac.yaml index be01dba..4114ca8 100644 --- a/operator/chart/portscanner/templates/generator-rbac.yaml +++ b/operator/chart/portscanner/templates/generator-rbac.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- if .Values.generator.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/operator/chart/portscanner/templates/operator-rbac.yaml b/operator/chart/portscanner/templates/operator-rbac.yaml index 679570d..6671d57 100644 --- a/operator/chart/portscanner/templates/operator-rbac.yaml +++ b/operator/chart/portscanner/templates/operator-rbac.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- if .Values.operator.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/operator/chart/portscanner/templates/operator-serviceaccount.yaml b/operator/chart/portscanner/templates/operator-serviceaccount.yaml index 2d3b56e..1cb22f9 100644 --- a/operator/chart/portscanner/templates/operator-serviceaccount.yaml +++ b/operator/chart/portscanner/templates/operator-serviceaccount.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- if .Values.operator.serviceAccount.create }} apiVersion: v1 diff --git a/operator/chart/portscanner/templates/priorityclasses.yaml b/operator/chart/portscanner/templates/priorityclasses.yaml index 6b5530c..72d7d8b 100644 --- a/operator/chart/portscanner/templates/priorityclasses.yaml +++ b/operator/chart/portscanner/templates/priorityclasses.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- if .Values.priorityClasses.create }} apiVersion: scheduling.k8s.io/v1 diff --git a/operator/chart/portscanner/templates/scanner-quota.yaml b/operator/chart/portscanner/templates/scanner-quota.yaml index e00d340..02f8dde 100644 --- a/operator/chart/portscanner/templates/scanner-quota.yaml +++ b/operator/chart/portscanner/templates/scanner-quota.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- $root := . -}} {{- if or (lt (int .Values.scanner.maxConcurrentPods) 1) (gt (int .Values.scanner.maxConcurrentPods) 100) -}} diff --git a/operator/chart/portscanner/templates/scanner-rbac.yaml b/operator/chart/portscanner/templates/scanner-rbac.yaml index a1630b4..e930547 100644 --- a/operator/chart/portscanner/templates/scanner-rbac.yaml +++ b/operator/chart/portscanner/templates/scanner-rbac.yaml @@ -1,6 +1,6 @@ {{/* -Copyright 2026 Roblox Corporation -SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT */}} {{- $root := . -}} {{- if $root.Values.scanner.serviceAccount.create }} diff --git a/operator/chart/portscanner/values.yaml b/operator/chart/portscanner/values.yaml index 7c7e2d7..03b4835 100644 --- a/operator/chart/portscanner/values.yaml +++ b/operator/chart/portscanner/values.yaml @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT nameOverride: "" fullnameOverride: "" diff --git a/operator/cmd/manager/main.go b/operator/cmd/manager/main.go index f50bd9e..4d0e3d1 100644 --- a/operator/cmd/manager/main.go +++ b/operator/cmd/manager/main.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package main diff --git a/operator/cmd/manager/main_test.go b/operator/cmd/manager/main_test.go index d38c2a1..75d44e3 100644 --- a/operator/cmd/manager/main_test.go +++ b/operator/cmd/manager/main_test.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package main diff --git a/operator/config/crd/bases/scanning.portscanner.io_scanners.yaml b/operator/config/crd/bases/scanning.portscanner.io_scanners.yaml index cd4f64d..d5470b6 100644 --- a/operator/config/crd/bases/scanning.portscanner.io_scanners.yaml +++ b/operator/config/crd/bases/scanning.portscanner.io_scanners.yaml @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/operator/config/crd/kustomization.yaml b/operator/config/crd/kustomization.yaml deleted file mode 100644 index 7742d33..0000000 --- a/operator/config/crd/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - bases/scanning.portscanner.io_scanners.yaml diff --git a/operator/config/default/kustomization.yaml b/operator/config/default/kustomization.yaml deleted file mode 100644 index b0837e6..0000000 --- a/operator/config/default/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: portscanner-system -resources: - - namespace.yaml - - ../crd - - ../rbac - - ../scanner - - ../priorities - - ../manager -images: - - name: portscanner-operator - newName: portscanner-operator - digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 diff --git a/operator/config/default/namespace.yaml b/operator/config/default/namespace.yaml deleted file mode 100644 index f9b5ffe..0000000 --- a/operator/config/default/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: Namespace -metadata: - name: portscanner-system - labels: - app.kubernetes.io/name: portscanner diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml deleted file mode 100644 index adfd90c..0000000 --- a/operator/config/manager/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - manager.yaml diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml deleted file mode 100644 index 17d7731..0000000 --- a/operator/config/manager/manager.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: portscanner-operator - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator - template: - metadata: - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator - spec: - serviceAccountName: portscanner-operator - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - containers: - - name: manager - # Replace the all-zero placeholder with the published operator digest. - image: portscanner-operator@sha256:0000000000000000000000000000000000000000000000000000000000000000 - imagePullPolicy: IfNotPresent - args: - - --leader-elect=true - # Replace the all-zero placeholder with the published scanner digest. - - --scanner-image=portscanner-scanner@sha256:0000000000000000000000000000000000000000000000000000000000000000 - - --scanner-service-account=portscanner-scanner - - --result-bucket=replace-with-results-bucket - - --result-prefix=scan-results - - --high-priority-class=portscanner-high - - --normal-priority-class=portscanner-normal - - --high-priority-threshold=100 - ports: - - name: metrics - containerPort: 8080 - protocol: TCP - - name: health - containerPort: 8081 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: health - readinessProbe: - httpGet: - path: /readyz - port: health - resources: - requests: - cpu: 50m - memory: 64Mi - ephemeral-storage: 32Mi - limits: - cpu: 500m - memory: 256Mi - ephemeral-storage: 256Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - volumeMounts: - - name: tmp - mountPath: /tmp - terminationGracePeriodSeconds: 10 - volumes: - - name: tmp - emptyDir: - sizeLimit: 64Mi diff --git a/operator/config/priorities/kustomization.yaml b/operator/config/priorities/kustomization.yaml deleted file mode 100644 index 1622228..0000000 --- a/operator/config/priorities/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - priorityclasses.yaml diff --git a/operator/config/priorities/priorityclasses.yaml b/operator/config/priorities/priorityclasses.yaml deleted file mode 100644 index bd088f4..0000000 --- a/operator/config/priorities/priorityclasses.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: portscanner-normal - labels: - app.kubernetes.io/name: portscanner -value: 100000 -globalDefault: false -preemptionPolicy: Never -description: Normal-priority one-shot port scanner Jobs. ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: portscanner-high - labels: - app.kubernetes.io/name: portscanner -value: 1000000 -globalDefault: false -preemptionPolicy: Never -description: High-priority one-shot port scanner Jobs. diff --git a/operator/config/rbac/kustomization.yaml b/operator/config/rbac/kustomization.yaml deleted file mode 100644 index 895d3fc..0000000 --- a/operator/config/rbac/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - service_account.yaml - - role.yaml - - role_binding.yaml - - leader_election_role.yaml - - leader_election_role_binding.yaml diff --git a/operator/config/rbac/leader_election_role.yaml b/operator/config/rbac/leader_election_role.yaml deleted file mode 100644 index 596ae11..0000000 --- a/operator/config/rbac/leader_election_role.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: portscanner-operator-leader-election - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - update - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch diff --git a/operator/config/rbac/leader_election_role_binding.yaml b/operator/config/rbac/leader_election_role_binding.yaml deleted file mode 100644 index f4b8054..0000000 --- a/operator/config/rbac/leader_election_role_binding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: portscanner-operator-leader-election - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: portscanner-operator-leader-election -subjects: - - kind: ServiceAccount - name: portscanner-operator diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml deleted file mode 100644 index 536cf70..0000000 --- a/operator/config/rbac/role.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: portscanner-operator -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs/status - verbs: - - get -- apiGroups: - - scanning.portscanner.io - resources: - - scanners - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - scanning.portscanner.io - resources: - - scanners/finalizers - verbs: - - update -- apiGroups: - - scanning.portscanner.io - resources: - - scanners/status - verbs: - - get - - patch - - update diff --git a/operator/config/rbac/role_binding.yaml b/operator/config/rbac/role_binding.yaml deleted file mode 100644 index c3dc30b..0000000 --- a/operator/config/rbac/role_binding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: portscanner-operator - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: portscanner-operator -subjects: - - kind: ServiceAccount - name: portscanner-operator - namespace: portscanner-system diff --git a/operator/config/rbac/service_account.yaml b/operator/config/rbac/service_account.yaml deleted file mode 100644 index 2da3d63..0000000 --- a/operator/config/rbac/service_account.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: portscanner-operator - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: operator diff --git a/operator/config/samples/scanning_v1alpha1_scanner.yaml b/operator/config/samples/scanning_v1alpha1_scanner.yaml index 766a69b..ce7747c 100644 --- a/operator/config/samples/scanning_v1alpha1_scanner.yaml +++ b/operator/config/samples/scanning_v1alpha1_scanner.yaml @@ -1,5 +1,5 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT # 192.0.2.0/24 is reserved by RFC 5737 for documentation and is not a real # Internet scan target. diff --git a/operator/config/scanner/kustomization.yaml b/operator/config/scanner/kustomization.yaml deleted file mode 100644 index 8ee67e3..0000000 --- a/operator/config/scanner/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - service_account.yaml - - role.yaml - - role_binding.yaml diff --git a/operator/config/scanner/role.yaml b/operator/config/scanner/role.yaml deleted file mode 100644 index 714f011..0000000 --- a/operator/config/scanner/role.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -# Scanner Jobs do not require Kubernetes API permissions. This empty Role makes -# the least-privilege boundary explicit and can be patched by an installation -# that provides a scanner implementation with additional documented needs. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: portscanner-scanner - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: scanner -rules: [] diff --git a/operator/config/scanner/role_binding.yaml b/operator/config/scanner/role_binding.yaml deleted file mode 100644 index ac6af37..0000000 --- a/operator/config/scanner/role_binding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: portscanner-scanner - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: scanner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: portscanner-scanner -subjects: - - kind: ServiceAccount - name: portscanner-scanner diff --git a/operator/config/scanner/service_account.yaml b/operator/config/scanner/service_account.yaml deleted file mode 100644 index 67fe3be..0000000 --- a/operator/config/scanner/service_account.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: portscanner-scanner - labels: - app.kubernetes.io/name: portscanner - app.kubernetes.io/component: scanner -automountServiceAccountToken: false diff --git a/operator/go.mod b/operator/go.mod index 51ce7a6..c3f2050 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT module github.com/Roblox/portscanner/operator @@ -44,11 +44,11 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/operator/go.sum b/operator/go.sum index ffb9353..afa8f0f 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -115,16 +115,16 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= diff --git a/operator/hack/boilerplate.go.txt b/operator/hack/boilerplate.go.txt index 0c65c42..b732a84 100644 --- a/operator/hack/boilerplate.go.txt +++ b/operator/hack/boilerplate.go.txt @@ -1,2 +1,2 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT diff --git a/operator/hack/boilerplate.yaml.txt b/operator/hack/boilerplate.yaml.txt index 89f8452..ba8c424 100644 --- a/operator/hack/boilerplate.yaml.txt +++ b/operator/hack/boilerplate.yaml.txt @@ -1,2 +1,2 @@ -# Copyright 2026 Roblox Corporation -# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT diff --git a/operator/internal/controller/integration_boundary_test.go b/operator/internal/controller/integration_boundary_test.go index 69768ed..e050db6 100644 --- a/operator/internal/controller/integration_boundary_test.go +++ b/operator/internal/controller/integration_boundary_test.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package controller diff --git a/operator/internal/controller/job_builder.go b/operator/internal/controller/job_builder.go index 56ed1b8..53020f6 100644 --- a/operator/internal/controller/job_builder.go +++ b/operator/internal/controller/job_builder.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package controller @@ -295,12 +295,9 @@ func scannerArguments(spec scanningv1alpha1.ScannerSpec) ([]string, error) { "--tcp-ports=" + coverage, } switch spec.Profile { - case scanningv1alpha1.ProfileFastFullTCP: - args = append(args, "--scan-mode=fast") - case scanningv1alpha1.ProfileTargetedTCP: - args = append(args, "--scan-mode=targeted") - case scanningv1alpha1.ProfileDeep: - args = append(args, "--scan-mode=deep", "--service-detection") + case scanningv1alpha1.ProfileFastFullTCP, + scanningv1alpha1.ProfileTargetedTCP, + scanningv1alpha1.ProfileDeep: default: return nil, fmt.Errorf("unsupported scan profile %q", spec.Profile) } diff --git a/operator/internal/controller/job_builder_test.go b/operator/internal/controller/job_builder_test.go index 843b66a..8e7f269 100644 --- a/operator/internal/controller/job_builder_test.go +++ b/operator/internal/controller/job_builder_test.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package controller @@ -33,7 +33,6 @@ func TestScannerArgumentsMapProfilesAndCoverage(t *testing.T) { "--target=192.0.2.10", "--profile=fast-full-tcp", "--tcp-ports=1-65535", - "--scan-mode=fast", }, }, { @@ -45,7 +44,6 @@ func TestScannerArgumentsMapProfilesAndCoverage(t *testing.T) { "--target=192.0.2.10", "--profile=targeted-tcp", "--tcp-ports=80-84,443", - "--scan-mode=targeted", }, }, { @@ -55,8 +53,6 @@ func TestScannerArgumentsMapProfilesAndCoverage(t *testing.T) { "--target=192.0.2.10", "--profile=deep", "--tcp-ports=1-65535", - "--scan-mode=deep", - "--service-detection", }, }, { diff --git a/operator/internal/controller/scanner_controller.go b/operator/internal/controller/scanner_controller.go index 71c6c0d..3c6b292 100644 --- a/operator/internal/controller/scanner_controller.go +++ b/operator/internal/controller/scanner_controller.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package controller diff --git a/operator/internal/controller/scanner_controller_test.go b/operator/internal/controller/scanner_controller_test.go index ed10406..71c4ab6 100644 --- a/operator/internal/controller/scanner_controller_test.go +++ b/operator/internal/controller/scanner_controller_test.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT package controller diff --git a/operator/internal/controller/scanner_envtest_test.go b/operator/internal/controller/scanner_envtest_test.go index d4472de..6122cc1 100644 --- a/operator/internal/controller/scanner_envtest_test.go +++ b/operator/internal/controller/scanner_envtest_test.go @@ -1,5 +1,5 @@ -// Copyright 2026 Roblox Corporation -// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2026 Portscanner contributors +// SPDX-License-Identifier: MIT //go:build envtest diff --git a/parser/Dockerfile b/parser/Dockerfile index 043b7db..e06a93e 100644 --- a/parser/Dockerfile +++ b/parser/Dockerfile @@ -24,7 +24,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps \ && rm -rf /tmp/portscanner-contracts /tmp/portscanner-parser RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ USER 65532:65532 ENTRYPOINT ["/usr/local/bin/python", "-m", "awslambdaric"] diff --git a/parser/pyproject.toml b/parser/pyproject.toml index 7f877e1..b82aae8 100644 --- a/parser/pyproject.toml +++ b/parser/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "portscanner-parser" -version = "0.1.0" +version = "1.0.0" description = "Generation-safe ACT scan result parser" requires-python = ">=3.12" license = "MIT" diff --git a/parser/requirements-runtime.txt b/parser/requirements-runtime.txt index 306b879..e79455c 100644 --- a/parser/requirements-runtime.txt +++ b/parser/requirements-runtime.txt @@ -8,17 +8,29 @@ annotated-types==0.8.0 \ awslambdaric==4.0.2 \ --hash=sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5 \ --hash=sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948 \ + --hash=sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e \ + --hash=sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f \ --hash=sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378 \ + --hash=sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594 \ + --hash=sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd \ + --hash=sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e \ + --hash=sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f \ --hash=sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb \ + --hash=sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e \ --hash=sha256:92cceacf2e37d455dd1c90a771b0b518f336971dace4cb771a5e43f9a595e867 \ --hash=sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318 \ --hash=sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005 \ + --hash=sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c \ + --hash=sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5 \ --hash=sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35 \ --hash=sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5 \ + --hash=sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c \ + --hash=sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68 \ --hash=sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372 \ --hash=sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6 \ --hash=sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3 \ - --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 + --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 \ + --hash=sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693 boto3==1.43.65 \ --hash=sha256:a8217fb68cae3f8a7575eef395383b68b846d92d29b0e8f6e948e6e9e08dcc3f \ --hash=sha256:f2331154aee1ae97ece48077d77f41d3bd5ea39eb4e3037030448b58695a3a79 diff --git a/parser/src/act_parser/config.py b/parser/src/act_parser/config.py index f8f0ea0..99acffa 100644 --- a/parser/src/act_parser/config.py +++ b/parser/src/act_parser/config.py @@ -23,8 +23,9 @@ class ConfigurationError(RuntimeError): class Settings: scan_result_bucket: str raw_result_bucket: str - finding_bucket: str + finding_bucket: str | None database_dsn: str + finding_export_enabled: bool = True max_envelope_bytes: int = 1_048_576 max_xml_bytes: int = 67_108_864 @@ -33,8 +34,9 @@ class Settings: class TargetEventSettings: target_event_bucket: str target_event_prefix: str - finding_bucket: str + finding_bucket: str | None database_dsn: str + finding_export_enabled: bool = True max_event_bytes: int = 65_536 @@ -105,38 +107,55 @@ def database_dsn() -> str: return _database_dsn() +def _finding_export_enabled() -> bool: + raw = os.getenv("FINDING_EXPORT_ENABLED") + if raw is None: + return True + if raw.lower() not in {"true", "false"}: + raise ConfigurationError("FINDING_EXPORT_ENABLED must be true or false") + return raw.lower() == "true" + + def load_settings() -> Settings: + finding_export_enabled = _finding_export_enabled() buckets = { "scan_result_bucket": os.getenv("SCAN_RESULT_BUCKET"), "raw_result_bucket": os.getenv("RAW_RESULT_BUCKET"), - "finding_bucket": os.getenv("FINDING_BUCKET"), } missing = [key for key, value in buckets.items() if not value] + finding_bucket = os.getenv("FINDING_BUCKET") + if finding_export_enabled and not finding_bucket: + missing.append("finding_bucket") if missing: raise ConfigurationError(f"missing bucket settings: {', '.join(sorted(missing))}") return Settings( scan_result_bucket=buckets["scan_result_bucket"] or "", raw_result_bucket=buckets["raw_result_bucket"] or "", - finding_bucket=buckets["finding_bucket"] or "", + finding_bucket=finding_bucket, database_dsn=database_dsn(), + finding_export_enabled=finding_export_enabled, max_envelope_bytes=int(os.getenv("MAX_SCAN_RESULT_ENVELOPE_BYTES", "1048576")), max_xml_bytes=int(os.getenv("MAX_RAW_XML_BYTES", "67108864")), ) def load_target_event_settings() -> TargetEventSettings: + finding_export_enabled = _finding_export_enabled() values = { "target_event_bucket": os.getenv("TARGET_EVENT_BUCKET"), "target_event_prefix": os.getenv("TARGET_EVENT_PREFIX"), - "finding_bucket": os.getenv("FINDING_BUCKET"), } missing = [key for key, value in values.items() if not value] + finding_bucket = os.getenv("FINDING_BUCKET") + if finding_export_enabled and not finding_bucket: + missing.append("finding_bucket") if missing: raise ConfigurationError(f"missing target event settings: {', '.join(sorted(missing))}") return TargetEventSettings( target_event_bucket=values["target_event_bucket"] or "", target_event_prefix=values["target_event_prefix"] or "", - finding_bucket=values["finding_bucket"] or "", + finding_bucket=finding_bucket, database_dsn=database_dsn(), + finding_export_enabled=finding_export_enabled, max_event_bytes=int(os.getenv("MAX_TARGET_EVENT_BYTES", "65536")), ) diff --git a/parser/src/act_parser/database.py b/parser/src/act_parser/database.py index fd1509a..bf2f19f 100644 --- a/parser/src/act_parser/database.py +++ b/parser/src/act_parser/database.py @@ -116,6 +116,7 @@ def apply_target_event( event: TargetEvent, *, finding_bucket: str | None = None, + queue_finding_handoffs: bool = True, ) -> bool: """Apply one inventory event; return False for an already-seen event.""" addresses = _target_addresses(event) @@ -300,23 +301,26 @@ def apply_target_event( or open_findings < 0 ): raise DataInvariantError("open finding count query returned an invalid count") - if open_findings and not finding_bucket: + if open_findings and queue_finding_handoffs and not finding_bucket: raise DataInvariantError( "finding bucket is required to publish removal transitions" ) self._close_for_ownership_removal( event, finding_bucket=finding_bucket, + queue_finding_handoffs=queue_finding_handoffs, ) elif accepted and generation_gap_reactivation: self._close_for_generation_gap_reactivation( event, finding_bucket=finding_bucket, + queue_finding_handoffs=queue_finding_handoffs, ) elif accepted and address_binding_changed: self._close_for_address_binding_change( event, finding_bucket=finding_bucket, + queue_finding_handoffs=queue_finding_handoffs, ) return True @@ -331,8 +335,9 @@ def ingest_scan( envelope_sha256: str, raw_result_version: str | None, enrichment_result_version: str | None, - finding_bucket: str, + finding_bucket: str | None, xml_completion_validated: bool, + queue_finding_handoffs: bool = True, ) -> IngestResult: handoff_keys: list[str] = [] with self.connection.transaction(): @@ -629,6 +634,7 @@ def ingest_scan( occurred_at=envelope.scan_completed_at, finding_bucket=finding_bucket, transition_types=transition_types, + queue_event_handoffs=queue_finding_handoffs, ) ) @@ -1154,7 +1160,7 @@ def _reconcile_finding_keys( source_key: str, source_attempt_id: str | None, occurred_at: datetime, - finding_bucket: str, + finding_bucket: str | None, transition_types: Mapping[tuple[str, int], str] | None = None, queue_event_handoffs: bool = True, ) -> list[str]: @@ -1319,10 +1325,12 @@ def _ensure_finding( source_key: str, source_attempt_id: str | None, occurred_at: datetime, - finding_bucket: str, + finding_bucket: str | None, exposure_transition: str | None, queue_event_handoff: bool, ) -> str | None: + if queue_event_handoff and not finding_bucket: + raise DataInvariantError("finding bucket is required when handoff export is enabled") fingerprint = finding_fingerprint( rule["rule_key"], exposure["target_id"], @@ -1524,7 +1532,7 @@ def _ensure_finding( event_key=event_key, fingerprint=fingerprint, source_attempt_id=source_attempt_id, - finding_bucket=finding_bucket, + finding_bucket=finding_bucket or "", ) def _resolve_finding( @@ -1537,9 +1545,11 @@ def _resolve_finding( source_key: str, source_attempt_id: str | None, occurred_at: datetime, - finding_bucket: str, + finding_bucket: str | None, queue_event_handoff: bool, ) -> str | None: + if queue_event_handoff and not finding_bucket: + raise DataInvariantError("finding bucket is required when handoff export is enabled") if finding["status"] != "open": return None fingerprint = finding["fingerprint"].strip() @@ -1605,7 +1615,7 @@ def _resolve_finding( event_key=event_key, fingerprint=fingerprint, source_attempt_id=source_attempt_id, - finding_bucket=finding_bucket, + finding_bucket=finding_bucket or "", ) def _insert_finding_event( @@ -1821,6 +1831,7 @@ def _close_for_generation_gap_reactivation( event: TargetEvent, *, finding_bucket: str | None, + queue_finding_handoffs: bool, ) -> None: keys = { (row["protocol"], row["port"]) @@ -1848,7 +1859,7 @@ def _close_for_generation_gap_reactivation( ) if not keys: return - if not finding_bucket: + if queue_finding_handoffs and not finding_bucket: raise DataInvariantError( "finding bucket is required to publish generation-gap transitions" ) @@ -1873,6 +1884,7 @@ def _close_for_generation_gap_reactivation( occurred_at=event.source_observed_at, finding_bucket=finding_bucket, transition_types=dict.fromkeys(keys, "closed"), + queue_event_handoffs=queue_finding_handoffs, ) def _close_for_address_binding_change( @@ -1880,6 +1892,7 @@ def _close_for_address_binding_change( event: TargetEvent, *, finding_bucket: str | None, + queue_finding_handoffs: bool, ) -> None: current_addresses = {parse_address(address) for address in event.addresses} keys = { @@ -1910,7 +1923,7 @@ def _close_for_address_binding_change( ) if not keys: return - if not finding_bucket: + if queue_finding_handoffs and not finding_bucket: raise DataInvariantError( "finding bucket is required to publish address-binding transitions" ) @@ -1938,6 +1951,7 @@ def _close_for_address_binding_change( occurred_at=event.source_observed_at, finding_bucket=finding_bucket, transition_types=transitions, + queue_event_handoffs=queue_finding_handoffs, ) def _close_for_ownership_removal( @@ -1945,6 +1959,7 @@ def _close_for_ownership_removal( event: TargetEvent, *, finding_bucket: str | None, + queue_finding_handoffs: bool, ) -> None: keys = { (row["protocol"], row["port"]) @@ -1989,16 +2004,20 @@ def _close_for_ownership_removal( source_key=event.event_id, source_attempt_id=None, occurred_at=event.removed_at or event.source_observed_at, - finding_bucket=finding_bucket or "", + finding_bucket=finding_bucket, transition_types=dict.fromkeys(keys, "closed"), + queue_event_handoffs=queue_finding_handoffs, ) def reconcile_all( self, invocation_key: str, *, - finding_bucket: str, + finding_bucket: str | None, + queue_finding_handoffs: bool = True, ) -> ReconciliationResult: + if queue_finding_handoffs and not finding_bucket: + raise DataInvariantError("finding bucket is required when handoff export is enabled") run_key = stable_hash("act-reconciliation-v1", invocation_key) with self.connection.transaction(): run = self.connection.execute( @@ -2014,7 +2033,11 @@ def reconcile_all( return ReconciliationResult( duplicate=True, findings_examined=run["findings_examined"], - handoff_keys=tuple(self._pending_handoff_keys(run_key=run_key)), + handoff_keys=( + tuple(self._pending_handoff_keys(run_key=run_key)) + if queue_finding_handoffs + else () + ), ) if run is None: self.connection.execute( @@ -2049,20 +2072,22 @@ def reconcile_all( source_attempt_id=None, occurred_at=now, finding_bucket=finding_bucket, + queue_event_handoffs=queue_finding_handoffs, ) ) - current = self.connection.execute( - "SELECT * FROM act.current_findings ORDER BY fingerprint" - ).fetchall() - queued.extend( - self._queue_current_snapshot( - finding["fingerprint"].strip(), - run_key=run_key, - finding_bucket=finding_bucket, + if queue_finding_handoffs: + current = self.connection.execute( + "SELECT * FROM act.current_findings ORDER BY fingerprint" + ).fetchall() + queued.extend( + self._queue_current_snapshot( + finding["fingerprint"].strip(), + run_key=run_key, + finding_bucket=finding_bucket or "", + ) + for finding in current ) - for finding in current - ) self.connection.execute( """ UPDATE act.reconciliation_runs diff --git a/parser/src/act_parser/handler.py b/parser/src/act_parser/handler.py index 5088861..0235e47 100644 --- a/parser/src/act_parser/handler.py +++ b/parser/src/act_parser/handler.py @@ -318,8 +318,10 @@ def _process_scan_result( enrichment_result_version=enrichment_version, finding_bucket=settings.finding_bucket, xml_completion_validated=xml_completion_validated, + queue_finding_handoffs=settings.finding_export_enabled, ) - HandoffPublisher(s3, repository).publish_all(keys=result.handoff_keys) + if settings.finding_export_enabled: + HandoffPublisher(s3, repository).publish_all(keys=result.handoff_keys) except AttemptConflictError as error: raise PermanentRecordError("attempt identity conflict") from error except UnknownTargetError as error: diff --git a/parser/src/act_parser/target_handler.py b/parser/src/act_parser/target_handler.py index 02a3790..2173ce1 100644 --- a/parser/src/act_parser/target_handler.py +++ b/parser/src/act_parser/target_handler.py @@ -413,8 +413,13 @@ def _process_target_event( try: with psycopg.connect(settings.database_dsn) as connection: repository = Repository(connection) - repository.apply_target_event(event, finding_bucket=settings.finding_bucket) - _publish_target_event_handoffs(s3, repository, event.event_id) + repository.apply_target_event( + event, + finding_bucket=settings.finding_bucket, + queue_finding_handoffs=settings.finding_export_enabled, + ) + if settings.finding_export_enabled: + _publish_target_event_handoffs(s3, repository, event.event_id) except RuleConfigurationError as error: raise RetryableRecordError("editable detection rule configuration is invalid") from error except DataInvariantError as error: diff --git a/parser/tests/test_handler.py b/parser/tests/test_handler.py index 8cc2f7b..db42ac2 100644 --- a/parser/tests/test_handler.py +++ b/parser/tests/test_handler.py @@ -4,6 +4,7 @@ import json from dataclasses import replace from datetime import UTC, datetime +from types import SimpleNamespace from typing import Any import pytest @@ -329,3 +330,66 @@ def ingest_scan(self, *_args: Any, **_kwargs: Any) -> None: envelope_key="result.json", envelope_version=None, ) + + +def test_finding_export_disabled_commits_database_without_publisher( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = Settings( + scan_result_bucket="scan-result-test", + raw_result_bucket="raw-result-test", + finding_bucket=None, + database_dsn="unused", + finding_export_enabled=False, + ) + envelope = _envelope("partial") + ingest_calls: list[dict[str, Any]] = [] + + class ConnectionContext: + def __enter__(self) -> object: + return object() + + def __exit__(self, *_args: object) -> None: + return None + + class DatabaseOnlyRepository: + def __init__(self, _connection: object) -> None: + pass + + def ingest_scan(self, *_args: Any, **kwargs: Any) -> object: + ingest_calls.append(kwargs) + return SimpleNamespace(handoff_keys=()) + + class MustNotPublish: + def __init__(self, *_args: object) -> None: + raise AssertionError("finding export disabled constructed a publisher") + + monkeypatch.setattr( + handler, + "_get_object", + lambda *_args, **_kwargs: (b"{}", hashlib.sha256(b"{}").hexdigest(), None), + ) + monkeypatch.setattr(handler, "validate_scan_result", lambda _payload: {}) + monkeypatch.setattr( + handler.ScanEnvelope, + "from_mapping", + classmethod(lambda _cls, _payload: envelope), + ) + monkeypatch.setattr( + handler, + "_load_observations", + lambda **_kwargs: ([], None, None, False), + ) + monkeypatch.setattr(handler.psycopg, "connect", lambda _dsn: ConnectionContext()) + monkeypatch.setattr(handler, "Repository", DatabaseOnlyRepository) + monkeypatch.setattr(handler, "HandoffPublisher", MustNotPublish) + + handler._process_scan_result( + s3=object(), + settings=settings, + envelope_key="result.json", + envelope_version=None, + ) + + assert ingest_calls[0]["finding_bucket"] is None + assert ingest_calls[0]["queue_finding_handoffs"] is False diff --git a/parser/tests/test_postgresql_integration.py b/parser/tests/test_postgresql_integration.py index a1b5ade..6212feb 100644 --- a/parser/tests/test_postgresql_integration.py +++ b/parser/tests/test_postgresql_integration.py @@ -192,6 +192,9 @@ def _ingest( repository: Repository, envelope: ScanEnvelope, observations: list[Observation], + *, + finding_bucket: str | None = FINDING_BUCKET, + queue_finding_handoffs: bool = True, ) -> Any: target_event = repository.connection.execute( """ @@ -213,8 +216,9 @@ def _ingest( envelope_sha256=hashlib.sha256(f"envelope:{envelope.attempt_id}".encode()).hexdigest(), raw_result_version=None, enrichment_result_version=None, - finding_bucket=FINDING_BUCKET, + finding_bucket=finding_bucket, xml_completion_validated=envelope.outcome == "complete", + queue_finding_handoffs=queue_finding_handoffs, ) @@ -503,25 +507,6 @@ def test_address_binding_change_closes_old_address_and_publishes_history( == 2 ) - migrator = Migrator( - connection, - discover_migrations(REPOSITORY_ROOT / "db" / "migrations"), - ) - assert migrator.down(target="000004", steps=None) == ["000005"] - assert ( - _scalar( - connection, - """ - SELECT closure_reason - FROM act.exposure_state - WHERE target_id = %s AND protocol = 'tcp' AND port = 22 - """, - (TARGET_ID,), - ) - == "address_binding_changed" - ) - assert migrator.up() == ["000005"] - _ingest( repository, _envelope( @@ -871,3 +856,60 @@ def test_handoff_retry_and_processor_current_only_export( serialized = str(snapshots) assert "raw_result" not in serialized assert "credentials" not in serialized + + +def test_database_findings_commit_without_export_handoffs( + repository: Repository, + connection: Any, +) -> None: + repository.apply_target_event( + _target_event("target-upsert-database-only", scan_reason="new_target"), + finding_bucket=None, + queue_finding_handoffs=False, + ) + result = _ingest( + repository, + _envelope("attempt-database-only", coverage=_coverage("18080")), + [_observation(18080)], + finding_bucket=None, + queue_finding_handoffs=False, + ) + + assert result.state_eligible + assert result.handoff_keys == () + assert ( + _scalar( + connection, + """ + SELECT count(*) + FROM act.findings + WHERE target_id = %s + AND protocol = 'tcp' + AND port = 18080 + AND status = 'open' + AND severity = 'low' + """, + (TARGET_ID,), + ) + == 1 + ) + assert _scalar(connection, "SELECT count(*) FROM act.finding_handoffs") == 0 + + repository.apply_target_event( + _target_event( + "target-remove-database-only", + generation=2, + event_type="remove", + minute=2, + ), + finding_bucket=None, + queue_finding_handoffs=False, + ) + assert ( + _scalar( + connection, + "SELECT count(*) FROM act.findings WHERE status = 'resolved'", + ) + == 1 + ) + assert _scalar(connection, "SELECT count(*) FROM act.finding_handoffs") == 0 diff --git a/parser/tests/test_target_handler.py b/parser/tests/test_target_handler.py index d66dcb3..748d220 100644 --- a/parser/tests/test_target_handler.py +++ b/parser/tests/test_target_handler.py @@ -277,8 +277,14 @@ class FakeRepository: def __init__(self, _connection: object) -> None: pass - def apply_target_event(self, local: Any, *, finding_bucket: str) -> bool: - applied.append((local, finding_bucket)) + def apply_target_event( + self, + local: Any, + *, + finding_bucket: str | None, + queue_finding_handoffs: bool, + ) -> bool: + applied.append((local, finding_bucket, queue_finding_handoffs)) return False def pending_target_event_handoff_keys(self, event_id: str) -> tuple[str, ...]: @@ -311,9 +317,77 @@ def publish_all(self, *, keys: tuple[str, ...], page_size: int = 1000) -> int: assert applied[0][0].event_type == "remove" assert applied[0][1] == "finding-test" + assert applied[0][2] is True assert publish_calls == [(("handoff-1",), 1000)] +def test_database_only_target_projection_skips_finding_handoffs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + event = _shared_event("target.removed") + applied: list[tuple[Any, str | None, bool]] = [] + + class S3: + def get_object(self, **_kwargs: Any) -> dict[str, Any]: + return { + "Body": BytesIO(b"{}"), + "ContentLength": 2, + "ETag": '"etag-1"', + "VersionId": "version-1", + } + + class ConnectionContext: + def __enter__(self) -> object: + return object() + + def __exit__(self, *_args: object) -> None: + return None + + class FakeRepository: + def __init__(self, _connection: object) -> None: + pass + + def apply_target_event( + self, + local: Any, + *, + finding_bucket: str | None, + queue_finding_handoffs: bool, + ) -> bool: + applied.append((local, finding_bucket, queue_finding_handoffs)) + return True + + class MustNotPublish: + def __init__(self, *_args: object) -> None: + raise AssertionError("database-only target projection created a publisher") + + settings = TargetEventSettings( + target_event_bucket="target-event-test", + target_event_prefix="target-events/aws/", + finding_bucket=None, + database_dsn="unused", + finding_export_enabled=False, + max_event_bytes=1024, + ) + monkeypatch.setattr(target_handler, "parse_target_event", lambda _document: event) + monkeypatch.setattr(target_handler.psycopg, "connect", lambda _dsn: ConnectionContext()) + monkeypatch.setattr(target_handler, "Repository", FakeRepository) + monkeypatch.setattr(target_handler, "HandoffPublisher", MustNotPublish) + + target_handler._process_target_event( + s3=S3(), + settings=settings, + reference=target_handler.S3ObjectReference( + "target-event-test", + "target-events/aws/event.json", + "etag-1", + "version-1", + ), + ) + + assert applied == [(target_handler._local_target_event(event), None, False)] + + def test_lambda_redrives_retryable_and_permanent_batch_failures( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/processor/Dockerfile b/processor/Dockerfile index cd8f2c6..5db4651 100644 --- a/processor/Dockerfile +++ b/processor/Dockerfile @@ -28,7 +28,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps /tmp/por && rm -rf /tmp/portscanner-processor RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ USER 65532:65532 ENTRYPOINT ["/usr/local/bin/python", "-m", "awslambdaric"] diff --git a/processor/pyproject.toml b/processor/pyproject.toml index 1b5bdcc..a1a1699 100644 --- a/processor/pyproject.toml +++ b/processor/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "portscanner-processor" -version = "0.1.0" +version = "1.0.0" description = "Scheduled ACT finding reconciliation and export" requires-python = ">=3.12" license = "MIT" diff --git a/processor/requirements-runtime.txt b/processor/requirements-runtime.txt index 306b879..e79455c 100644 --- a/processor/requirements-runtime.txt +++ b/processor/requirements-runtime.txt @@ -8,17 +8,29 @@ annotated-types==0.8.0 \ awslambdaric==4.0.2 \ --hash=sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5 \ --hash=sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948 \ + --hash=sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e \ + --hash=sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f \ --hash=sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378 \ + --hash=sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594 \ + --hash=sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd \ + --hash=sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e \ + --hash=sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f \ --hash=sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb \ + --hash=sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e \ --hash=sha256:92cceacf2e37d455dd1c90a771b0b518f336971dace4cb771a5e43f9a595e867 \ --hash=sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318 \ --hash=sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005 \ + --hash=sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c \ + --hash=sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5 \ --hash=sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35 \ --hash=sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5 \ + --hash=sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c \ + --hash=sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68 \ --hash=sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372 \ --hash=sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6 \ --hash=sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3 \ - --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 + --hash=sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51 \ + --hash=sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693 boto3==1.43.65 \ --hash=sha256:a8217fb68cae3f8a7575eef395383b68b846d92d29b0e8f6e948e6e9e08dcc3f \ --hash=sha256:f2331154aee1ae97ece48077d77f41d3bd5ea39eb4e3037030448b58695a3a79 diff --git a/processor/src/act_processor/handler.py b/processor/src/act_processor/handler.py index cfea4b7..10d4fc8 100644 --- a/processor/src/act_processor/handler.py +++ b/processor/src/act_processor/handler.py @@ -1,9 +1,10 @@ -"""EventBridge-scheduled full finding reconciliation and handoff repair.""" +"""Scheduled reconciliation and narrow managed-canary verification.""" from __future__ import annotations import os from collections.abc import Mapping +from dataclasses import dataclass from typing import Any import boto3 @@ -12,26 +13,226 @@ from act_parser.database import Repository from act_parser.handoff import HandoffPublisher +_MAX_PORT = 65_535 +_MAX_HANDOFF_PAGE_SIZE = 1000 -def lambda_handler(event: Mapping[str, Any], _context: Any) -> dict[str, Any]: + +@dataclass(frozen=True, slots=True) +class _CanaryScope: + account_id: str + region: str + network_interface_id: str + private_ip: str + public_ip: str + tag_key: str + tag_value: str + port: int + + +def _enabled(name: str, *, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + if raw.lower() not in {"true", "false"}: + raise ConfigurationError(f"{name} must be true or false") + return raw.lower() == "true" + + +def _canary_scope() -> _CanaryScope: + names = { + "account_id": "MANAGED_CANARY_ACCOUNT_ID", + "region": "MANAGED_CANARY_REGION", + "network_interface_id": "MANAGED_CANARY_ENI_ID", + "private_ip": "MANAGED_CANARY_PRIVATE_IP", + "public_ip": "MANAGED_CANARY_PUBLIC_IP", + "tag_key": "MANAGED_CANARY_TAG_KEY", + "tag_value": "MANAGED_CANARY_TAG_VALUE", + "port": "MANAGED_CANARY_TCP_PORT", + } + values = {field: (os.getenv(name) or "").strip() for field, name in names.items()} + missing = [names[field] for field, value in values.items() if not value] + if missing: + raise ConfigurationError( + f"managed canary status configuration is incomplete: {', '.join(sorted(missing))}" + ) + try: + port = int(values["port"]) + except ValueError as error: + raise ConfigurationError("MANAGED_CANARY_TCP_PORT must be an integer") from error + if not 1 <= port <= _MAX_PORT: + raise ConfigurationError("MANAGED_CANARY_TCP_PORT must be within 1..65535") + return _CanaryScope( + account_id=values["account_id"], + region=values["region"], + network_interface_id=values["network_interface_id"], + private_ip=values["private_ip"], + public_ip=values["public_ip"], + tag_key=values["tag_key"], + tag_value=values["tag_value"], + port=port, + ) + + +def _count(row: Mapping[str, Any], name: str) -> int: + value = row.get(name) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError("managed canary status query returned invalid counts") + return value + + +def _managed_canary_status(connection: Any, scope: _CanaryScope) -> dict[str, Any]: + row = connection.execute( + """ + WITH exact_targets AS ( + SELECT target_id + FROM act.targets + WHERE provider = 'aws' + AND status = 'active' + AND provider_scope_id = %s + AND location = %s + AND context ->> 'network_interface_id' = %s + AND context ->> 'private_ip' = %s + AND context ->> 'public_ip' = %s + AND context -> 'tags' ->> %s = %s + ) + SELECT + (SELECT count(*) FROM exact_targets) AS target_count, + ( + SELECT count(*) + FROM act.target_events AS event + WHERE event.target_id IN (SELECT target_id FROM exact_targets) + AND event.accepted + ) AS event_count, + ( + SELECT count(*) + FROM act.scan_attempts AS attempt + WHERE attempt.target_id IN (SELECT target_id FROM exact_targets) + ) AS attempt_count, + ( + SELECT count(*) + FROM act.scan_attempts AS attempt + WHERE attempt.target_id IN (SELECT target_id FROM exact_targets) + AND attempt.outcome = 'complete' + AND attempt.state_eligible + AND attempt.profile = 'targeted-tcp' + AND EXISTS ( + SELECT 1 + FROM act.scan_attempt_coverage AS coverage + WHERE coverage.attempt_id = attempt.attempt_id + AND coverage.protocol = 'tcp' + AND coverage.complete + AND coverage.port_from = %s + AND coverage.port_to = %s + ) + AND NOT EXISTS ( + SELECT 1 + FROM act.scan_attempt_coverage AS coverage + WHERE coverage.attempt_id = attempt.attempt_id + AND ( + coverage.protocol IS DISTINCT FROM 'tcp' + OR coverage.complete IS DISTINCT FROM TRUE + OR coverage.port_from IS DISTINCT FROM %s + OR coverage.port_to IS DISTINCT FROM %s + ) + ) + ) AS complete_coverage_count, + ( + SELECT count(*) + FROM act.exposure_state AS exposure + WHERE exposure.target_id IN (SELECT target_id FROM exact_targets) + AND exposure.protocol = 'tcp' + AND exposure.port = %s + AND exposure.state = 'open' + ) AS open_exposure_count, + ( + SELECT count(*) + FROM act.findings AS finding + WHERE finding.target_id IN (SELECT target_id FROM exact_targets) + AND finding.protocol = 'tcp' + AND finding.port = %s + AND finding.status = 'open' + AND finding.severity = 'low' + ) AS low_finding_count, + ( + SELECT count(*) + FROM act.findings AS finding + WHERE finding.target_id IN (SELECT target_id FROM exact_targets) + AND finding.status = 'open' + AND finding.severity IN ('high', 'critical') + ) AS unexpected_high_finding_count + """, + ( + scope.account_id, + scope.region, + scope.network_interface_id, + scope.private_ip, + scope.public_ip, + scope.tag_key, + scope.tag_value, + scope.port, + scope.port, + scope.port, + scope.port, + scope.port, + scope.port, + ), + ).fetchone() + if not isinstance(row, Mapping): + raise RuntimeError("managed canary status query returned no row") + counts = { + name: _count(row, name) + for name in ( + "target_count", + "event_count", + "attempt_count", + "complete_coverage_count", + "open_exposure_count", + "low_finding_count", + "unexpected_high_finding_count", + ) + } + ready = ( + counts["target_count"] == 1 + and counts["event_count"] == 1 + and counts["attempt_count"] == 1 + and counts["complete_coverage_count"] == 1 + and counts["open_exposure_count"] == 1 + and counts["low_finding_count"] == 1 + and counts["unexpected_high_finding_count"] == 0 + ) + return { + "operation": "managed-canary-status", + "status": "ready" if ready else "pending", + "ready": ready, + **counts, + } + + +def _scheduled_reconciliation(event: Mapping[str, Any]) -> dict[str, Any]: invocation_key = event.get("id") if not isinstance(invocation_key, str) or not invocation_key: raise ValueError("scheduled event must contain a stable id") + finding_export_enabled = _enabled("FINDING_EXPORT_ENABLED", default=True) finding_bucket = os.getenv("FINDING_BUCKET") - if not finding_bucket: + if finding_export_enabled and not finding_bucket: raise ConfigurationError("FINDING_BUCKET is required") - s3 = boto3.client("s3") with psycopg.connect(database_dsn()) as connection: repository = Repository(connection) result = repository.reconcile_all( invocation_key, finding_bucket=finding_bucket, + queue_finding_handoffs=finding_export_enabled, ) - - publisher = HandoffPublisher(s3, repository) - page_size = min(1000, int(os.getenv("HANDOFF_PAGE_SIZE", "1000"))) - published = publisher.publish_all(keys=None, page_size=page_size) + published = 0 + if finding_export_enabled: + page_size = int(os.getenv("HANDOFF_PAGE_SIZE", "1000")) + if not 1 <= page_size <= _MAX_HANDOFF_PAGE_SIZE: + raise ConfigurationError("HANDOFF_PAGE_SIZE must be between 1 and 1000") + published = HandoffPublisher(boto3.client("s3"), repository).publish_all( + keys=None, + page_size=page_size, + ) return { "run_id": invocation_key, @@ -39,3 +240,11 @@ def lambda_handler(event: Mapping[str, Any], _context: Any) -> dict[str, Any]: "findings_examined": result.findings_examined, "objects_published": published, } + + +def lambda_handler(event: Mapping[str, Any], _context: Any) -> dict[str, Any]: + if event.get("operation") == "managed-canary-status": + scope = _canary_scope() + with psycopg.connect(database_dsn()) as connection: + return _managed_canary_status(connection, scope) + return _scheduled_reconciliation(event) diff --git a/processor/tests/test_handler.py b/processor/tests/test_handler.py index 2228471..22bdc6b 100644 --- a/processor/tests/test_handler.py +++ b/processor/tests/test_handler.py @@ -22,14 +22,20 @@ def test_requires_stable_scheduled_event_id() -> None: def test_runs_full_reconciliation_and_flushes_handoffs( monkeypatch: pytest.MonkeyPatch, ) -> None: - calls: list[tuple[str, str]] = [] + calls: list[tuple[str, str | None, bool]] = [] class FakeRepository: def __init__(self, _connection: object) -> None: pass - def reconcile_all(self, invocation_key: str, *, finding_bucket: str) -> object: - calls.append((invocation_key, finding_bucket)) + def reconcile_all( + self, + invocation_key: str, + *, + finding_bucket: str | None, + queue_finding_handoffs: bool, + ) -> object: + calls.append((invocation_key, finding_bucket, queue_finding_handoffs)) return SimpleNamespace( duplicate=False, findings_examined=3, @@ -62,7 +68,113 @@ def publish_all( result = handler.lambda_handler({"id": "scheduled-run-1"}, None) - assert calls == [("scheduled-run-1", "finding-test-bucket")] + assert calls == [("scheduled-run-1", "finding-test-bucket", True)] assert result["findings_examined"] == 3 assert result["objects_published"] == 5 assert publish_calls == [(None, 2)] + + +def test_database_only_reconciliation_skips_s3_export( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str | None, bool]] = [] + + class FakeRepository: + def __init__(self, _connection: object) -> None: + pass + + def reconcile_all( + self, + invocation_key: str, + *, + finding_bucket: str | None, + queue_finding_handoffs: bool, + ) -> object: + calls.append((invocation_key, finding_bucket, queue_finding_handoffs)) + return SimpleNamespace(duplicate=False, findings_examined=2, handoff_keys=()) + + monkeypatch.setenv("FINDING_EXPORT_ENABLED", "false") + monkeypatch.delenv("FINDING_BUCKET", raising=False) + monkeypatch.setattr(handler, "database_dsn", lambda: "unused") + monkeypatch.setattr(handler.psycopg, "connect", lambda _dsn: _ConnectionContext()) + monkeypatch.setattr(handler, "Repository", FakeRepository) + monkeypatch.setattr( + handler.boto3, + "client", + lambda _name: (_ for _ in ()).throw(AssertionError("S3 client was created")), + ) + + result = handler.lambda_handler({"id": "database-run-1"}, None) + + assert calls == [("database-run-1", None, False)] + assert result["findings_examined"] == 2 + assert result["objects_published"] == 0 + + +def test_managed_canary_status_returns_only_bounded_counts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + row = { + "target_count": 1, + "event_count": 1, + "attempt_count": 1, + "complete_coverage_count": 1, + "open_exposure_count": 1, + "low_finding_count": 1, + "unexpected_high_finding_count": 0, + } + + class Result: + def fetchone(self) -> dict[str, int]: + return row + + class Connection: + def __init__(self) -> None: + self.parameters: tuple[object, ...] | None = None + + def execute(self, _query: str, parameters: tuple[object, ...]) -> Result: + self.parameters = parameters + return Result() + + class ConnectionContext: + def __init__(self, connection: Connection) -> None: + self.connection = connection + + def __enter__(self) -> Connection: + return self.connection + + def __exit__(self, *_args: object) -> None: + return None + + connection = Connection() + environment = { + "MANAGED_CANARY_ACCOUNT_ID": "123456789012", + "MANAGED_CANARY_REGION": "us-east-1", + "MANAGED_CANARY_ENI_ID": "eni-0123456789abcdef0", + "MANAGED_CANARY_PRIVATE_IP": "10.255.255.4", + "MANAGED_CANARY_PUBLIC_IP": "203.0.113.10", + "MANAGED_CANARY_TAG_KEY": "service", + "MANAGED_CANARY_TAG_VALUE": "test-managed-canary", + "MANAGED_CANARY_TCP_PORT": "18080", + } + for name, value in environment.items(): + monkeypatch.setenv(name, value) + monkeypatch.setattr(handler, "database_dsn", lambda: "credentials-must-not-return") + monkeypatch.setattr( + handler.psycopg, + "connect", + lambda _dsn: ConnectionContext(connection), + ) + + result = handler.lambda_handler({"operation": "managed-canary-status"}, None) + + assert result == { + "operation": "managed-canary-status", + "status": "ready", + "ready": True, + **row, + } + assert connection.parameters is not None + assert connection.parameters[-1] == 18080 + assert "credentials-must-not-return" not in repr(result) + assert "203.0.113.10" not in repr(result) diff --git a/pyproject.toml b/pyproject.toml index 719e359..34c87da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [project] name = "portscanner-workspace" -version = "0.1.0" +version = "1.0.0" description = "Public monorepo tooling for the Portscanner project" requires-python = ">=3.12" dependencies = [] @@ -32,13 +32,11 @@ package = false members = [ "contracts", "db/migrator", - "generator*", - "inventory*", - "parser*", - "processor*", - "packages/*", + "generator", + "inventory", + "parser", + "processor", "scanner/nmap", - "services/*", ] [tool.uv.sources] diff --git a/scanner/nmap/Dockerfile b/scanner/nmap/Dockerfile index 9329f4c..9dcd236 100644 --- a/scanner/nmap/Dockerfile +++ b/scanner/nmap/Dockerfile @@ -45,7 +45,7 @@ RUN python -m pip install --no-cache-dir --no-build-isolation --no-deps ./contra rm -rf /opt/build RUN python -m pip uninstall --yes hatchling setuptools trove-classifiers pathspec packaging pluggy -COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/ +COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/ WORKDIR /opt/portscanner-scanner diff --git a/scanner/nmap/README.md b/scanner/nmap/README.md index eca9913..f856f04 100644 --- a/scanner/nmap/README.md +++ b/scanner/nmap/README.md @@ -80,9 +80,7 @@ Supply these as CLI fields or their named environment variables: - `--s3-prefix` / `RESULTS_PREFIX` Equivalent `PORTSCANNER_*` environment names are accepted for operator -integration. The optional compatibility fields `--scan-mode` and -`--service-detection` are validated against the selected profile; they cannot -enable a different or unbounded scan behavior. +integration. `deadline-at` is generator dispatch metadata and does not stop an already dispatched scan. `not-after` is the absolute execution cutoff. Before each Nmap @@ -129,6 +127,8 @@ portscanner-verify \ --target-resource-id '' \ --target-private-address '' \ --target-generation 1 \ + --deadline-at '2030-01-01T00:05:00Z' \ + --not-after '2030-01-01T00:30:00Z' \ --image-version 'sha256:<64-lowercase-hex-characters>' \ --s3-bucket '' \ --s3-prefix verify-results diff --git a/scanner/nmap/pyproject.toml b/scanner/nmap/pyproject.toml index 488eeaf..e9734fa 100644 --- a/scanner/nmap/pyproject.toml +++ b/scanner/nmap/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "hatchling.build" [project] name = "portscanner-scanner" -version = "0.1.0" +version = "1.0.0" description = "Single-target hardened Nmap VERIFY worker" readme = "README.md" requires-python = ">=3.12" diff --git a/scanner/nmap/src/portscanner_scanner/__init__.py b/scanner/nmap/src/portscanner_scanner/__init__.py index e33793f..2e63821 100644 --- a/scanner/nmap/src/portscanner_scanner/__init__.py +++ b/scanner/nmap/src/portscanner_scanner/__init__.py @@ -5,4 +5,4 @@ __all__ = ["__version__"] -__version__ = "0.1.0" +__version__ = "1.0.0" diff --git a/scanner/nmap/src/portscanner_scanner/cli.py b/scanner/nmap/src/portscanner_scanner/cli.py index 201b4d9..05e316a 100644 --- a/scanner/nmap/src/portscanner_scanner/cli.py +++ b/scanner/nmap/src/portscanner_scanner/cli.py @@ -127,16 +127,6 @@ def build_parser() -> argparse.ArgumentParser: default=_first_environment(("SCAN_PORTS", "PORTSCANNER_TCP_PORTS")), help="explicit TCP ports/ranges for targeted-tcp or deep", ) - parser.add_argument( - "--scan-mode", - choices=("fast", "targeted", "deep"), - help="operator compatibility field; must agree with --profile", - ) - parser.add_argument( - "--service-detection", - action="store_true", - help="operator compatibility field valid only for deep", - ) parser.add_argument( "--deep-script", action="append", @@ -301,15 +291,6 @@ def build_parser() -> argparse.ArgumentParser: def _request_from_args(args: argparse.Namespace) -> ScanRequest: - expected_mode = { - ScanProfile.FAST_FULL_TCP.value: "fast", - ScanProfile.TARGETED_TCP.value: "targeted", - ScanProfile.DEEP.value: "deep", - }[args.profile] - if args.scan_mode is not None and args.scan_mode != expected_mode: - raise ValueError("--scan-mode conflicts with --profile") - if args.service_detection and args.profile != ScanProfile.DEEP.value: - raise ValueError("--service-detection is only valid for deep") return ScanRequest( target=args.target, profile=args.profile, diff --git a/scanner/nmap/tests/test_worker_and_cli.py b/scanner/nmap/tests/test_worker_and_cli.py index 3275f2b..5787fdd 100644 --- a/scanner/nmap/tests/test_worker_and_cli.py +++ b/scanner/nmap/tests/test_worker_and_cli.py @@ -392,8 +392,6 @@ def test_cli_runs_with_fake_nmap_and_s3( "targeted-tcp", "--tcp-ports", "80-82", - "--scan-mode", - "targeted", "--event-id", "1" * 64, "--directive-id", diff --git a/terraform/aws/README.md b/terraform/aws/README.md index 4db3813..3aaebee 100644 --- a/terraform/aws/README.md +++ b/terraform/aws/README.md @@ -1,220 +1,220 @@ -# Portable AWS deployment + -This tree deploys a portable, digest-pinned pipeline: +# AWS deployment -`AWS Config or EC2 snapshot + filtered EC2 hints -> inventory Lambda + single-table DynamoDB outbox -> immutable S3 target events -> generator and database projector Lambdas -> private EKS operator/Nmap jobs -> filtered S3 result notification -> parser Lambda -> private Aurora PostgreSQL` +This directory deploys the complete AWS-hosted Portscanner pipeline: -EventBridge events are hints, not inventory truth. Periodic Config snapshots and rescans reconcile missed, duplicated, or reordered events. +`EC2 inventory → DynamoDB outbox → S3/SQS → generator → EKS/Helm/Nmap → S3/SQS → parser → Aurora PostgreSQL` -Inventory state, transactional outbox rows, and signal dedupe markers share one `pk`/`sk` DynamoDB table. Pending outbox rows have no TTL and are indexed for scheduled repair; after immutable S3 publication and both SQS sends succeed, the dispatcher marks a row delivered and starts its seven-day TTL. DynamoDB Streams provides the fast path, while the replay schedule repairs records that outlive the stream's 24-hour retention. Generator claims use a separate `dispatch_id` table. Findings are immutable S3 handoffs; the finding queue notifies external consumers and is not consumed by the scheduled in-stack processor. +The default is a one-target evaluation. Periodic inventory, signal hints, and +finding export remain disabled until explicitly configured. -For the guarded single-account evaluation path, start with -[`docs/getting-started.md`](../../docs/getting-started.md). +## Before you deploy -## Layout +Use a dedicated AWS sandbox and review the plan. Even while scanning is paused, +EKS, Aurora, a NAT gateway, an EC2 canary, logs, and retained data incur +charges. -- `state-bootstrap`: isolated S3/DynamoDB state bootstrap. -- `application`: central application module. -- `member-account`: independently deployable Config, collector role, and filtered event forwarding. -- `modules`: network, storage, database, EKS, ECR, identities, functions, and signal sources. -- `examples`: synthetic roots for created VPC, existing VPC, multi-account central, and member accounts. -- `scripts`: validation and staged deployment helpers. +The bootstrap checks for: -All module sources and chart paths are repository-relative. Terraform never invokes Docker or builds an image. The examples expose names, accounts, networks, subnets, member maps, API client security groups, and installer principals as variables with only synthetic defaults. Override them from ignored private tfvars files before applying; do not edit tracked examples with live values. +- Terraform 1.7.4; +- AWS CLI v2 credentials for the exact deployment account and Region; +- Docker with Buildx, Trivy, `jq`, Python 3, Git, and `tar`; and +- a clean committed checkout so published image digests identify reviewed + source. -## Staged deployment +The deployer needs permission to create the VPC, IAM, EKS, EC2, ECR, Aurora, +S3, SQS, DynamoDB, KMS, Secrets Manager, Lambda, EventBridge, Config, +CloudTrail, CloudWatch, and state resources selected by the environment file. +Use a stable IAM role or user ARN for EKS access, not an STS assumed-role ARN. -Run from a central example root. `TF_BACKEND_CONFIG` points to an S3 backend file containing the bootstrap outputs and no credentials: +## One environment file + +Copy the tracked template: + +```sh +cp terraform/aws/deployment/environment.auto.tfvars.json.example \ + terraform/aws/deployment/environment.auto.tfvars.json +$EDITOR terraform/aws/deployment/environment.auto.tfvars.json +``` + +The ignored JSON file is the only user-maintained deployment configuration. +It contains no credentials. Set: + +- the environment name, AWS account, and Region; +- the stable EKS installer principal and the Terraform runner's public `/32`; +- VPC, architecture, capacity, rate, and retention policy; +- whether the managed canary is present; and +- explicit integration account, CIDR, ENI class, and opt-in tag scope. + +AWS credentials stay in the normal CLI/provider credential chain. Database +passwords are RDS/Secrets Manager managed. Backend and immutable image inputs +are generated under ignored `.portscanner/` storage with mode `0600`. + +## Deploy and evaluate + +Run: ```sh -export TF_BACKEND_CONFIG="$PWD/backend.hcl" -export TF_ROOT="terraform/aws/examples/created-vpc" -export PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID="REPLACE_WITH_12_DIGIT_ACCOUNT_ID" -export PORTSCANNER_EXPECTED_AWS_REGION="us-east-1" -terraform/aws/scripts/deploy.sh "${TF_ROOT}" foundation +./terraform/aws/scripts/bootstrap.sh +./terraform/aws/scripts/deploy.sh evaluate ``` -The helper accepts one or more variable files, verifies the live STS account, forces the -expected account and Region into a temporary saved plan, and requires typing the stage -name before applying that exact plan. It removes the potentially sensitive binary plan -afterward. Controlled automation must explicitly set `PORTSCANNER_AUTO_APPROVE=true`; -the script rejects targeted, destroy, replacement, and refresh-only arguments. +Bootstrap: -The infrastructure progression is explicit; the pause stages reverse only dispatch: +1. verifies tools, the live STS account/Region, configuration shape, and clean + source; +2. creates or verifies an encrypted S3 backend and DynamoDB lock table; +3. applies a saved, explicitly confirmed foundation plan with dispatch off; +4. builds all seven images for one architecture, scans every immutable digest, + and pushes them to the environment's ECR repositories; and +5. writes generated digest and migration-checksum inputs atomically. -1. `foundation`: runtime, migration, Helm, and dispatch are disabled. -2. Build and scan all seven images in an external pipeline for the selected architecture, push them to the output ECR repositories, and record immutable `sha256:` digests. -3. `runtime`: create digest-pinned Lambda functions with mappings and schedules paused. -4. `migrate`: invoke the migrator keyed by its image digest plus migration checksum, then install the Helm release. -5. `canary`: enable queue/stream processing and durable outbox replay while automatic inventory schedules and EventBridge rules remain paused. -6. `pause-canary`: stop the canary pipeline while retaining its fail-closed one-target boundary. -7. `activate`: explicitly enable recurring schedules and filtered EventBridge rules. -8. `pause`: retain runtime, operator, and evidence while disabling mappings, schedules, - and filtered EventBridge rules again. +It is resumable and refuses state, backend, account, Region, or image +collisions. -The helper rejects a pause plan if it contains migration, image, Helm, or other changes -outside dispatch mappings and rules. +Evaluation then: -When EKS or Helm refresh is unavailable, `scripts/emergency-pause.sh ` -reads exact controls from state and disables the managed Lambda event-source mappings -and EventBridge rules directly through AWS APIs. It does not plan, contact Kubernetes, -or terminate scanner Jobs already running. +1. creates the digest-pinned Lambda runtime with event sources paused; +2. runs the checksum-locked database baseline and installs the Helm operator; +3. enables only the one-target canary path; +4. snapshots the Terraform-owned EC2 ENI and scans only TCP 18080; +5. verifies one complete attempt, one open exposure, and one low PostgreSQL + finding; +6. replays the snapshot to prove one-event/one-attempt idempotency; and +7. returns all dispatch and reconciliation controls to `pause-canary`. -After the foundation apply, validate the image mapping without contacting AWS or -building anything: +Every Terraform apply still uses a displayed saved plan and an exact +confirmation. A single shell command does not mean hidden approval. + +The canary is a tiny instance/EIP in a separate, unpeered VPC. It has no SSH +key, instance role, or general ingress/egress. TCP 18080 accepts traffic only +from the scanner NAT EIP. `localhost`, private addresses, and documentation +addresses are not runnable targets. + +To install or repair the runtime without scanning: ```sh -terraform/aws/scripts/build-images.sh --dry-run "${TF_ROOT}" arm64 +./terraform/aws/scripts/deploy.sh ready +``` + +## Add AWS inventory + +After evaluation, edit the same JSON: + +1. add exact `authorized_account_ids` and `allowed_target_cidrs`; +2. retain deny-before-allow CIDR policy; +3. select supported ENI interface types; +4. require an opt-in ENI tag, such as `application=portscanner`; and +5. set `recurring_inventory_enabled` to `true`. + +Direct EC2 snapshots are the simplest authoritative source: + +```json +"config_mode": "disabled", +"snapshot_regions": ["us-east-1"] ``` -The dry run still requires `git`, `terraform`, the AWS CLI, Docker buildx, `jq`, Python -3, and `tar` to be installed, and it reads the applied `repository_urls` and -`deployment_state` outputs. It accepts only `application`, `created-vpc`, -`existing-vpc`, and `multi-account-central`; all export both outputs. It explicitly -rejects `member-account`. Valid foundation and later deployment states are accepted so -the same helper publishes upgrade images. Dry run makes no AWS identity call, registry -login, build, or push, and emits no digest input. A normal build additionally requires -a configured AWS Region and an authenticated identity in the exact account and Region -owning the output repositories, plus Trivy for the mandatory release-digest scan. - -Build and push from a clean, reviewed checkout. The helper combines the checked-out -commit (or reviewed Git tag) with `arm64`/`x86_64`, so immutable ECR tags cannot be -reused across incompatible architectures. Redirect stdout to a private, non-committed -file; all progress is written to stderr: +Then review and activate: ```sh -SOURCE_REVISION="$(git rev-parse HEAD)" -terraform/aws/scripts/build-images.sh \ - "${TF_ROOT}" arm64 "${SOURCE_REVISION}" \ - >"$HOME/portscanner-image-inputs.tfvars" +./terraform/aws/scripts/deploy.sh activate ``` -The optional revision/tag must be Docker-tag safe, cannot be `latest`, and must resolve -to the checked-out `HEAD`. `PORTSCANNER_ALLOW_DIRTY=true` is accepted only with -`--dry-run`; deployment inputs always require a clean checkout. The helper -authenticates once per exact ECR registry using `aws ecr get-login-password`. Before -each build it checks the immutable commit tag: an existing valid digest is reused, while -a successful tagged-image listing that confirms absence is built and pushed. This makes -a partially completed seven-image publication safe to resume without overwriting -immutable tags. Any lookup error or malformed result fails closed. The helper removes -its temporary Docker authentication configuration and does not run Terraform init, -plan, or apply. It scans every reused or newly pushed immutable digest for fixable -HIGH/CRITICAL vulnerabilities and emits no Terraform input if any scan fails. - -Inventory, generator, parser, processor, migrator, and scanner all use the repository -root as their build context. Only operator uses `operator/` as its context. The helper -uses `--pull`, the selected platform, and all seven exact Dockerfiles. It attaches -provenance and SBOM manifests only where the target runtime accepts them, as described -below. - -Review the captured file before supplying it to the `runtime` and later stages. The -helper prints this deterministic shape only after every push and digest lookup succeeds; -it never creates a variable file itself: - -```hcl -image_digests = { - inventory = "sha256:<64 lowercase hex characters>" - generator = "sha256:<64 lowercase hex characters>" - parser = "sha256:<64 lowercase hex characters>" - processor = "sha256:<64 lowercase hex characters>" - migrator = "sha256:<64 lowercase hex characters>" - operator = "sha256:<64 lowercase hex characters>" - scanner = "sha256:<64 lowercase hex characters>" -} - -migration_checksum = "<64 lowercase hex characters>" +New and changed targets use explicit full-TCP policy. Do not broaden account, +CIDR, tag, rate, Pod, or Job limits without authorization and traffic review. +Periodic snapshots are authoritative; signal events only move likely changes +ahead of baseline coverage. + +## Optional integrations + +- **AWS Config:** choose `config_mode=create` or supply a reviewed existing + aggregator. Existing mode cannot prove source accounts, Regions, recorder + health, or freshness. +- **Signal hints:** set `signal_hints_enabled=true` and choose a reviewed + CloudTrail mode. EventBridge/CloudTrail signals always resolve through a + provider reread before dispatch. +- **Finding export:** set `finding_export_enabled=true` to add the versioned S3 + bucket, SQS/DLQ notification, IAM, and alarms. SQS messages point to S3 + objects; consumers must follow [the contract guide](../../contracts/README.md). +- **Multi-account:** use `examples/multi-account-central` plus + `member-account`. Member roles trust exact central principals and an + external ID; each hot-path Region needs its own reviewed forwarding root. +- **Existing VPC:** `examples/existing-vpc` validates subnet ownership, AZ + spread, DNS, egress, endpoint identity/private DNS, and endpoint security + groups. The canonical quickstart intentionally creates networking instead. + +GCP, Azure, private-address, and IPv6 inventory are not production deployment +options in this release. + +## Pause and emergency stop + +Normal pause keeps infrastructure and evidence: + +```sh +./terraform/aws/scripts/deploy.sh pause +``` + +A pause plan is accepted only when it disables managed event-source mappings +and EventBridge rules without changing images, migrations, Helm, or other +resources. + +If Terraform or EKS refresh is unavailable: + +```sh +./terraform/aws/scripts/emergency-pause.sh terraform/aws/deployment +``` + +Emergency pause uses exact controls from Terraform state. Neither pause method +terminates an already-created Kubernetes Job. Inspect and cancel active +Scanner/Job resources separately. + +## Retire the canary + +Pause first, wait for the pipeline to drain, set +`environment.managed_canary.enabled=false`, then run: + +```sh +./terraform/aws/scripts/deploy.sh retire-canary +``` + +Retirement requires `kubectl`. It verifies all queues twice, requires every +Scanner and scanner Job to be terminal, proves no environment field other than +the canary switch changed, restricts the saved plan to the reviewed canary +boundary, and only then releases the EIP. Never release an EIP while queued or +active work can still reference it. + +## Destruction and retained state + +`destroy` is available only when the environment explicitly sets both +`disposable=true` and `destroy_data_on_teardown=true`: + +```sh +./terraform/aws/scripts/deploy.sh destroy +``` + +It pauses first, displays a saved destroy plan, and requires typing +`destroy `. An installed deployment must also pass the same +queue, Scanner, and Job retirement-readiness checks before planning. State +bootstrap resources are deliberately left behind. Retained/non-disposable +environments should use deletion protection, +PITR, resilient nodes/database instances, one NAT per AZ, durable retention, +and a separately reviewed retirement procedure with a final database snapshot. + +## Development + +Terraform never builds images with provisioners or uses `-target`. Validate +the roots, modules, stage scripts, Helm chart, and generated artifacts from the +repository root: + +```sh +make terraform-script-tests +make terraform-validate +make kubernetes-validate +make containers ``` -The migration checksum is SHA-256 over the complete, strictly named -`_..sql` set directly in `db/migrations`, sorted by UTF-8 file -name. Any other file, directory, or symlink is rejected. Each file name and content is -length-prefixed with an eight-byte big-endian length before hashing, so both affect the -result without ambiguous concatenation. - -The EKS API is private by default and must be reachable from the machine running the -migration/install stage, for example through an approved VPN or build runner in the VPC. -Supply that runner or VPN security group through -`eks_api_client_security_group_ids`; Terraform grants port 443 by security-group -reference. A disposable evaluation may instead set -`eks_endpoint_public_access = true` with one or more restricted canonical IPv4 prefixes -in `eks_public_access_cidrs`; private access remains enabled and `0.0.0.0/0` is rejected. -Supply stable IAM role/user ARNs through `eks_installer_principal_arns`; Terraform -creates EKS access entries and cluster-scoped `AmazonEKSClusterAdminPolicy` -associations. Bootstrap creator admin is disabled by default and does not replace them. -The runner must have the AWS CLI because Helm refreshes credentials with -`aws eks get-token`; no plan-cached token is used. The generator Lambda's private -runtime security group is allowed automatically. - -## Image architecture - -The deployment deliberately uses one architecture for Lambda images, EKS nodes, the operator, and scanner. Passing `arm64` to the helper selects `linux/arm64` and requires the applied `workload_architecture.architecture` to be `arm64`; passing `x86_64` selects `linux/amd64` and requires `x86_64`. The emitted input preserves the applied architecture, AMI type, and exact node-instance list. Terraform rejects a Lambda/EKS architecture mismatch and queries EC2 instance-type metadata to reject node types that do not support the selected architecture. - -For every reused or newly pushed digest, the helper also inspects remote Buildx image -metadata and requires exactly the selected Linux runtime platform before scanning or -emitting Terraform input. - -Terraform only creates repositories and consumes digests. A typical external builder passes `--platform linux/arm64` (or `linux/amd64`) to BuildKit, verifies the image manifest architecture, scans the immutable artifact, and supplies the resulting digest. Lambda base images, the Go operator build, and the Debian scanner image all support explicit platform builds. - -EKS defaults to Kubernetes 1.36 and also accepts 1.35. Those versions keep the -operator's Kubernetes 1.36 client libraries within supported minor-version skew. - -The helper disables attached BuildKit provenance/SBOM manifests for the five Lambda -images so ECR stores the single-architecture image manifest Lambda expects. It enables -attached attestations for the operator and scanner images when BuildKit supports them. -Generate and retain separate SBOM/provenance artifacts for Lambda images in release CI; -do not change their deployed ECR digest into an attestation-bearing OCI index. - -## Networking and production overrides - -Created networking has public NAT subnets, private Lambda/EKS subnets, isolated Aurora subnets, and S3/DynamoDB gateway endpoints. Its canonical IPv4 VPC CIDR must be `/16` through `/24`, because the module adds four subnet bits and AWS subnets cannot be narrower than `/28`. The low-volume default uses one NAT gateway, one Spot node, one Aurora instance, conservative Lambda concurrency, short data retention, and no DynamoDB PITR. Production should normally override: - -- `nat_gateway_mode = "one_per_az"` -- `node_capacity_type = "ON_DEMAND"` and a resilient node range -- `database_instance_count >= 2` -- `database_deletion_protection = true` -- `database_skip_final_snapshot = false` -- `dynamodb_point_in_time_recovery = true` -- backup, log, object, and queue retention -- reserved concurrency and schedule rates after load testing - -Existing mode requires both VPC DNS support and DNS hostnames, and validates VPC membership, AZ spread, public-IP settings, absence of direct internet-gateway routes on private subnets, and absence of default routes on isolated subnets. It refuses to proceed without `existing_private_subnet_egress_mode`. NAT and transit modes require a matching default route from every private subnet. Endpoint mode requires explicit endpoint IDs for Config, DynamoDB, EC2, ECR API/DKR, EKS, EKS Auth, Logs, S3, Secrets Manager, SQS, and STS. Terraform verifies endpoint VPC, availability, expected service, interface private DNS, and S3/DynamoDB route-table associations. China endpoint names are validated per service because gateway and interface services use a mix of `com.amazonaws` and `cn.com.amazonaws` prefixes. Each interface endpoint also requires an explicitly selected attached security group; Terraform adds TCP/443 ingress from the application Lambda and EKS workload security groups. Review endpoint policies separately. - -SQS visibility defaults to 360 seconds and must remain at least six times the queue-triggered Lambda timeout. - -## Multi-account controls - -The central custom bus policy uses an exact account allowlist unless an optional organization ID is supplied. No Organizations API or organization membership is required by default. Member accounts trust one exact central collector role with an external ID and grant only `ec2:Describe*`; event forwarding grants only `events:PutEvents` to one central bus ARN. - -For direct member inventory, apply the central foundation first, use its `central_collector_principal_arns` output as each member's exact trusted principals, then feed each member's `collector_role_arn` output and paired external ID back through the central account-keyed maps. When any non-local scope is authorized, provide a collector entry for every authorized scope. Use one deterministic collector role path/name and one external ID across those accounts; Terraform derives the runtime's `{account_id}` role template while IAM remains restricted to the exact resulting ARNs. The snapshot and signal Lambdas may assume only those roles. - -AWS Config can be created, disabled in favor of direct EC2 collection, or supplied as an existing aggregator. Snapshot schedules carry an explicit account scope. Create mode provisions its same-account source authorization and scopes the aggregator to the one provider Region where it creates a recorder; it never claims all-Region coverage from that recorder. Every member account in that same source Region must separately authorize the exact central account and aggregator Region before central apply. Other Config source Regions require direct EC2 snapshots or an externally provisioned existing aggregator. An existing aggregator is referenced by name only: Terraform cannot prove its source accounts, authorizations, recording regions, resource coverage, or freshness. Config is eventually consistent and only returns resources recorded in its configured sources, so retain periodic reconciliation and use the member collector role path where direct EC2 reads are required. - -Central management API-call forwarding supports `cloudtrail_mode = "create"`, -`"existing"`, or `"disabled"`. Create mode provisions a multi-region trail; existing -mode references a supplied member/organization management trail; disabled mode omits -CloudTrail API-call hints while retaining native EC2 state-change hints. An ARN-only -reference cannot prove selectors, logging health, or multi-region setting, so verify -those live before activation. CloudTrail EventBridge rules exactly match the inventory -runtime's EC2 API allowlist. EventBridge is regional: the application `signal_region` -and each member `signal_region` output identify the single hot-path Region covered by -that module instance. Deploy uniquely named forwarding roots per additional Region and -rely on authoritative snapshots elsewhere. All rules and runtime event mappings remain -disabled until activation. - -## Cost and destruction warning - -Even idle environments incur charges for NAT gateways, Aurora, EKS, worker nodes, CloudTrail delivery, logs, and retained data. Review the plan and AWS pricing before applying. - -Destruction is deliberately non-trivial: data buckets and ECR repositories do not -force-delete by default, Aurora production safeguards are opt-in variables, and -bootstrap state resources have `prevent_destroy`. ECR count expiration is disabled; -optional lifecycle expiration affects only untagged images. Tagged active and rollback -digests are release-operator-managed and must never be expired automatically. Drain -queues, export findings, retain a final database snapshot, empty retained object -versions, and remove bootstrap lifecycle protection only through a separately reviewed -retirement change. - -Only disposable test environments should set `force_destroy_buckets = true`; this is -needed for unattended cleanup after Config or CloudTrail has written versioned objects. -They may also set `force_delete_repositories = true` so an explicit Terraform destroy -can remove the evaluation images. Production must leave both settings false. +No automated test scans a live address. diff --git a/terraform/aws/application/README.md b/terraform/aws/application/README.md deleted file mode 100644 index 48936e8..0000000 --- a/terraform/aws/application/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Central application module - -This module composes the portable AWS foundation and runtime. Use it from a root under `../examples` so the root configures the AWS provider without a developer profile. - -The six deployment switches default to false: - -- `deploy_runtime` -- `run_migration` -- `install_operator` -- `enable_event_dispatch` -- `enable_automatic_inventory` -- `canary_mode` - -Checks reject skipped stages, missing image digests, malformed migration checksums, empty activation scope, and automatic inventory without a live dispatch pipeline. Central-bus authorization is required when automatic EventBridge inventory is enabled; a direct assumed-role canary does not require that unrelated signal path. Lambda mappings and schedules are created disabled during the runtime stage. The canary stage enables queue/stream processing plus durable outbox repair; full activation separately enables automatic inventory schedules and EventBridge hints. Terraform sends the expected migration checksum to the keyed `aws_lambda_invocation` and requires the response to return the same verified checksum before the Helm release can consume its completion token. - -The application secret is created without a value. The migrator reads the RDS-managed master secret, creates or rotates the least-privilege database user, and writes only the application credential to that secret. Secret material is not supplied as a Terraform variable. - -Runtime wiring uses: - -- one inventory table for target state, transactional outbox entries, and signal dedupe, plus one dispatch/idempotency table; -- signal, priority, coverage, target-event, result, and external finding queues, each with a DLQ; -- an inventory-stream outbox Lambda that writes immutable target events, routes their S3 notification envelopes, marks successful rows delivered, and schedules sparse-index replay for stream-retention outages; -- a target projector and result parser with private Aurora access; -- a scheduled finding reconciliation processor with no SQS event source; and -- the repository Helm chart at `operator/chart/portscanner`, with Scanner-only generator RBAC owned by that chart. - -The results notification is restricted to `results/**/scan-result.json`. Finding notifications are restricted to `findings/` and are intended for an external handoff consumer. Every PostgreSQL runtime reads only the migrator-populated application secret; only the migrator can read the master secret or write the application secret. - -The module exposes the external finding queue URL/ARN and finding bucket name/ARN for a -least-privilege downstream consumer. Created networking also exposes the stable scanner -NAT egress addresses so an authorized canary can restrict its ingress during evaluation. - -When `config_mode` is `create` or `existing`, snapshot schedules query the configured aggregator with explicit account scopes. Create mode authorizes its local source and aggregates only the provider Region containing its one recorder; member accounts in that same source Region need separate authorizations. Other Config source Regions require direct EC2 snapshots or an externally provisioned existing aggregator. `disabled` selects direct EC2 collection and requires `snapshot_regions` (the deployment region is the default). A referenced existing aggregator is not introspected, so its account authorizations, region coverage, recorder state, and freshness remain live deployment prerequisites. - -The Helm runner needs both network and identity authorization. By default, -`eks_api_client_security_group_ids` opens only security-group-referenced private API -access. A disposable evaluation may instead enable the public endpoint with restricted -`eks_public_access_cidrs`; unrestricted access is rejected and private access remains -enabled. `eks_installer_principal_arns` creates EKS ClusterAdmin access entries. -Bootstrap creator admin defaults off. Helm uses AWS CLI exec tokens, so migrate/install -runners need a working `aws eks get-token` credential path. - -Existing VPCs must declare and satisfy NAT gateway, transit gateway, or validated -AWS-service endpoint egress, with VPC DNS support and DNS hostnames enabled. Endpoint -mode supports only paused infrastructure; scanner dispatch requires NAT/TGW public -egress and fails closed otherwise. NAT routes must match explicitly supplied public NAT -gateway IDs; TGW egress needs a reviewed acknowledgement because downstream TGW -attachments and active/non-blackhole route state cannot be proven from this root. -Endpoint mode requires explicit endpoint IDs and -attached interface endpoint security groups; -Terraform validates partition-aware endpoint identity, state, private DNS, gateway -routes, and manages workload TLS ingress. Queue visibility is at least six times Lambda -timeout. -ECR lifecycle expiration defaults off and, when enabled, affects only untagged images; -tagged release retention remains operator-managed. - -Cost and destruction warning: the foundation includes NAT, EKS, a managed node group, -Aurora, S3, SQS, DynamoDB, Config/CloudTrail, ECR, and logs. Production must enable -database deletion protection, final snapshots, DynamoDB PITR, resilient -NAT/nodes/database instances, and policy-appropriate retention. Buckets and ECR -repositories do not force-delete by default; the opt-in cleanup switches are only for -disposable evaluations. diff --git a/terraform/aws/application/main.tf b/terraform/aws/application/main.tf index 07f7f30..00c6646 100644 --- a/terraform/aws/application/main.tf +++ b/terraform/aws/application/main.tf @@ -15,7 +15,11 @@ locals { snapshot_regions = length(var.snapshot_regions) > 0 ? var.snapshot_regions : [ data.aws_region.current.region ] - operator_chart_path = abspath("${path.module}/../../../operator/chart/portscanner") + periodic_snapshots_enabled = var.periodic_snapshots_enabled + periodic_coverage_enabled = var.periodic_coverage_enabled + signal_hints_enabled = var.signal_hints_enabled + processor_reconciliation_enabled = var.processor_reconciliation_enabled + operator_chart_path = abspath("${path.module}/../../../operator/chart/portscanner") cloudtrail_arn = ( var.cloudtrail_mode == "create" ? @@ -44,6 +48,52 @@ locals { var.allowed_member_account_ids, toset([data.aws_caller_identity.current.account_id]) ) + managed_canary_scanner_source_cidrs = concat( + [for address in module.network.created_nat_public_ips : "${address}/32"], + [ + for address in sort(tolist(var.managed_canary_scanner_source_ipv4s)) : + "${address}/32" + ] + ) + managed_canary = var.managed_canary_enabled ? { + account_id = data.aws_caller_identity.current.account_id + region = data.aws_region.current.region + network_interface_id = module.managed_canary[0].network_interface_id + private_ip = module.managed_canary[0].private_ip + public_ip = module.managed_canary[0].public_ip + tag_key = module.managed_canary[0].inventory_tag_key + tag_value = module.managed_canary[0].inventory_tag_value + tcp_port = module.managed_canary[0].listener_port + } : null + effective_authorized_account_ids = ( + var.managed_canary_enabled && var.canary_mode ? + toset([data.aws_caller_identity.current.account_id]) : + var.authorized_account_ids + ) + effective_allowed_target_cidrs = ( + var.managed_canary_enabled && var.canary_mode ? + toset([module.managed_canary[0].public_cidr]) : + var.allowed_target_cidrs + ) + effective_allowed_tag_keys = setunion( + var.allowed_tag_keys, + var.managed_canary_enabled ? toset(["service"]) : toset([]) + ) + effective_allowed_eni_interface_types = ( + var.managed_canary_enabled && var.canary_mode ? + toset(["interface"]) : + var.inventory_allowed_eni_interface_types + ) + effective_required_target_tag_key = ( + var.managed_canary_enabled && var.canary_mode ? + module.managed_canary[0].inventory_tag_key : + var.inventory_required_target_tag_key + ) + effective_required_target_tag_value = ( + var.managed_canary_enabled && var.canary_mode ? + module.managed_canary[0].inventory_tag_value : + var.inventory_required_target_tag_value + ) } resource "terraform_data" "deployment_stage_validation" { @@ -101,14 +151,25 @@ resource "terraform_data" "deployment_stage_validation" { var.deploy_runtime && var.run_migration && var.install_operator && - length(var.authorized_account_ids) > 0 + length(local.effective_authorized_account_ids) > 0 ) error_message = "Activation requires runtime, migration, operator installation, and explicit authorized_account_ids." } precondition { - condition = !var.enable_automatic_inventory || var.enable_event_dispatch - error_message = "Automatic inventory requires the queue and stream dispatch pipeline." + condition = !( + local.periodic_snapshots_enabled || + local.periodic_coverage_enabled || + local.signal_hints_enabled + ) || var.enable_event_dispatch + error_message = "Periodic snapshots, coverage, and signal hints require the queue and stream dispatch pipeline." + } + + precondition { + condition = !local.processor_reconciliation_enabled || ( + var.deploy_runtime && var.run_migration + ) + error_message = "Processor reconciliation requires deployed, migrated runtime." } precondition { @@ -116,13 +177,44 @@ resource "terraform_data" "deployment_stage_validation" { var.deploy_runtime && var.run_migration && var.install_operator && - !var.enable_automatic_inventory && - length(var.authorized_account_ids) > 0 && - length(var.allowed_target_cidrs) == 1 && - can(regex("/32$", try(one(var.allowed_target_cidrs), ""))) + !local.periodic_snapshots_enabled && + !local.periodic_coverage_enabled && + !local.signal_hints_enabled && + length(local.effective_authorized_account_ids) > 0 && + length(local.effective_allowed_target_cidrs) == 1 && + can(regex("/32$", try(one(local.effective_allowed_target_cidrs), ""))) ) error_message = "canary_mode requires installed migrated runtime, explicit account scope, one exact /32 target allowlist, and automatic inventory disabled." } + + precondition { + condition = !var.managed_canary_enabled || ( + length(local.managed_canary_scanner_source_cidrs) > 0 + ) + error_message = "managed_canary_enabled requires at least one scanner NAT EIP." + } + + precondition { + condition = !(var.managed_canary_enabled && var.canary_mode) ? true : alltrue([ + for cidr in var.denied_target_cidrs : + cidrhost( + "${module.managed_canary[0].public_ip}/${split("/", cidr)[1]}", + 0 + ) != split("/", cidr)[0] + ]) + error_message = "The managed canary EIP must not be included by denied_target_cidrs." + } + + precondition { + condition = ( + var.inventory_required_target_tag_key == null && + var.inventory_required_target_tag_value == null + ) || ( + try(length(var.inventory_required_target_tag_key) > 0, false) && + try(length(var.inventory_required_target_tag_value) > 0, false) + ) + error_message = "inventory_required_target_tag_key and inventory_required_target_tag_value must be configured together." + } } } @@ -139,9 +231,11 @@ resource "terraform_data" "authorized_scope_validation" { } precondition { - condition = !var.enable_automatic_inventory || var.allowed_organization_id != null || alltrue([ - for account_id in var.authorized_account_ids : - contains(local.bus_authorized_account_ids, account_id) + condition = !( + local.periodic_snapshots_enabled || local.signal_hints_enabled + ) || var.allowed_organization_id != null || alltrue([ + for account_id in local.effective_authorized_account_ids : + contains(local.bus_authorized_account_ids, account_id) ]) error_message = "Every activated account scope must be the local account or an account authorized by the central bus." } @@ -195,6 +289,17 @@ module "network" { scanner_public_egress_required = var.enable_event_dispatch } +module "managed_canary" { + count = var.managed_canary_enabled ? 1 : 0 + source = "../modules/managed-canary" + + name_prefix = local.name + vpc_cidr = var.managed_canary_vpc_cidr + scanner_source_ipv4_cidrs = local.managed_canary_scanner_source_cidrs + instance_type = var.managed_canary_instance_type + listener_port = 18080 +} + module "storage" { source = "../modules/storage" @@ -208,8 +313,10 @@ module "storage" { bucket_expiration_days = var.bucket_expiration_days force_destroy_buckets = var.force_destroy_buckets dynamodb_point_in_time_recovery = var.dynamodb_point_in_time_recovery - cloudtrail_source_arns = compact([local.cloudtrail_arn]) + cloudtrail_source_arns = var.cloudtrail_mode == "create" ? compact([local.cloudtrail_arn]) : [] enable_config_delivery = var.config_mode == "create" + finding_export_enabled = var.finding_export_enabled + enable_cloudtrail_storage = var.cloudtrail_mode == "create" signal_event_rule_arns = local.signal_rule_arns } @@ -245,19 +352,19 @@ module "signals" { name_prefix = local.name signal_queue_arn = module.storage.queue_arns["signal"] signal_dead_letter_queue_arn = module.storage.dead_letter_queue_arns["signal"] - enable_event_dispatch = var.enable_automatic_inventory + enable_event_dispatch = local.signal_hints_enabled config_mode = var.config_mode config_delivery_bucket_name = module.storage.bucket_names["events"] existing_config_aggregator_name = var.existing_config_aggregator_name cloudtrail_mode = var.cloudtrail_mode existing_cloudtrail_arn = var.existing_cloudtrail_arn cloudtrail_name = local.cloudtrail_name - cloudtrail_bucket_name = module.storage.bucket_names["cloudtrail"] + cloudtrail_bucket_name = try(module.storage.bucket_names["cloudtrail"], null) create_central_event_bus = var.create_central_event_bus allowed_member_account_ids = var.allowed_member_account_ids allowed_organization_id = var.allowed_organization_id - authorized_account_ids = var.authorized_account_ids - config_aggregator_account_ids = var.authorized_account_ids + authorized_account_ids = local.effective_authorized_account_ids + config_aggregator_account_ids = local.effective_authorized_account_ids depends_on = [module.storage] } @@ -275,6 +382,7 @@ module "identities" { eks_cluster_arn = local.eks_cluster_arn member_collector_role_arns = values(var.member_collector_role_arns) target_event_object_prefix = "target-events/aws/" + finding_export_enabled = var.finding_export_enabled } module "functions" { @@ -284,7 +392,11 @@ module "functions" { deploy_runtime = var.deploy_runtime run_migration = var.run_migration enable_event_dispatch = var.enable_event_dispatch - enable_automatic_inventory = var.enable_automatic_inventory + periodic_snapshots_enabled = local.periodic_snapshots_enabled + periodic_coverage_enabled = local.periodic_coverage_enabled + signal_hints_enabled = local.signal_hints_enabled + processor_reconciliation_enabled = local.processor_reconciliation_enabled + finding_export_enabled = var.finding_export_enabled canary_mode = var.canary_mode image_digests = var.image_digests repository_urls = module.repositories.repository_urls @@ -305,11 +417,16 @@ module "functions" { snapshot_backend = local.snapshot_backend snapshot_account_id = data.aws_caller_identity.current.account_id snapshot_regions = local.snapshot_regions - allowed_tag_keys = var.allowed_tag_keys - allowed_target_cidrs = var.allowed_target_cidrs + allowed_tag_keys = local.effective_allowed_tag_keys + allowed_eni_interface_types = local.effective_allowed_eni_interface_types + required_target_tag_key = local.effective_required_target_tag_key + required_target_tag_value = local.effective_required_target_tag_value + snapshot_max_pages = var.snapshot_max_pages + managed_canary = local.managed_canary + allowed_target_cidrs = local.effective_allowed_target_cidrs denied_target_cidrs = var.denied_target_cidrs target_event_prefix = "target-events/aws" - authorized_account_ids = var.authorized_account_ids + authorized_account_ids = local.effective_authorized_account_ids member_collector_role_arns = var.member_collector_role_arns member_collector_external_ids = var.member_collector_external_ids eks_cluster_name = local.cluster_name @@ -350,7 +467,7 @@ module "eks" { existing_interface_endpoint_security_group_ids = module.network.existing_interface_endpoint_security_group_ids installer_principal_arns = var.eks_installer_principal_arns scanner_pod_role_arn = module.identities.scanner_pod_role_arn - allowed_target_cidrs = var.allowed_target_cidrs + allowed_target_cidrs = local.effective_allowed_target_cidrs denied_target_cidrs = var.denied_target_cidrs operator_max_concurrent_reconciles = var.operator_max_concurrent_reconciles scanner_max_concurrent_pods = var.scanner_max_concurrent_pods diff --git a/terraform/aws/application/main.tftest.hcl b/terraform/aws/application/main.tftest.hcl new file mode 100644 index 0000000..c3d04db --- /dev/null +++ b/terraform/aws/application/main.tftest.hcl @@ -0,0 +1,89 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_partition" { + defaults = { + partition = "aws" + } + } + + mock_data "aws_region" { + defaults = { + region = "us-east-1" + } + } + + mock_data "aws_availability_zones" { + defaults = { + names = ["us-east-1a", "us-east-1b"] + } + } + + mock_data "aws_ec2_instance_type" { + defaults = { + supported_architectures = ["arm64"] + } + } + + mock_data "aws_ami" { + defaults = { + id = "ami-0123456789abcdef0" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +mock_provider "helm" {} + +run "database_boundary_has_nullable_integration_outputs" { + command = plan + + variables { + config_mode = "disabled" + cloudtrail_mode = "disabled" + } + + assert { + condition = ( + output.finding_queue_url == null && + output.finding_queue_arn == null && + output.finding_bucket_name == null && + output.finding_bucket_arn == null && + output.cloudtrail_arn == null && + output.managed_canary_public_ip == null && + output.managed_canary_status_function_arn == null && + !contains(keys(output.bucket_names), "findings") && + !contains(keys(output.bucket_names), "cloudtrail") + ) + error_message = "The default PostgreSQL boundary must expose safe nulls for disabled integrations." + } +} + +run "managed_canary_composes_after_scanner_network" { + command = plan + + variables { + config_mode = "disabled" + cloudtrail_mode = "disabled" + managed_canary_enabled = true + } + + assert { + condition = ( + output.managed_canary.listener_port == 18080 && + output.managed_canary.inventory_tag_key == "service" && + output.managed_canary_snapshot_invocation.payload.operation == "managed-canary" && + output.managed_canary_status_invocation.payload.operation == "managed-canary-status" + ) + error_message = "Application composition must expose only the trusted canary and status operations." + } +} diff --git a/terraform/aws/application/outputs.tf b/terraform/aws/application/outputs.tf index 8101d79..b40512f 100644 --- a/terraform/aws/application/outputs.tf +++ b/terraform/aws/application/outputs.tf @@ -56,22 +56,22 @@ output "target_event_queue_url" { output "finding_queue_url" { description = "External finding handoff notification queue; no in-stack Lambda consumes it." - value = module.storage.queue_urls["finding"] + value = module.storage.finding_queue_url } output "finding_queue_arn" { description = "ARN of the external finding handoff queue for consumer IAM policies." - value = module.storage.queue_arns["finding"] + value = module.storage.finding_queue_arn } output "finding_bucket_name" { description = "Bucket containing immutable finding documents referenced by finding queue notifications." - value = module.storage.bucket_names["findings"] + value = module.storage.finding_bucket_name } output "finding_bucket_arn" { description = "ARN of the immutable finding bucket for consumer IAM policies." - value = module.storage.bucket_arns["findings"] + value = module.storage.finding_bucket_arn } output "repository_urls" { @@ -127,6 +127,94 @@ output "function_arns" { value = module.functions.function_arns } +output "managed_canary" { + description = "Terraform-owned target identity and lifecycle-relevant values. Null when disabled." + value = var.managed_canary_enabled ? { + account_id = data.aws_caller_identity.current.account_id + region = data.aws_region.current.region + vpc_id = module.managed_canary[0].vpc_id + subnet_id = module.managed_canary[0].subnet_id + security_group_id = module.managed_canary[0].security_group_id + eip_allocation_id = module.managed_canary[0].eip_allocation_id + public_ip = module.managed_canary[0].public_ip + public_cidr = module.managed_canary[0].public_cidr + network_interface_id = module.managed_canary[0].network_interface_id + private_ip = module.managed_canary[0].private_ip + instance_id = module.managed_canary[0].instance_id + instance_state = module.managed_canary[0].instance_state + listener_port = module.managed_canary[0].listener_port + inventory_tag_key = module.managed_canary[0].inventory_tag_key + inventory_tag_value = module.managed_canary[0].inventory_tag_value + } : null +} + +output "managed_canary_eip_allocation_id" { + value = var.managed_canary_enabled ? module.managed_canary[0].eip_allocation_id : null +} + +output "managed_canary_public_ip" { + value = var.managed_canary_enabled ? module.managed_canary[0].public_ip : null +} + +output "managed_canary_public_cidr" { + value = var.managed_canary_enabled ? module.managed_canary[0].public_cidr : null +} + +output "managed_canary_network_interface_id" { + value = var.managed_canary_enabled ? module.managed_canary[0].network_interface_id : null +} + +output "managed_canary_instance_id" { + value = var.managed_canary_enabled ? module.managed_canary[0].instance_id : null +} + +output "managed_canary_listener_port" { + value = var.managed_canary_enabled ? module.managed_canary[0].listener_port : null +} + +output "managed_canary_snapshot_function_name" { + value = var.managed_canary_enabled ? module.functions.snapshot_function_name : null +} + +output "managed_canary_snapshot_function_arn" { + value = var.managed_canary_enabled ? module.functions.snapshot_function_arn : null +} + +output "managed_canary_status_function_name" { + value = var.managed_canary_enabled ? module.functions.processor_function_name : null +} + +output "managed_canary_status_function_arn" { + value = var.managed_canary_enabled ? module.functions.processor_function_arn : null +} + +output "managed_canary_snapshot_invocation" { + description = "Narrow payload and function used by deployment automation for the one-target inventory invocation." + value = var.managed_canary_enabled ? { + function_name = module.functions.snapshot_function_name + function_arn = module.functions.snapshot_function_arn + payload = { + operation = "managed-canary" + } + } : null +} + +output "managed_canary_status_invocation" { + description = "Narrow processor status operation for deployment automation." + value = var.managed_canary_enabled ? { + function_name = module.functions.processor_function_name + function_arn = module.functions.processor_function_arn + payload = { + operation = "managed-canary-status" + } + } : null +} + +output "operator_namespace" { + description = "Namespace containing the Helm release and scanner Jobs." + value = var.eks_namespace +} + output "emergency_pause_controls" { description = "Exact AWS-side dispatch controls used by emergency-pause.sh without refreshing Helm or EKS." value = { @@ -171,12 +259,23 @@ output "central_collector_principal_arns" { output "deployment_state" { value = { - runtime_created = var.deploy_runtime - migration_run = var.run_migration - operator_installed = var.install_operator - dispatch_enabled = var.enable_event_dispatch - automatic_inventory_enabled = var.enable_automatic_inventory - canary_mode = var.canary_mode + runtime_created = var.deploy_runtime + migration_run = var.run_migration + operator_installed = var.install_operator + dispatch_enabled = var.enable_event_dispatch + automatic_inventory_enabled = ( + local.periodic_snapshots_enabled || + local.periodic_coverage_enabled || + local.signal_hints_enabled || + local.processor_reconciliation_enabled + ) + periodic_snapshots_enabled = local.periodic_snapshots_enabled + periodic_coverage_enabled = local.periodic_coverage_enabled + signal_hints_enabled = local.signal_hints_enabled + processor_reconciliation_enabled = local.processor_reconciliation_enabled + finding_export_enabled = var.finding_export_enabled + managed_canary_enabled = var.managed_canary_enabled + canary_mode = var.canary_mode } } diff --git a/terraform/aws/application/variables.tf b/terraform/aws/application/variables.tf index 113d533..a320588 100644 --- a/terraform/aws/application/variables.tf +++ b/terraform/aws/application/variables.tf @@ -101,6 +101,39 @@ variable "existing_private_interface_endpoint_security_group_ids" { default = {} } +variable "managed_canary_enabled" { + description = "Provision the isolated Terraform-owned TCP 18080 evaluation target. Advanced roots default to disabled." + type = bool + default = false +} + +variable "managed_canary_vpc_cidr" { + description = "Dedicated unpeered managed-canary VPC /28." + type = string + default = "10.255.255.0/28" +} + +variable "managed_canary_instance_type" { + description = "Tiny ARM EC2 instance type for the managed canary." + type = string + default = "t4g.nano" +} + +variable "managed_canary_scanner_source_ipv4s" { + description = "Scanner egress EIPs for existing/TGW networking; created-VPC NAT EIPs are derived automatically." + type = set(string) + default = [] + + validation { + condition = alltrue([ + for address in var.managed_canary_scanner_source_ipv4s : + can(cidrnetmask("${address}/32")) && + try(cidrhost("${address}/32", 0) == address, false) + ]) + error_message = "managed_canary_scanner_source_ipv4s must contain canonical IPv4 addresses." + } +} + variable "lambda_timeout_seconds" { description = "Timeout for queue-triggered Lambda functions." type = number @@ -319,6 +352,43 @@ variable "allowed_tag_keys" { } } +variable "inventory_allowed_eni_interface_types" { + description = "Optional supported ENI classes. Empty disables the class gate for advanced-root compatibility." + type = set(string) + default = [] + + validation { + condition = alltrue([ + for value in var.inventory_allowed_eni_interface_types : + can(regex("^[a-z0-9-]+$", value)) + ]) + error_message = "inventory_allowed_eni_interface_types must contain lowercase AWS ENI class names." + } +} + +variable "inventory_required_target_tag_key" { + description = "Optional exact ENI opt-in tag key paired with inventory_required_target_tag_value." + type = string + default = null +} + +variable "inventory_required_target_tag_value" { + description = "Optional exact ENI opt-in tag value paired with inventory_required_target_tag_key." + type = string + default = null +} + +variable "snapshot_max_pages" { + description = "Hard per-operation page bound for EC2 and Config snapshots." + type = number + default = 1000 + + validation { + condition = var.snapshot_max_pages >= 1 && var.snapshot_max_pages <= 10000 + error_message = "snapshot_max_pages must be between 1 and 10000." + } +} + variable "cloudtrail_mode" { description = "create, existing, or disabled. Disabled retains native EC2 state hints but omits local API-call hints." type = string @@ -483,8 +553,32 @@ variable "enable_event_dispatch" { default = false } -variable "enable_automatic_inventory" { - description = "Enable recurring snapshots, rescans, processing repair, and EventBridge inventory hints. Requires enable_event_dispatch." +variable "periodic_snapshots_enabled" { + description = "Enable authoritative periodic snapshots independently." + type = bool + default = false +} + +variable "periodic_coverage_enabled" { + description = "Enable recurring full-TCP coverage independently." + type = bool + default = false +} + +variable "signal_hints_enabled" { + description = "Enable EventBridge/CloudTrail signal intake independently." + type = bool + default = false +} + +variable "processor_reconciliation_enabled" { + description = "Enable scheduled PostgreSQL finding reconciliation independently." + type = bool + default = false +} + +variable "finding_export_enabled" { + description = "Create and publish optional S3/SQS finding handoffs. PostgreSQL is the default boundary." type = bool default = false } diff --git a/terraform/aws/examples/created-vpc/.terraform.lock.hcl b/terraform/aws/deployment/.terraform.lock.hcl similarity index 100% rename from terraform/aws/examples/created-vpc/.terraform.lock.hcl rename to terraform/aws/deployment/.terraform.lock.hcl diff --git a/terraform/aws/deployment/environment.auto.tfvars.json.example b/terraform/aws/deployment/environment.auto.tfvars.json.example new file mode 100644 index 0000000..5f9fc92 --- /dev/null +++ b/terraform/aws/deployment/environment.auto.tfvars.json.example @@ -0,0 +1,64 @@ +{ + "environment": { + "name": "portscanner-eval", + "aws": { + "account_id": "REPLACE_WITH_12_DIGIT_ACCOUNT_ID", + "region": "us-east-1" + }, + "network": { + "vpc_cidr": "10.64.0.0/20", + "availability_zones": [], + "az_count": 2, + "nat_gateway_mode": "single" + }, + "runner": { + "eks_installer_principal_arn": "REPLACE_WITH_STABLE_IAM_ROLE_OR_USER_ARN", + "restricted_public_cidr": "REPLACE_WITH_RUNNER_PUBLIC_IPV4/32" + }, + "retention": { + "disposable": true, + "destroy_data_on_teardown": true, + "database_backup_days": 1, + "lambda_log_days": 7, + "ecr_untagged_image_days": 7, + "bucket_expiration_days": { + "events": 7, + "results": 7, + "findings": 30, + "cloudtrail": 30 + } + }, + "scanner": { + "operator_max_concurrent_reconciles": 1, + "max_concurrent_pods": 1, + "max_jobs": 4, + "min_rate": 100, + "max_rate": 250, + "architecture": "arm64", + "node_instance_type": "t4g.medium" + }, + "managed_canary": { + "enabled": true, + "vpc_cidr": "10.255.255.0/28", + "instance_type": "t4g.nano", + "listen_port": 18080, + "expected_finding_severity": "low" + }, + "integrations": { + "recurring_inventory_enabled": false, + "signal_hints_enabled": false, + "finding_export_enabled": false, + "config_mode": "disabled", + "existing_config_aggregator_name": "", + "snapshot_regions": [], + "cloudtrail_mode": "disabled", + "existing_cloudtrail_arn": "", + "authorized_account_ids": [], + "allowed_target_cidrs": [], + "denied_target_cidrs": [], + "allowed_eni_interface_types": ["interface"], + "required_target_tag_key": "application", + "required_target_tag_value": "portscanner" + } + } +} diff --git a/terraform/aws/deployment/main.tf b/terraform/aws/deployment/main.tf new file mode 100644 index 0000000..e660fc9 --- /dev/null +++ b/terraform/aws/deployment/main.tf @@ -0,0 +1,182 @@ +locals { + configured_architecture = var.environment.scanner.architecture + configured_node_ami_type = ( + local.configured_architecture == "arm64" ? + "AL2023_ARM_64_STANDARD" : + "AL2023_x86_64_STANDARD" + ) + configured_node_instance_types = [var.environment.scanner.node_instance_type] + + workload_architecture = coalesce(var.lambda_architecture, local.configured_architecture) + node_ami_type = coalesce(var.node_ami_type, local.configured_node_ami_type) + node_instance_types = var.node_instance_types == null ? local.configured_node_instance_types : var.node_instance_types + + config_mode = ( + var.environment.integrations.recurring_inventory_enabled ? + var.environment.integrations.config_mode : + "disabled" + ) + cloudtrail_mode = ( + var.environment.integrations.signal_hints_enabled ? + var.environment.integrations.cloudtrail_mode : + "disabled" + ) + + destroy_data = ( + var.environment.retention.disposable && + var.environment.retention.destroy_data_on_teardown + ) + database_skip_final_snapshot = local.destroy_data + database_final_snapshot_identifier = ( + local.database_skip_final_snapshot ? + null : + "${var.environment.name}-final" + ) + retirement_environment = merge( + var.environment, + { + managed_canary = merge( + var.environment.managed_canary, + { enabled = true } + ) + } + ) + retirement_configuration_fingerprint = sha256(jsonencode(local.retirement_environment)) +} + +provider "aws" { + region = var.environment.aws.region + allowed_account_ids = [var.environment.aws.account_id] + + default_tags { + tags = { + Environment = var.environment.name + ManagedBy = "Terraform" + Project = "portscanner" + } + } +} + +resource "terraform_data" "canonical_configuration" { + input = { + environment_name = var.environment.name + environment = var.environment + retirement_configuration_fingerprint = local.retirement_configuration_fingerprint + } + + lifecycle { + precondition { + condition = ( + var.lambda_architecture == null || + var.lambda_architecture == local.configured_architecture + ) + error_message = "Generated lambda_architecture does not match environment.scanner.architecture." + } + + precondition { + condition = ( + var.node_ami_type == null || + var.node_ami_type == local.configured_node_ami_type + ) + error_message = "Generated node_ami_type does not match environment.scanner.architecture." + } + + precondition { + condition = ( + var.node_instance_types == null || + var.node_instance_types == local.configured_node_instance_types + ) + error_message = "Generated node_instance_types do not match environment.scanner.node_instance_type." + } + } +} + +module "portscanner" { + source = "../application" + + name_prefix = var.environment.name + create_vpc = true + vpc_cidr = var.environment.network.vpc_cidr + availability_zones = length(var.environment.network.availability_zones) == 0 ? null : var.environment.network.availability_zones + az_count = var.environment.network.az_count + nat_gateway_mode = var.environment.network.nat_gateway_mode + + managed_canary_enabled = var.environment.managed_canary.enabled + managed_canary_vpc_cidr = var.environment.managed_canary.vpc_cidr + managed_canary_instance_type = var.environment.managed_canary.instance_type + + config_mode = local.config_mode + existing_config_aggregator_name = length(var.environment.integrations.existing_config_aggregator_name) == 0 ? null : var.environment.integrations.existing_config_aggregator_name + snapshot_regions = var.environment.integrations.snapshot_regions + cloudtrail_mode = local.cloudtrail_mode + existing_cloudtrail_arn = length(var.environment.integrations.existing_cloudtrail_arn) == 0 ? null : var.environment.integrations.existing_cloudtrail_arn + + create_central_event_bus = false + authorized_account_ids = var.environment.integrations.authorized_account_ids + allowed_target_cidrs = var.environment.integrations.allowed_target_cidrs + denied_target_cidrs = var.environment.integrations.denied_target_cidrs + inventory_allowed_eni_interface_types = ( + var.environment.integrations.allowed_eni_interface_types + ) + inventory_required_target_tag_key = ( + var.environment.integrations.required_target_tag_key + ) + inventory_required_target_tag_value = ( + var.environment.integrations.required_target_tag_value + ) + operator_max_concurrent_reconciles = var.environment.scanner.operator_max_concurrent_reconciles + scanner_max_concurrent_pods = var.environment.scanner.max_concurrent_pods + scanner_max_jobs = var.environment.scanner.max_jobs + scanner_min_rate = var.environment.scanner.min_rate + scanner_max_rate = var.environment.scanner.max_rate + + deploy_runtime = var.deploy_runtime + run_migration = var.run_migration + install_operator = var.install_operator + enable_event_dispatch = var.enable_event_dispatch + periodic_snapshots_enabled = ( + var.enable_automatic_inventory && + var.environment.integrations.recurring_inventory_enabled + ) + periodic_coverage_enabled = ( + var.enable_automatic_inventory && + var.environment.integrations.recurring_inventory_enabled + ) + signal_hints_enabled = ( + var.enable_automatic_inventory && + var.environment.integrations.signal_hints_enabled + ) + processor_reconciliation_enabled = ( + var.enable_automatic_inventory && + var.environment.integrations.recurring_inventory_enabled + ) + finding_export_enabled = var.environment.integrations.finding_export_enabled + canary_mode = var.canary_mode + image_digests = var.image_digests + migration_checksum = var.migration_checksum + + eks_endpoint_public_access = true + eks_public_access_cidrs = [var.environment.runner.restricted_public_cidr] + eks_installer_principal_arns = [var.environment.runner.eks_installer_principal_arn] + lambda_architecture = local.workload_architecture + node_ami_type = local.node_ami_type + node_instance_types = local.node_instance_types + + bucket_expiration_days = var.environment.retention.bucket_expiration_days + ecr_untagged_image_expiration_days = var.environment.retention.ecr_untagged_image_days + force_destroy_buckets = local.destroy_data + force_delete_repositories = local.destroy_data + lambda_log_retention_days = var.environment.retention.lambda_log_days + + database_instance_count = 1 + database_backup_retention_days = var.environment.retention.database_backup_days + database_deletion_protection = !var.environment.retention.disposable + database_skip_final_snapshot = local.database_skip_final_snapshot + database_final_snapshot_identifier = local.database_final_snapshot_identifier + dynamodb_point_in_time_recovery = !var.environment.retention.disposable + + node_desired_size = 1 + node_min_size = 1 + node_max_size = 2 + node_capacity_type = "SPOT" +} diff --git a/terraform/aws/deployment/main.tftest.hcl b/terraform/aws/deployment/main.tftest.hcl new file mode 100644 index 0000000..97ab067 --- /dev/null +++ b/terraform/aws/deployment/main.tftest.hcl @@ -0,0 +1,127 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_partition" { + defaults = { + partition = "aws" + } + } + + mock_data "aws_region" { + defaults = { + region = "us-east-1" + } + } + + mock_data "aws_availability_zones" { + defaults = { + names = ["us-east-1a", "us-east-1b"] + } + } + + mock_data "aws_ec2_instance_type" { + defaults = { + supported_architectures = ["arm64"] + } + } + + mock_data "aws_ami" { + defaults = { + id = "ami-0123456789abcdef0" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +mock_provider "helm" {} + +run "canonical_environment_maps_to_paused_database_first_stack" { + command = plan + + variables { + environment = { + name = "test-eval" + aws = { + account_id = "123456789012" + region = "us-east-1" + } + network = { + vpc_cidr = "10.64.0.0/20" + availability_zones = [] + az_count = 2 + nat_gateway_mode = "single" + } + runner = { + eks_installer_principal_arn = "arn:aws:iam::123456789012:role/test-installer" + restricted_public_cidr = "203.0.113.10/32" + } + retention = { + disposable = true + destroy_data_on_teardown = true + database_backup_days = 1 + lambda_log_days = 7 + ecr_untagged_image_days = 7 + bucket_expiration_days = { + events = 7 + results = 7 + findings = 30 + cloudtrail = 30 + } + } + scanner = { + operator_max_concurrent_reconciles = 1 + max_concurrent_pods = 1 + max_jobs = 4 + min_rate = 100 + max_rate = 250 + architecture = "arm64" + node_instance_type = "t4g.medium" + } + managed_canary = { + enabled = true + vpc_cidr = "10.255.255.0/28" + instance_type = "t4g.nano" + listen_port = 18080 + expected_finding_severity = "low" + } + integrations = { + recurring_inventory_enabled = false + signal_hints_enabled = false + finding_export_enabled = false + config_mode = "disabled" + existing_config_aggregator_name = "" + snapshot_regions = [] + cloudtrail_mode = "disabled" + existing_cloudtrail_arn = "" + authorized_account_ids = [] + allowed_target_cidrs = [] + denied_target_cidrs = [] + allowed_eni_interface_types = ["interface"] + required_target_tag_key = "application" + required_target_tag_value = "portscanner" + } + } + } + + assert { + condition = ( + output.deployment_state.managed_canary_enabled && + !output.deployment_state.dispatch_enabled && + output.managed_canary.listener_port == 18080 && + output.finding_bucket_name == null && + output.finding_queue_url == null && + output.operator_namespace == "portscanner-system" && + length(output.retirement_configuration_fingerprint) == 64 + ) + error_message = "The canonical environment must plan a paused managed canary and database-only finding boundary." + } +} diff --git a/terraform/aws/deployment/outputs.tf b/terraform/aws/deployment/outputs.tf new file mode 100644 index 0000000..43aef77 --- /dev/null +++ b/terraform/aws/deployment/outputs.tf @@ -0,0 +1,165 @@ +output "vpc_id" { + value = module.portscanner.vpc_id +} + +output "private_subnet_ids" { + value = module.portscanner.private_subnet_ids +} + +output "isolated_subnet_ids" { + value = module.portscanner.isolated_subnet_ids +} + +output "scanner_egress_public_ips" { + value = module.portscanner.scanner_egress_public_ips +} + +output "bucket_names" { + value = module.portscanner.bucket_names +} + +output "bucket_arns" { + value = module.portscanner.bucket_arns +} + +output "queue_urls" { + value = module.portscanner.queue_urls +} + +output "queue_arns" { + value = module.portscanner.queue_arns +} + +output "dead_letter_queue_arns" { + value = module.portscanner.dead_letter_queue_arns +} + +output "table_names" { + value = module.portscanner.table_names +} + +output "inventory_table_name" { + value = module.portscanner.inventory_table_name +} + +output "dispatch_table_name" { + value = module.portscanner.dispatch_table_name +} + +output "target_event_queue_url" { + value = module.portscanner.target_event_queue_url +} + +output "finding_queue_url" { + value = module.portscanner.finding_queue_url +} + +output "finding_queue_arn" { + value = module.portscanner.finding_queue_arn +} + +output "finding_bucket_name" { + value = module.portscanner.finding_bucket_name +} + +output "finding_bucket_arn" { + value = module.portscanner.finding_bucket_arn +} + +output "repository_urls" { + value = module.portscanner.repository_urls +} + +output "database_cluster_endpoint" { + value = module.portscanner.database_cluster_endpoint +} + +output "database_master_secret_arn" { + value = module.portscanner.database_master_secret_arn + sensitive = true +} + +output "database_application_secret_arn" { + value = module.portscanner.database_application_secret_arn +} + +output "eks_cluster_name" { + value = module.portscanner.eks_cluster_name +} + +output "eks_cluster_endpoint" { + value = module.portscanner.eks_cluster_endpoint +} + +output "central_event_bus_arn" { + value = module.portscanner.central_event_bus_arn +} + +output "config_aggregator_name" { + value = module.portscanner.config_aggregator_name +} + +output "cloudtrail_arn" { + value = module.portscanner.cloudtrail_arn +} + +output "signal_region" { + value = module.portscanner.signal_region +} + +output "created_config_source_regions" { + value = module.portscanner.created_config_source_regions +} + +output "function_arns" { + value = module.portscanner.function_arns +} + +output "managed_canary" { + value = module.portscanner.managed_canary +} + +output "managed_canary_snapshot_invocation" { + value = module.portscanner.managed_canary_snapshot_invocation +} + +output "managed_canary_status_invocation" { + value = module.portscanner.managed_canary_status_invocation +} + +output "operator_namespace" { + value = module.portscanner.operator_namespace +} + +output "retirement_configuration_fingerprint" { + description = "Stable when managed_canary.enabled is the only environment change." + value = local.retirement_configuration_fingerprint +} + +output "emergency_pause_controls" { + value = module.portscanner.emergency_pause_controls +} + +output "alarm_names" { + value = module.portscanner.alarm_names +} + +output "alarm_arns" { + value = module.portscanner.alarm_arns +} + +output "central_collector_principal_arn" { + value = module.portscanner.central_collector_principal_arn +} + +output "central_collector_principal_arns" { + value = module.portscanner.central_collector_principal_arns +} + +output "deployment_state" { + value = module.portscanner.deployment_state +} + +output "workload_architecture" { + value = module.portscanner.workload_architecture +} diff --git a/terraform/aws/deployment/variables.tf b/terraform/aws/deployment/variables.tf new file mode 100644 index 0000000..5a19798 --- /dev/null +++ b/terraform/aws/deployment/variables.tf @@ -0,0 +1,304 @@ +variable "environment" { + description = "The sole user-maintained, non-secret deployment configuration." + type = object({ + name = string + aws = object({ + account_id = string + region = string + }) + network = object({ + vpc_cidr = string + availability_zones = list(string) + az_count = number + nat_gateway_mode = string + }) + runner = object({ + eks_installer_principal_arn = string + restricted_public_cidr = string + }) + retention = object({ + disposable = bool + destroy_data_on_teardown = bool + database_backup_days = number + lambda_log_days = number + ecr_untagged_image_days = number + bucket_expiration_days = map(number) + }) + scanner = object({ + operator_max_concurrent_reconciles = number + max_concurrent_pods = number + max_jobs = number + min_rate = number + max_rate = number + architecture = string + node_instance_type = string + }) + managed_canary = object({ + enabled = bool + vpc_cidr = string + instance_type = string + listen_port = number + expected_finding_severity = string + }) + integrations = object({ + recurring_inventory_enabled = bool + signal_hints_enabled = bool + finding_export_enabled = bool + config_mode = string + existing_config_aggregator_name = string + snapshot_regions = list(string) + cloudtrail_mode = string + existing_cloudtrail_arn = string + authorized_account_ids = set(string) + allowed_target_cidrs = set(string) + denied_target_cidrs = set(string) + allowed_eni_interface_types = set(string) + required_target_tag_key = string + required_target_tag_value = string + }) + }) + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,19}$", var.environment.name)) + error_message = "environment.name must be 2-20 lowercase alphanumeric or hyphen characters." + } + + validation { + condition = can(regex("^[0-9]{12}$", var.environment.aws.account_id)) + error_message = "environment.aws.account_id must be a 12-digit AWS account ID." + } + + validation { + condition = can(regex("^[a-z]{2}(?:-[a-z0-9]+)+-[0-9]+$", var.environment.aws.region)) + error_message = "environment.aws.region must be an AWS Region name." + } + + validation { + condition = ( + can(cidrnetmask(var.environment.network.vpc_cidr)) && + try(cidrhost(var.environment.network.vpc_cidr, 0) == split("/", var.environment.network.vpc_cidr)[0], false) && + try(tonumber(split("/", var.environment.network.vpc_cidr)[1]) >= 16, false) && + try(tonumber(split("/", var.environment.network.vpc_cidr)[1]) <= 24, false) + ) + error_message = "environment.network.vpc_cidr must be a canonical IPv4 /16 through /24." + } + + validation { + condition = ( + var.environment.network.az_count >= 2 && + var.environment.network.az_count <= 3 && + floor(var.environment.network.az_count) == var.environment.network.az_count && + contains(["single", "one_per_az"], var.environment.network.nat_gateway_mode) + ) + error_message = "environment.network must select 2-3 AZs and single or one_per_az NAT." + } + + validation { + condition = ( + can(regex( + "^arn:[^:]+:iam::${var.environment.aws.account_id}:(?:role|user)/.+$", + var.environment.runner.eks_installer_principal_arn + )) && + var.environment.runner.restricted_public_cidr != "0.0.0.0/0" && + can(cidrnetmask(var.environment.runner.restricted_public_cidr)) && + try(cidrhost(var.environment.runner.restricted_public_cidr, 0) == split("/", var.environment.runner.restricted_public_cidr)[0], false) && + try(tonumber(split("/", var.environment.runner.restricted_public_cidr)[1]) == 32, false) + ) + error_message = "environment.runner requires a same-account IAM role/user ARN and one canonical public IPv4 /32." + } + + validation { + condition = ( + !var.environment.retention.destroy_data_on_teardown || + var.environment.retention.disposable + ) + error_message = "destroy_data_on_teardown may be true only for an explicitly disposable environment." + } + + validation { + condition = ( + var.environment.retention.database_backup_days >= 1 && + var.environment.retention.database_backup_days <= 35 && + var.environment.retention.lambda_log_days >= 1 && + var.environment.retention.ecr_untagged_image_days >= 1 && + toset(keys(var.environment.retention.bucket_expiration_days)) == toset([ + "cloudtrail", + "events", + "findings", + "results" + ]) && + alltrue([ + for days in values(var.environment.retention.bucket_expiration_days) : + days >= 1 && floor(days) == days + ]) + ) + error_message = "retention values must be positive, and bucket_expiration_days must define cloudtrail, events, findings, and results." + } + + validation { + condition = ( + var.environment.scanner.operator_max_concurrent_reconciles >= 1 && + var.environment.scanner.operator_max_concurrent_reconciles <= 32 && + floor(var.environment.scanner.operator_max_concurrent_reconciles) == var.environment.scanner.operator_max_concurrent_reconciles && + var.environment.scanner.max_concurrent_pods >= 1 && + var.environment.scanner.max_concurrent_pods <= 100 && + floor(var.environment.scanner.max_concurrent_pods) == var.environment.scanner.max_concurrent_pods && + var.environment.scanner.max_jobs >= 1 && + var.environment.scanner.max_jobs <= 1000 && + floor(var.environment.scanner.max_jobs) == var.environment.scanner.max_jobs && + var.environment.scanner.min_rate >= 1 && + var.environment.scanner.max_rate <= 5000 && + var.environment.scanner.min_rate <= var.environment.scanner.max_rate + ) + error_message = "environment.scanner limits must be positive, integral, within application bounds, and min_rate must not exceed max_rate." + } + + validation { + condition = ( + (var.environment.scanner.architecture == "arm64" && startswith(var.environment.scanner.node_instance_type, "t4g.")) || + (var.environment.scanner.architecture == "x86_64" && startswith(var.environment.scanner.node_instance_type, "t3.")) + ) + error_message = "The low-cost canonical root supports arm64 with t4g.* or x86_64 with t3.* nodes." + } + + validation { + condition = ( + can(cidrnetmask(var.environment.managed_canary.vpc_cidr)) && + try(cidrhost(var.environment.managed_canary.vpc_cidr, 0) == split("/", var.environment.managed_canary.vpc_cidr)[0], false) && + try(tonumber(split("/", var.environment.managed_canary.vpc_cidr)[1]) == 28, false) && + startswith(var.environment.managed_canary.instance_type, "t4g.") && + var.environment.managed_canary.listen_port == 18080 && + var.environment.managed_canary.expected_finding_severity == "low" + ) + error_message = "managed_canary requires a canonical /28, ARM t4g instance, fixed TCP 18080 listener, and low expected severity." + } + + validation { + condition = ( + contains(["create", "existing", "disabled"], var.environment.integrations.config_mode) && + contains(["create", "existing", "disabled"], var.environment.integrations.cloudtrail_mode) && + ( + var.environment.integrations.config_mode != "existing" || + length(var.environment.integrations.existing_config_aggregator_name) > 0 + ) && + ( + var.environment.integrations.cloudtrail_mode != "existing" || + length(var.environment.integrations.existing_cloudtrail_arn) > 0 + ) + ) + error_message = "integration modes must be create, existing, or disabled and existing modes require their referenced resource." + } + + validation { + condition = alltrue([ + for account_id in var.environment.integrations.authorized_account_ids : + can(regex("^[0-9]{12}$", account_id)) + ]) + error_message = "integrations.authorized_account_ids must contain only 12-digit AWS account IDs." + } + + validation { + condition = alltrue([ + for region in var.environment.integrations.snapshot_regions : + can(regex("^[a-z]{2}(?:-[a-z0-9]+)+-[0-9]+$", region)) + ]) + error_message = "integrations.snapshot_regions must contain valid AWS Region names." + } + + validation { + condition = alltrue([ + for cidr in setunion( + var.environment.integrations.allowed_target_cidrs, + var.environment.integrations.denied_target_cidrs + ) : + can(cidrnetmask(cidr)) && + try(cidrhost(cidr, 0) == split("/", cidr)[0], false) + ]) + error_message = "integration target CIDRs must be canonical IP network prefixes." + } + + validation { + condition = ( + length(var.environment.integrations.allowed_eni_interface_types) > 0 && + alltrue([ + for interface_type in var.environment.integrations.allowed_eni_interface_types : + can(regex("^[a-z0-9-]+$", interface_type)) + ]) && + contains( + ["application", "environment", "name", "service"], + var.environment.integrations.required_target_tag_key + ) && + length(var.environment.integrations.required_target_tag_value) > 0 + ) + error_message = "integrations requires supported ENI interface types and one non-empty allowed target tag." + } +} + +variable "deploy_runtime" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "run_migration" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "install_operator" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "enable_event_dispatch" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "enable_automatic_inventory" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "canary_mode" { + description = "Internal staged-deployment gate; do not set in environment configuration." + type = bool + default = false +} + +variable "image_digests" { + description = "Generated immutable image digests; written by bootstrap.sh." + type = map(string) + default = {} +} + +variable "migration_checksum" { + description = "Generated migration checksum; written by bootstrap.sh." + type = string + default = "" +} + +variable "lambda_architecture" { + description = "Generated confirmation of the applied workload architecture." + type = string + default = null + nullable = true +} + +variable "node_ami_type" { + description = "Generated confirmation of the applied EKS AMI architecture." + type = string + default = null + nullable = true +} + +variable "node_instance_types" { + description = "Generated confirmation of the applied EKS instance types." + type = list(string) + default = null + nullable = true +} diff --git a/terraform/aws/deployment/versions.tf b/terraform/aws/deployment/versions.tf new file mode 100644 index 0000000..9279f99 --- /dev/null +++ b/terraform/aws/deployment/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_version = "= 1.7.4" + + backend "s3" {} + + required_providers { + aws = { + source = "hashicorp/aws" + version = "= 6.15.0" + } + } +} diff --git a/terraform/aws/examples/README.md b/terraform/aws/examples/README.md deleted file mode 100644 index 44d7ad0..0000000 --- a/terraform/aws/examples/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Synthetic examples - -These roots are validation templates, not account-specific configuration: - -- `created-vpc` uses documentation network `192.0.2.0/24`. -- `existing-vpc` uses synthetic VPC/subnet IDs and demonstrates validation. -- `multi-account-central` uses documentation network `198.51.100.0/24` and member account `123456789012`. -- `member-account` uses only account `123456789012`, an exact synthetic role ARN, and a synthetic central bus ARN. - -Dispatch is disabled everywhere. Names, accounts, VPC/subnets, member maps, EKS API -client security groups, and installer principals are variables so live configuration can -stay in ignored private tfvars files. Replace every synthetic value, configure remote -state, review costs, and run the foundation stage before any apply. Do not edit tracked -examples with environment values. The examples intentionally use low-volume defaults -that are not resilient production settings. - -The existing-VPC root intentionally defaults its egress mode to null; a live plan must -declare validated NAT gateway, transit gateway, or complete endpoint egress. Endpoint -mode requires concrete endpoint IDs plus one attached security group for each interface -endpoint; Terraform manages workload TLS ingress on those selected groups. Central -application roots export both `repository_urls` and `deployment_state` for foundation -and upgrade image publication. The member root does not and is explicitly rejected by -the image helper. - -The examples retain the default ARM64 pairing (`lambda_architecture = "arm64"`, -`node_ami_type = "AL2023_ARM_64_STANDARD"`, and -`node_instance_types = ["t4g.medium"]`). Build all seven images for `linux/arm64`, or -change all three settings and external image builds together for x86_64 (for example, -`AL2023_x86_64_STANDARD` with `["t3.medium"]`). The image helper verifies the applied -architecture contract. The `finding` queue is an external notification handoff; no -example attaches an in-stack consumer. - -For a guarded single-account walkthrough, copy -`created-vpc/evaluation.tfvars.example` to the ignored -`created-vpc/evaluation.tfvars`, replace every `REPLACE_*` value, and follow -`docs/getting-started.md`. That file is intentionally disposable-evaluation policy, not -a production baseline. - -Destroying an example can delete compute and database resources. S3/ECR retention and state safeguards may intentionally block destroy until data is reviewed and preserved. diff --git a/terraform/aws/examples/created-vpc/evaluation.tfvars.example b/terraform/aws/examples/created-vpc/evaluation.tfvars.example deleted file mode 100644 index c8586cf..0000000 --- a/terraform/aws/examples/created-vpc/evaluation.tfvars.example +++ /dev/null @@ -1,47 +0,0 @@ -# Copy this file outside the repository or to evaluation.tfvars (ignored by Git), -# replace every REPLACE_* value, and keep it private. - -aws_region = "us-east-1" -name_prefix = "portscanner-eval" -vpc_cidr = "10.64.0.0/20" - -# The provider refuses to use credentials for any other account. -expected_deployment_account_id = "REPLACE_WITH_12_DIGIT_ACCOUNT_ID" - -# Direct EC2 snapshots avoid creating or changing the account's singleton AWS Config -# recorder. CloudTrail is also disabled for the one-shot evaluation; native EC2 state -# hints remain defined but paused, and local API-call hints are omitted. -config_mode = "disabled" -snapshot_regions = [] -cloudtrail_mode = "disabled" - -# Both boundaries apply at dispatch. Keep the CIDR scope to one explicitly authorized -# public canary address until the complete evaluation has been reviewed. -authorized_account_ids = ["REPLACE_WITH_12_DIGIT_ACCOUNT_ID"] -allowed_target_cidrs = ["REPLACE_WITH_AUTHORIZED_PUBLIC_IPV4/32"] -denied_target_cidrs = [] - -# One target, one reconciler, and conservative Nmap rates for the first proof. -operator_max_concurrent_reconciles = 1 -scanner_max_concurrent_pods = 1 -scanner_max_jobs = 4 -scanner_min_rate = 100 -scanner_max_rate = 250 - -# This is appropriate only for a short-lived evaluation whose evidence may be deleted. -force_destroy_buckets = true -force_delete_repositories = true - -# The evaluation opts into an IAM-authenticated public EKS endpoint restricted to the -# Terraform runner's current public address. Never use 0.0.0.0/0. Production should -# disable this path and use private runner/VPN security groups instead. -eks_endpoint_public_access = true -eks_public_access_cidrs = ["REPLACE_WITH_RUNNER_PUBLIC_IPV4/32"] - -# Use the stable IAM role/user ARN used by Terraform. An assumed-role STS ARN is not -# accepted; use its underlying IAM role ARN. -eks_installer_principal_arns = ["REPLACE_WITH_STABLE_IAM_ROLE_OR_USER_ARN"] - -# Leave this empty for the restricted public evaluation path. Production should disable -# public access and set one or more approved private runner/VPN security groups here. -eks_api_client_security_group_ids = [] diff --git a/terraform/aws/examples/created-vpc/main.tf b/terraform/aws/examples/created-vpc/main.tf deleted file mode 100644 index 881b421..0000000 --- a/terraform/aws/examples/created-vpc/main.tf +++ /dev/null @@ -1,360 +0,0 @@ -terraform { - required_version = "= 1.7.4" - - backend "s3" {} - - required_providers { - aws = { - source = "hashicorp/aws" - version = "= 6.15.0" - } - } -} - -variable "aws_region" { - description = "Deployment region." - type = string - default = "us-east-1" -} - -variable "expected_deployment_account_id" { - description = "Fail-closed AWS account boundary for this Terraform root." - type = string - default = "123456789012" - - validation { - condition = can(regex("^[0-9]{12}$", var.expected_deployment_account_id)) - error_message = "expected_deployment_account_id must be a 12-digit AWS account ID." - } -} - -variable "name_prefix" { - description = "Synthetic default; override from an ignored private tfvars file." - type = string - default = "scan-example" -} - -variable "vpc_cidr" { - description = "Canonical created-VPC CIDR." - type = string - default = "192.0.2.0/24" -} - -variable "availability_zones" { - description = "Optional explicit created-VPC availability zones." - type = list(string) - default = null -} - -variable "az_count" { - description = "Created-VPC availability-zone count." - type = number - default = 2 -} - -variable "nat_gateway_mode" { - description = "Created-VPC NAT topology." - type = string - default = "single" -} - -variable "config_mode" { - description = "AWS Config mode: create, existing, or disabled. Use disabled for a simple direct-EC2 evaluation." - type = string - default = "create" -} - -variable "existing_config_aggregator_name" { - description = "Existing AWS Config aggregator name when config_mode is existing." - type = string - default = null -} - -variable "snapshot_regions" { - description = "Regions reconciled by direct EC2 snapshots when Config is disabled; empty uses aws_region." - type = list(string) - default = [] -} - -variable "cloudtrail_mode" { - description = "create, existing, or disabled. Disabled omits local API-call hints." - type = string - default = "create" -} - -variable "existing_cloudtrail_arn" { - description = "Existing multi-region management trail ARN when cloudtrail_mode is existing." - type = string - default = null -} - -variable "authorized_account_ids" { - description = "Synthetic account scope; override before apply." - type = set(string) - default = ["123456789012"] -} - -variable "allowed_target_cidrs" { - description = "Optional deployment-wide scanner allowlist. Set a narrow authorized CIDR for evaluation." - type = set(string) - default = [] -} - -variable "denied_target_cidrs" { - description = "Optional deployment-wide scanner denylist, enforced before the allowlist." - type = set(string) - default = [] -} - -variable "operator_max_concurrent_reconciles" { - description = "Maximum concurrent Scanner reconciliations." - type = number - default = 4 -} - -variable "scanner_max_concurrent_pods" { - description = "Hard cap on simultaneously active scanner Pods." - type = number - default = 4 -} - -variable "scanner_max_jobs" { - description = "Hard cap on scanner Job objects." - type = number - default = 16 -} - -variable "scanner_min_rate" { - description = "Minimum Nmap probe rate for each scanner Job." - type = number - default = 100 -} - -variable "scanner_max_rate" { - description = "Maximum Nmap probe rate for each scanner Job." - type = number - default = 500 -} - -variable "eks_api_client_security_group_ids" { - description = "Runner or VPN security groups with private EKS API reachability." - type = set(string) - default = [] -} - -variable "eks_endpoint_public_access" { - description = "Opt in to a restricted public EKS API endpoint for a disposable evaluation." - type = bool - default = false -} - -variable "eks_public_access_cidrs" { - description = "Restricted canonical IPv4 CIDRs allowed to reach the opt-in public EKS API endpoint." - type = set(string) - default = [] -} - -variable "eks_installer_principal_arns" { - description = "IAM roles or users explicitly authorized to run the Helm install." - type = set(string) - default = [] -} - -variable "lambda_architecture" { - description = "arm64 or x86_64; must match node_ami_type and published images." - type = string - default = "arm64" -} - -variable "node_ami_type" { - description = "EKS managed-node AMI type matching lambda_architecture." - type = string - default = "AL2023_ARM_64_STANDARD" -} - -variable "node_instance_types" { - description = "EKS instance types compatible with node_ami_type." - type = list(string) - default = ["t4g.medium"] -} - -variable "ecr_untagged_image_expiration_days" { - description = "Optional untagged-only ECR expiration; null disables expiration." - type = number - default = null -} - -variable "force_destroy_buckets" { - description = "Delete all object versions on destroy. Enable only for a disposable evaluation." - type = bool - default = false -} - -variable "force_delete_repositories" { - description = "Delete ECR images with repositories during destroy. Enable only for a disposable evaluation." - type = bool - default = false -} - -variable "alarm_action_arns" { - type = list(string) - default = [] -} - -variable "ok_action_arns" { - type = list(string) - default = [] -} - -variable "deploy_runtime" { - type = bool - default = false -} - -variable "run_migration" { - type = bool - default = false -} - -variable "install_operator" { - type = bool - default = false -} - -variable "enable_event_dispatch" { - type = bool - default = false -} - -variable "enable_automatic_inventory" { - type = bool - default = false -} - -variable "canary_mode" { - type = bool - default = false -} - -variable "image_digests" { - description = "Digests produced by an external image build pipeline." - type = map(string) - default = {} -} - -variable "migration_checksum" { - type = string - default = "" -} - -provider "aws" { - region = var.aws_region - allowed_account_ids = [var.expected_deployment_account_id] - - default_tags { - tags = { - ManagedBy = "Terraform" - Environment = var.name_prefix - } - } -} - -module "portscanner" { - source = "../../application" - - name_prefix = var.name_prefix - create_vpc = true - vpc_cidr = var.vpc_cidr - availability_zones = var.availability_zones - az_count = var.az_count - nat_gateway_mode = var.nat_gateway_mode - - config_mode = var.config_mode - existing_config_aggregator_name = var.existing_config_aggregator_name - snapshot_regions = var.snapshot_regions - cloudtrail_mode = var.cloudtrail_mode - existing_cloudtrail_arn = var.existing_cloudtrail_arn - - create_central_event_bus = false - authorized_account_ids = var.authorized_account_ids - allowed_target_cidrs = var.allowed_target_cidrs - denied_target_cidrs = var.denied_target_cidrs - operator_max_concurrent_reconciles = var.operator_max_concurrent_reconciles - scanner_max_concurrent_pods = var.scanner_max_concurrent_pods - scanner_max_jobs = var.scanner_max_jobs - scanner_min_rate = var.scanner_min_rate - scanner_max_rate = var.scanner_max_rate - - deploy_runtime = var.deploy_runtime - run_migration = var.run_migration - install_operator = var.install_operator - enable_event_dispatch = var.enable_event_dispatch - enable_automatic_inventory = var.enable_automatic_inventory - canary_mode = var.canary_mode - image_digests = var.image_digests - migration_checksum = var.migration_checksum - alarm_action_arns = var.alarm_action_arns - ok_action_arns = var.ok_action_arns - - eks_api_client_security_group_ids = var.eks_api_client_security_group_ids - eks_endpoint_public_access = var.eks_endpoint_public_access - eks_public_access_cidrs = var.eks_public_access_cidrs - eks_installer_principal_arns = var.eks_installer_principal_arns - lambda_architecture = var.lambda_architecture - node_ami_type = var.node_ami_type - node_instance_types = var.node_instance_types - - ecr_untagged_image_expiration_days = var.ecr_untagged_image_expiration_days - force_destroy_buckets = var.force_destroy_buckets - force_delete_repositories = var.force_delete_repositories - - database_instance_count = 1 - database_deletion_protection = false - database_skip_final_snapshot = true - dynamodb_point_in_time_recovery = false - - node_desired_size = 1 - node_min_size = 1 - node_max_size = 2 - node_capacity_type = "SPOT" -} - -output "repository_urls" { - value = module.portscanner.repository_urls -} - -output "deployment_state" { - value = module.portscanner.deployment_state -} - -output "workload_architecture" { - value = module.portscanner.workload_architecture -} - -output "emergency_pause_controls" { - value = module.portscanner.emergency_pause_controls -} - -output "function_arns" { - value = module.portscanner.function_arns -} - -output "scanner_egress_public_ips" { - value = module.portscanner.scanner_egress_public_ips -} - -output "external_finding_queue_url" { - value = module.portscanner.finding_queue_url -} - -output "external_finding_queue_arn" { - value = module.portscanner.finding_queue_arn -} - -output "external_finding_bucket_name" { - value = module.portscanner.finding_bucket_name -} - -output "external_finding_bucket_arn" { - value = module.portscanner.finding_bucket_arn -} diff --git a/terraform/aws/examples/existing-vpc/main.tf b/terraform/aws/examples/existing-vpc/main.tf index b7ee108..c99234e 100644 --- a/terraform/aws/examples/existing-vpc/main.tf +++ b/terraform/aws/examples/existing-vpc/main.tf @@ -285,16 +285,19 @@ module "portscanner" { scanner_min_rate = var.scanner_min_rate scanner_max_rate = var.scanner_max_rate - deploy_runtime = var.deploy_runtime - run_migration = var.run_migration - install_operator = var.install_operator - enable_event_dispatch = var.enable_event_dispatch - enable_automatic_inventory = var.enable_automatic_inventory - canary_mode = var.canary_mode - image_digests = var.image_digests - migration_checksum = var.migration_checksum - alarm_action_arns = [] - ok_action_arns = [] + deploy_runtime = var.deploy_runtime + run_migration = var.run_migration + install_operator = var.install_operator + enable_event_dispatch = var.enable_event_dispatch + periodic_snapshots_enabled = var.enable_automatic_inventory + periodic_coverage_enabled = var.enable_automatic_inventory + signal_hints_enabled = var.enable_automatic_inventory + processor_reconciliation_enabled = var.enable_automatic_inventory + canary_mode = var.canary_mode + image_digests = var.image_digests + migration_checksum = var.migration_checksum + alarm_action_arns = [] + ok_action_arns = [] eks_api_client_security_group_ids = var.eks_api_client_security_group_ids eks_endpoint_public_access = var.eks_endpoint_public_access diff --git a/terraform/aws/examples/multi-account-central/main.tf b/terraform/aws/examples/multi-account-central/main.tf index 8d04d8f..5bb1bf7 100644 --- a/terraform/aws/examples/multi-account-central/main.tf +++ b/terraform/aws/examples/multi-account-central/main.tf @@ -261,16 +261,19 @@ module "portscanner" { member_collector_role_arns = var.member_collector_role_arns member_collector_external_ids = var.member_collector_external_ids - deploy_runtime = var.deploy_runtime - run_migration = var.run_migration - install_operator = var.install_operator - enable_event_dispatch = var.enable_event_dispatch - enable_automatic_inventory = var.enable_automatic_inventory - canary_mode = var.canary_mode - image_digests = var.image_digests - migration_checksum = var.migration_checksum - alarm_action_arns = [] - ok_action_arns = [] + deploy_runtime = var.deploy_runtime + run_migration = var.run_migration + install_operator = var.install_operator + enable_event_dispatch = var.enable_event_dispatch + periodic_snapshots_enabled = var.enable_automatic_inventory + periodic_coverage_enabled = var.enable_automatic_inventory + signal_hints_enabled = var.enable_automatic_inventory + processor_reconciliation_enabled = var.enable_automatic_inventory + canary_mode = var.canary_mode + image_digests = var.image_digests + migration_checksum = var.migration_checksum + alarm_action_arns = [] + ok_action_arns = [] eks_api_client_security_group_ids = var.eks_api_client_security_group_ids eks_endpoint_public_access = var.eks_endpoint_public_access diff --git a/terraform/aws/member-account/README.md b/terraform/aws/member-account/README.md deleted file mode 100644 index a5b311e..0000000 --- a/terraform/aws/member-account/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Member-account root - -This independently deployable module can: - -- create a secure local AWS Config recorder and delivery bucket; -- authorize one exact central account/region to aggregate Config; -- create an EC2 inventory role trusted by the exact central snapshot/signal IAM roles plus external ID; and -- forward only the inventory runtime's supported EC2 API calls and EC2 instance state-change hints from the default bus to one exact central bus ARN. - -Config recording and EventBridge forwarding are optional. Forwarding defaults disabled, -and no AWS Organizations integration is required. API-call forwarding cannot be enabled -with `cloudtrail_mode = "disabled"`: choose `create` for a project-owned multi-Region -management trail or `existing` with a live-verified member/organization trail ARN. The -collector policy contains only `ec2:Describe*`; its trust policy names only the supplied -central roles, and the forwarding role contains only `events:PutEvents` for the supplied -bus. - -Config aggregation authorization does not create or verify a central aggregator and does not guarantee that every desired region/resource is being recorded. Apply member authorization before configuring that account as a source of a central aggregator, and verify recorder health and aggregator freshness operationally. The exact collector role and external ID remain available for direct EC2 snapshots where Config coverage is insufficient. Set one shared `collector_role_name` and external ID across directly collected accounts so the central runtime can derive its account-scoped role template. - -Both Config recording and EventBridge rules are regional. The aggregation authorization -runs in this provider Region and authorizes the exact central aggregator Region. The -application's create-mode aggregator includes only its own provider source Region, so -other source Regions need an existing aggregator configured for them or direct EC2 -snapshots. The `signal_region` output is the only Region whose API/state hints this -module instance forwards. Deploy separate uniquely named/state roots for additional -hot-path Regions; one multi-Region CloudTrail does not make an EventBridge rule global. - -Cost and destruction warning: Config recording, S3 versions, and EventBridge delivery can incur ongoing charges. The Config bucket does not force-delete, so preserve or explicitly empty retained versions before retirement. diff --git a/terraform/aws/member-account/main.tf b/terraform/aws/member-account/main.tf index 98aab9b..75e3773 100644 --- a/terraform/aws/member-account/main.tf +++ b/terraform/aws/member-account/main.tf @@ -64,12 +64,6 @@ variable "collector_role_name" { } } -variable "central_collector_principal_arn" { - description = "Optional snapshot-only central IAM role ARN retained for compatibility." - type = string - default = null -} - variable "central_collector_principal_arns" { description = "Exact central snapshot and signal IAM role ARNs trusted to assume the member collector role." type = set(string) @@ -183,15 +177,10 @@ locals { cloudtrail = "${var.name_prefix}-ec2-write-forward" state = "${var.name_prefix}-ec2-state-forward" } - collector_principal_arns = setunion( - var.central_collector_principal_arns, - var.central_collector_principal_arn == null ? toset([]) : toset([ - var.central_collector_principal_arn - ]) - ) - config_source_arn = "arn:${data.aws_partition.current.partition}:config:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:*" - cloudtrail_name = "${var.name_prefix}-management" - created_cloudtrail_arn = "arn:${data.aws_partition.current.partition}:cloudtrail:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:trail/${local.cloudtrail_name}" + collector_principal_arns = var.central_collector_principal_arns + config_source_arn = "arn:${data.aws_partition.current.partition}:config:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:*" + cloudtrail_name = "${var.name_prefix}-management" + created_cloudtrail_arn = "arn:${data.aws_partition.current.partition}:cloudtrail:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:trail/${local.cloudtrail_name}" effective_cloudtrail_arn = ( var.cloudtrail_mode == "create" ? local.created_cloudtrail_arn : var.cloudtrail_mode == "existing" ? var.existing_cloudtrail_arn : diff --git a/terraform/aws/modules/database/main.tf b/terraform/aws/modules/database/main.tf index e5a3bd4..ce203cc 100644 --- a/terraform/aws/modules/database/main.tf +++ b/terraform/aws/modules/database/main.tf @@ -149,14 +149,19 @@ resource "aws_security_group" "database" { } locals { - database_client_security_group_ids = setunion( - toset([var.application_security_group_id]), - var.additional_client_security_group_ids + database_client_security_groups = merge( + { + application = var.application_security_group_id + }, + { + for security_group_id in var.additional_client_security_group_ids : + "additional-${security_group_id}" => security_group_id + } ) } resource "aws_vpc_security_group_ingress_rule" "database" { - for_each = local.database_client_security_group_ids + for_each = local.database_client_security_groups security_group_id = aws_security_group.database.id referenced_security_group_id = each.value diff --git a/terraform/aws/modules/eks/main.tf b/terraform/aws/modules/eks/main.tf index f993a53..65625a2 100644 --- a/terraform/aws/modules/eks/main.tf +++ b/terraform/aws/modules/eks/main.tf @@ -314,9 +314,14 @@ data "aws_ec2_instance_type" "node" { } locals { - api_client_security_group_ids = setunion( - toset([var.generator_security_group_id]), - var.additional_api_client_security_group_ids + api_client_security_groups = merge( + { + generator = var.generator_security_group_id + }, + { + for security_group_id in var.additional_api_client_security_group_ids : + "additional-${security_group_id}" => security_group_id + } ) } @@ -482,7 +487,7 @@ resource "aws_eks_cluster" "this" { } resource "aws_vpc_security_group_ingress_rule" "api_client" { - for_each = local.api_client_security_group_ids + for_each = local.api_client_security_groups security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id referenced_security_group_id = each.value diff --git a/terraform/aws/modules/functions/main.tf b/terraform/aws/modules/functions/main.tf index 290ee5a..02acfc3 100644 --- a/terraform/aws/modules/functions/main.tf +++ b/terraform/aws/modules/functions/main.tf @@ -32,12 +32,36 @@ variable "enable_event_dispatch" { default = false } -variable "enable_automatic_inventory" { - description = "Enable recurring inventory and repair schedules after queue and stream dispatch is active." +variable "periodic_snapshots_enabled" { + description = "Enable authoritative periodic snapshot schedules independently." type = bool default = false } +variable "periodic_coverage_enabled" { + description = "Enable recurring coverage requests and their generator consumer independently." + type = bool + default = false +} + +variable "signal_hints_enabled" { + description = "Enable the signal queue consumer independently." + type = bool + default = false +} + +variable "processor_reconciliation_enabled" { + description = "Enable scheduled PostgreSQL finding reconciliation independently." + type = bool + default = false +} + +variable "finding_export_enabled" { + description = "Enable optional finding handoff creation and S3 publication." + type = bool + default = true +} + variable "canary_mode" { description = "Require the snapshot Lambda to accept only explicit one-target canary invocations." type = bool @@ -157,6 +181,45 @@ variable "allowed_tag_keys" { default = [] } +variable "allowed_eni_interface_types" { + description = "Optional supported ENI interface classes; empty preserves existing direct-EC2 onboarding." + type = set(string) + default = [] +} + +variable "required_target_tag_key" { + description = "Optional exact ENI opt-in tag key paired with required_target_tag_value." + type = string + default = null +} + +variable "required_target_tag_value" { + description = "Optional exact ENI opt-in tag value paired with required_target_tag_key." + type = string + default = null +} + +variable "snapshot_max_pages" { + description = "Hard per-operation snapshot pagination limit." + type = number + default = 1000 +} + +variable "managed_canary" { + description = "Trusted Terraform-owned canary identity; null retains the advanced external-canary path." + type = object({ + account_id = string + region = string + network_interface_id = string + private_ip = string + public_ip = string + tag_key = string + tag_value = string + tcp_port = number + }) + default = null +} + variable "target_event_prefix" { description = "Immutable target-event object prefix." type = string @@ -330,6 +393,11 @@ variable "processor_schedule_expression" { } locals { + effective_periodic_snapshots_enabled = var.periodic_snapshots_enabled + effective_periodic_coverage_enabled = var.periodic_coverage_enabled + effective_signal_hints_enabled = var.signal_hints_enabled + effective_processor_reconciliation_enabled = var.processor_reconciliation_enabled + required_images = toset([ "inventory", "generator", @@ -350,6 +418,24 @@ locals { DISCOVERY_EXTERNAL_ID = one(local.member_collector_external_id_values) }) + managed_canary_environment = var.managed_canary == null ? tomap({}) : tomap({ + MANAGED_CANARY_ACCOUNT_ID = var.managed_canary.account_id + MANAGED_CANARY_REGION = var.managed_canary.region + MANAGED_CANARY_ENI_ID = var.managed_canary.network_interface_id + MANAGED_CANARY_PRIVATE_IP = var.managed_canary.private_ip + MANAGED_CANARY_PUBLIC_IP = var.managed_canary.public_ip + MANAGED_CANARY_TAG_KEY = var.managed_canary.tag_key + MANAGED_CANARY_TAG_VALUE = var.managed_canary.tag_value + MANAGED_CANARY_TCP_PORT = tostring(var.managed_canary.tcp_port) + }) + + required_target_tag_environment = ( + var.required_target_tag_key == null || var.required_target_tag_value == null + ) ? tomap({}) : tomap({ + REQUIRED_TARGET_TAG_KEY = var.required_target_tag_key + REQUIRED_TARGET_TAG_VALUE = var.required_target_tag_value + }) + inventory_environment = merge({ INVENTORY_TABLE = var.table_names["inventory"] TARGET_EVENT_BUCKET = var.bucket_names["events"] @@ -359,13 +445,21 @@ locals { AWS_REGIONS = join(",", var.snapshot_regions) AUTHORIZED_ACCOUNT_IDS = join(",", sort(tolist(var.authorized_account_ids))) ALLOWED_TAG_KEYS = join(",", sort(tolist(var.allowed_tag_keys))) - }, local.member_discovery_environment) + ALLOWED_TARGET_CIDRS = join(",", sort(tolist(var.allowed_target_cidrs))) + DENIED_TARGET_CIDRS = join(",", sort(tolist(var.denied_target_cidrs))) + ALLOWED_ENI_INTERFACE_TYPES = join( + ",", + sort(tolist(var.allowed_eni_interface_types)) + ) + SNAPSHOT_MAX_PAGES = tostring(var.snapshot_max_pages) + }, local.member_discovery_environment, local.required_target_tag_environment) snapshot_environment = merge( local.inventory_environment, { CANARY_MODE = tostring(var.canary_mode) }, + local.managed_canary_environment, var.snapshot_backend == "config" ? { CONFIG_AGGREGATOR_NAME = var.config_aggregator_name } : { @@ -374,19 +468,26 @@ locals { ) generator_environment = merge(local.inventory_environment, { - TARGET_EVENT_BUCKET = var.bucket_names["events"] - TARGET_EVENT_PREFIX = var.target_event_prefix - IDEMPOTENCY_TABLE = var.table_names["dispatch"] - INVENTORY_TABLE = var.table_names["inventory"] - EKS_CLUSTER_NAME = var.eks_cluster_name - K8S_NAMESPACE = var.eks_namespace - K8S_API_GROUP = "scanning.portscanner.io" - K8S_API_VERSION = "v1alpha1" - K8S_CRD_PLURAL = "scanners" - ALLOWED_TARGET_CIDRS = join(",", sort(tolist(var.allowed_target_cidrs))) - DENIED_TARGET_CIDRS = join(",", sort(tolist(var.denied_target_cidrs))) + TARGET_EVENT_BUCKET = var.bucket_names["events"] + TARGET_EVENT_PREFIX = var.target_event_prefix + IDEMPOTENCY_TABLE = var.table_names["dispatch"] + INVENTORY_TABLE = var.table_names["inventory"] + EKS_CLUSTER_NAME = var.eks_cluster_name + K8S_NAMESPACE = var.eks_namespace + K8S_API_GROUP = "scanning.portscanner.io" + K8S_API_VERSION = "v1alpha1" + K8S_CRD_PLURAL = "scanners" }) + finding_environment = merge( + { + FINDING_EXPORT_ENABLED = tostring(var.finding_export_enabled) + }, + var.finding_export_enabled ? { + FINDING_BUCKET = var.bucket_names["findings"] + } : {} + ) + function_definitions = { snapshot = { image = "inventory" @@ -448,12 +549,11 @@ locals { command = "act_parser.target_handler.lambda_handler" timeout = var.lambda_timeout_seconds memory = var.memory_size_mb - environment = { + environment = merge({ TARGET_EVENT_BUCKET = var.bucket_names["events"] TARGET_EVENT_PREFIX = var.target_event_prefix - FINDING_BUCKET = var.bucket_names["findings"] DB_SECRET_ID = var.database_application_secret_arn - } + }, local.finding_environment) } parser = { image = "parser" @@ -461,12 +561,11 @@ locals { command = "act_parser.handler.lambda_handler" timeout = var.lambda_timeout_seconds memory = var.memory_size_mb - environment = { + environment = merge({ SCAN_RESULT_BUCKET = var.bucket_names["results"] RAW_RESULT_BUCKET = var.bucket_names["results"] - FINDING_BUCKET = var.bucket_names["findings"] DB_SECRET_ID = var.database_application_secret_arn - } + }, local.finding_environment) } processor = { image = "processor" @@ -474,10 +573,13 @@ locals { command = "act_processor.handler.lambda_handler" timeout = var.lambda_timeout_seconds memory = var.memory_size_mb - environment = { - FINDING_BUCKET = var.bucket_names["findings"] - DB_SECRET_ID = var.database_application_secret_arn - } + environment = merge( + { + DB_SECRET_ID = var.database_application_secret_arn + }, + local.finding_environment, + local.managed_canary_environment + ) } migrator = { image = "migrator" @@ -518,7 +620,10 @@ locals { queue = "result" } } - automatic_sqs_mappings = toset(["signals", "generator_coverage"]) + staged_sqs_mapping_enabled = { + signals = local.effective_signal_hints_enabled + generator_coverage = local.effective_periodic_coverage_enabled + } snapshot_scope_account_ids = length(var.authorized_account_ids) > 0 ? var.authorized_account_ids : toset([ var.snapshot_account_id @@ -528,7 +633,7 @@ locals { "snapshot-${account_id}" => { expression = var.snapshot_schedule_expression function = "snapshot" - gate = "automatic" + gate = "periodic-snapshots" input = { account_id = account_id } @@ -538,13 +643,13 @@ locals { rescan = { expression = var.rescan_schedule_expression function = "rescan" - gate = "automatic" + gate = "periodic-coverage" input = null } processor = { expression = var.processor_schedule_expression function = "processor" - gate = "automatic" + gate = "processor-reconciliation" input = null } "outbox-replay" = { @@ -585,17 +690,30 @@ resource "terraform_data" "runtime_validation" { } precondition { - condition = !var.enable_automatic_inventory || var.enable_event_dispatch - error_message = "enable_automatic_inventory requires enable_event_dispatch." + condition = !( + local.effective_periodic_snapshots_enabled || + local.effective_periodic_coverage_enabled || + local.effective_signal_hints_enabled + ) || var.enable_event_dispatch + error_message = "Periodic snapshots, coverage, and signal hints require enable_event_dispatch." + } + + precondition { + condition = !local.effective_processor_reconciliation_enabled || ( + var.deploy_runtime && var.run_migration + ) + error_message = "Processor reconciliation requires deployed, migrated runtime." } precondition { condition = !var.canary_mode || ( var.deploy_runtime && var.run_migration && - !var.enable_automatic_inventory + !local.effective_periodic_snapshots_enabled && + !local.effective_periodic_coverage_enabled && + !local.effective_signal_hints_enabled ) - error_message = "canary_mode requires migrated runtime and automatic inventory disabled." + error_message = "canary_mode requires migrated runtime with recurring inventory and signal hints disabled." } precondition { @@ -646,6 +764,44 @@ resource "terraform_data" "runtime_validation" { error_message = "allowed_tag_keys contains a key outside the shared AWS contract." } + precondition { + condition = ( + var.required_target_tag_key == null && + var.required_target_tag_value == null + ) || ( + try(length(var.required_target_tag_key) > 0, false) && + try(length(var.required_target_tag_value) > 0, false) + ) + error_message = "required_target_tag_key and required_target_tag_value must be configured together." + } + + precondition { + condition = var.snapshot_max_pages >= 1 && var.snapshot_max_pages <= 10000 + error_message = "snapshot_max_pages must be between 1 and 10000." + } + + precondition { + condition = var.managed_canary == null ? true : ( + var.managed_canary.tcp_port == 18080 && + ( + !var.canary_mode || + ( + contains(var.authorized_account_ids, var.managed_canary.account_id) && + contains(var.snapshot_regions, var.managed_canary.region) && + contains(var.allowed_target_cidrs, "${var.managed_canary.public_ip}/32") + ) + ) + ) + error_message = "Canary mode requires the managed canary to be the authorized exact /32 target on TCP 18080." + } + + precondition { + condition = !var.finding_export_enabled || ( + contains(keys(var.bucket_names), "findings") + ) + error_message = "finding_export_enabled requires the findings bucket." + } + precondition { condition = alltrue([ for function_name in keys(local.function_definitions) : @@ -782,9 +938,11 @@ resource "aws_lambda_event_source_mapping" "sqs" { event_source_arn = var.queue_arns[each.value.queue] function_name = aws_lambda_function.this[each.value.function].arn - enabled = contains(local.automatic_sqs_mappings, each.key) ? ( - var.enable_automatic_inventory - ) : var.enable_event_dispatch + enabled = lookup( + local.staged_sqs_mapping_enabled, + each.key, + var.enable_event_dispatch + ) batch_size = 5 function_response_types = ["ReportBatchItemFailures"] @@ -856,9 +1014,12 @@ resource "aws_cloudwatch_event_rule" "schedule" { name = "${var.name_prefix}-${each.key}-schedule" description = "Conservative ${each.key} schedule" schedule_expression = each.value.expression - state = each.value.gate == "dispatch" ? ( - var.enable_event_dispatch ? "ENABLED" : "DISABLED" - ) : (var.enable_automatic_inventory ? "ENABLED" : "DISABLED") + state = ( + each.value.gate == "dispatch" ? var.enable_event_dispatch : + each.value.gate == "periodic-snapshots" ? local.effective_periodic_snapshots_enabled : + each.value.gate == "periodic-coverage" ? local.effective_periodic_coverage_enabled : + local.effective_processor_reconciliation_enabled + ) ? "ENABLED" : "DISABLED" } resource "aws_cloudwatch_event_target" "schedule" { @@ -887,6 +1048,36 @@ output "function_arns" { value = { for name, function in aws_lambda_function.this : name => function.arn } } +output "function_names" { + value = { for name, function in aws_lambda_function.this : name => function.function_name } +} + +output "snapshot_function_arn" { + value = try(aws_lambda_function.this["snapshot"].arn, null) +} + +output "snapshot_function_name" { + value = try(aws_lambda_function.this["snapshot"].function_name, null) +} + +output "processor_function_arn" { + value = try(aws_lambda_function.this["processor"].arn, null) +} + +output "processor_function_name" { + value = try(aws_lambda_function.this["processor"].function_name, null) +} + +output "stage_gates" { + value = { + periodic_snapshots_enabled = local.effective_periodic_snapshots_enabled + periodic_coverage_enabled = local.effective_periodic_coverage_enabled + signal_hints_enabled = local.effective_signal_hints_enabled + processor_reconciliation_enabled = local.effective_processor_reconciliation_enabled + finding_export_enabled = var.finding_export_enabled + } +} + output "event_source_mapping_uuids" { description = "Exact Lambda mapping UUIDs managed by this module for AWS-only emergency pause." value = concat( diff --git a/terraform/aws/modules/functions/main.tftest.hcl b/terraform/aws/modules/functions/main.tftest.hcl index 1139d0b..79fb9ca 100644 --- a/terraform/aws/modules/functions/main.tftest.hcl +++ b/terraform/aws/modules/functions/main.tftest.hcl @@ -105,9 +105,8 @@ run "canary_enables_pipeline_without_automatic_inventory" { command = plan variables { - enable_event_dispatch = true - enable_automatic_inventory = false - canary_mode = true + enable_event_dispatch = true + canary_mode = true } assert { @@ -141,9 +140,8 @@ run "paused_canary_retains_runtime_guard_with_dispatch_disabled" { command = plan variables { - enable_event_dispatch = false - enable_automatic_inventory = false - canary_mode = true + enable_event_dispatch = false + canary_mode = true } assert { @@ -164,7 +162,7 @@ run "automatic_inventory_requires_dispatch_pipeline" { variables { enable_event_dispatch = false - enable_automatic_inventory = true + periodic_snapshots_enabled = true } expect_failures = [terraform_data.runtime_validation] @@ -174,8 +172,11 @@ run "activation_enables_recurring_schedules" { command = plan variables { - enable_event_dispatch = true - enable_automatic_inventory = true + enable_event_dispatch = true + periodic_snapshots_enabled = true + periodic_coverage_enabled = true + signal_hints_enabled = true + processor_reconciliation_enabled = true } assert { @@ -192,3 +193,123 @@ run "activation_enables_recurring_schedules" { error_message = "Full activation must enable every recurring schedule." } } + +run "independent_stage_gates_control_each_recurring_path" { + command = plan + + variables { + enable_event_dispatch = true + periodic_snapshots_enabled = false + periodic_coverage_enabled = true + signal_hints_enabled = false + processor_reconciliation_enabled = true + finding_export_enabled = false + } + + assert { + condition = ( + aws_lambda_event_source_mapping.sqs["signals"].enabled == false && + aws_lambda_event_source_mapping.sqs["generator_coverage"].enabled && + alltrue([ + for name, rule in aws_cloudwatch_event_rule.schedule : + rule.state == ( + startswith(name, "snapshot-") ? "DISABLED" : + name == "rescan" ? "ENABLED" : + name == "processor" ? "ENABLED" : + "ENABLED" + ) + ]) + ) + error_message = "Each recurring stage must follow its own explicit gate." + } + + assert { + condition = ( + aws_lambda_function.this["parser"].environment[0].variables["FINDING_EXPORT_ENABLED"] == "false" && + !contains(keys(aws_lambda_function.this["parser"].environment[0].variables), "FINDING_BUCKET") && + !contains(keys(aws_lambda_function.this["target_projector"].environment[0].variables), "FINDING_BUCKET") && + !contains(keys(aws_lambda_function.this["processor"].environment[0].variables), "FINDING_BUCKET") + ) + error_message = "Database-only runtimes must not require a finding bucket environment variable." + } +} + +run "managed_canary_identity_is_trusted_runtime_configuration" { + command = plan + + variables { + enable_event_dispatch = true + canary_mode = true + allowed_tag_keys = ["application", "service"] + managed_canary = { + account_id = "123456789012" + region = "us-east-1" + network_interface_id = "eni-0123456789abcdef0" + private_ip = "10.255.255.4" + public_ip = "203.0.113.10" + tag_key = "service" + tag_value = "test-managed-canary" + tcp_port = 18080 + } + } + + assert { + condition = ( + aws_lambda_function.this["snapshot"].environment[0].variables["MANAGED_CANARY_ENI_ID"] == "eni-0123456789abcdef0" && + aws_lambda_function.this["snapshot"].environment[0].variables["MANAGED_CANARY_TCP_PORT"] == "18080" && + aws_lambda_function.this["processor"].environment[0].variables["MANAGED_CANARY_PUBLIC_IP"] == "203.0.113.10" + ) + error_message = "Snapshot and status operations must use Terraform-owned canary identity." + } +} + +run "processor_reconciliation_is_independent_of_dispatch" { + command = plan + + variables { + enable_event_dispatch = false + periodic_snapshots_enabled = false + periodic_coverage_enabled = false + signal_hints_enabled = false + processor_reconciliation_enabled = true + } + + assert { + condition = ( + aws_cloudwatch_event_rule.schedule["processor"].state == "ENABLED" && + aws_cloudwatch_event_rule.schedule["outbox-replay"].state == "DISABLED" && + alltrue([ + for mapping in values(aws_lambda_event_source_mapping.sqs) : + mapping.enabled == false + ]) && + aws_lambda_event_source_mapping.outbox[0].enabled == false + ) + error_message = "Database reconciliation must be independently schedulable without dispatch." + } +} + +run "managed_target_does_not_narrow_normal_inventory_scope" { + command = plan + + variables { + allowed_target_cidrs = [] + managed_canary = { + account_id = "123456789012" + region = "us-east-1" + network_interface_id = "eni-0123456789abcdef0" + private_ip = "10.255.255.4" + public_ip = "203.0.113.10" + tag_key = "service" + tag_value = "test-managed-canary" + tcp_port = 18080 + } + } + + assert { + condition = ( + aws_lambda_function.this["snapshot"].environment[0].variables["CANARY_MODE"] == "false" && + aws_lambda_function.this["snapshot"].environment[0].variables["ALLOWED_TARGET_CIDRS"] == "" + ) + error_message = "A retained managed canary must not silently narrow advanced normal-inventory scope." + } +} diff --git a/terraform/aws/modules/identities/main.tf b/terraform/aws/modules/identities/main.tf index 77cff63..d8c4264 100644 --- a/terraform/aws/modules/identities/main.tf +++ b/terraform/aws/modules/identities/main.tf @@ -74,6 +74,12 @@ variable "finding_object_prefix" { default = "findings/" } +variable "finding_export_enabled" { + description = "Grant finding-object publication only when the optional export integration exists." + type = bool + default = true +} + data "aws_caller_identity" "current" {} data "aws_partition" "current" {} data "aws_region" "current" {} @@ -309,7 +315,7 @@ locals { Resource = var.member_collector_role_arns } ]) - target_projector = [ + target_projector = concat([ { Sid = "ConsumeTargetEvents" Effect = "Allow" @@ -329,7 +335,8 @@ locals { "s3:GetObjectVersion" ] Resource = ["${var.bucket_arns["events"]}/${var.target_event_object_prefix}*"] - }, + } + ], var.finding_export_enabled ? [ { Sid = "PublishFindingObjects" Effect = "Allow" @@ -338,15 +345,16 @@ locals { "s3:PutObject" ] Resource = ["${var.bucket_arns["findings"]}/${var.finding_object_prefix}*"] - }, + } + ] : [], [ { Sid = "ReadApplicationDatabaseCredential" Effect = "Allow" Action = ["secretsmanager:GetSecretValue"] Resource = [var.database_application_secret_arn] } - ] - parser = [ + ]) + parser = concat([ { Sid = "ConsumeScanResults" Effect = "Allow" @@ -366,7 +374,8 @@ locals { "s3:GetObjectVersion" ] Resource = ["${var.bucket_arns["results"]}/${var.result_object_prefix}*"] - }, + } + ], var.finding_export_enabled ? [ { Sid = "PublishFindingObjects" Effect = "Allow" @@ -375,15 +384,16 @@ locals { "s3:PutObject" ] Resource = ["${var.bucket_arns["findings"]}/${var.finding_object_prefix}*"] - }, + } + ] : [], [ { Sid = "ReadApplicationDatabaseCredential" Effect = "Allow" Action = ["secretsmanager:GetSecretValue"] Resource = [var.database_application_secret_arn] } - ] - processor = [ + ]) + processor = concat(var.finding_export_enabled ? [ { Sid = "PublishFindingObjects" Effect = "Allow" @@ -392,14 +402,15 @@ locals { "s3:PutObject" ] Resource = ["${var.bucket_arns["findings"]}/${var.finding_object_prefix}*"] - }, + } + ] : [], [ { Sid = "ReadApplicationDatabaseCredential" Effect = "Allow" Action = ["secretsmanager:GetSecretValue"] Resource = [var.database_application_secret_arn] } - ] + ]) migrator = [ { Sid = "ReadMasterCredential" diff --git a/terraform/aws/modules/identities/main.tftest.hcl b/terraform/aws/modules/identities/main.tftest.hcl new file mode 100644 index 0000000..30d8f63 --- /dev/null +++ b/terraform/aws/modules/identities/main.tftest.hcl @@ -0,0 +1,74 @@ +mock_provider "aws" { + mock_data "aws_caller_identity" { + defaults = { + account_id = "123456789012" + } + } + + mock_data "aws_partition" { + defaults = { + partition = "aws" + } + } + + mock_data "aws_region" { + defaults = { + region = "us-east-1" + } + } + + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} + +variables { + name_prefix = "test-identities" + bucket_arns = { + events = "arn:aws:s3:::test-events" + results = "arn:aws:s3:::test-results" + } + queue_arns = { + signal = "arn:aws:sqs:us-east-1:123456789012:test-signal" + priority = "arn:aws:sqs:us-east-1:123456789012:test-priority" + coverage = "arn:aws:sqs:us-east-1:123456789012:test-coverage" + "target-event" = "arn:aws:sqs:us-east-1:123456789012:test-target-event" + result = "arn:aws:sqs:us-east-1:123456789012:test-result" + } + table_arns = { + inventory = "arn:aws:dynamodb:us-east-1:123456789012:table/test-inventory" + dispatch = "arn:aws:dynamodb:us-east-1:123456789012:table/test-dispatch" + } + table_stream_arns = { + inventory = "arn:aws:dynamodb:us-east-1:123456789012:table/test-inventory/stream/test" + } + database_master_secret_arn = "arn:aws:secretsmanager:us-east-1:123456789012:secret:test-master" + database_application_secret_arn = "arn:aws:secretsmanager:us-east-1:123456789012:secret:test-application" + eks_cluster_arn = "arn:aws:eks:us-east-1:123456789012:cluster/test" + finding_export_enabled = false +} + +run "database_boundary_omits_finding_object_permissions" { + command = plan + + assert { + condition = ( + !strcontains(aws_iam_role_policy.function["target_projector"].policy, "PublishFindingObjects") && + !strcontains(aws_iam_role_policy.function["parser"].policy, "PublishFindingObjects") && + !strcontains(aws_iam_role_policy.function["processor"].policy, "PublishFindingObjects") && + !strcontains(aws_iam_role_policy.function["parser"].policy, "test-findings") + ) + error_message = "Database-only runtimes must not require finding bucket permissions." + } + + assert { + condition = ( + strcontains(aws_iam_role_policy.function["target_projector"].policy, "test-events") && + strcontains(aws_iam_role_policy.function["parser"].policy, "test-results") && + strcontains(aws_iam_role_policy.function["processor"].policy, "ReadApplicationDatabaseCredential") + ) + error_message = "Core database and input-object permissions must remain present." + } +} diff --git a/terraform/aws/modules/managed-canary/main.tf b/terraform/aws/modules/managed-canary/main.tf new file mode 100644 index 0000000..9c9a71a --- /dev/null +++ b/terraform/aws/modules/managed-canary/main.tf @@ -0,0 +1,352 @@ +terraform { + required_version = "= 1.7.4" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "= 6.15.0" + } + } +} + +variable "name_prefix" { + description = "Portable prefix for the isolated managed canary." + type = string +} + +variable "vpc_cidr" { + description = "Dedicated unpeered canary VPC and public-subnet CIDR." + type = string + default = "10.255.255.0/28" + + validation { + condition = ( + can(cidrnetmask(var.vpc_cidr)) && + try(cidrhost(var.vpc_cidr, 0) == split("/", var.vpc_cidr)[0], false) && + try(tonumber(split("/", var.vpc_cidr)[1]) == 28, false) + ) + error_message = "vpc_cidr must be a canonical IPv4 /28." + } +} + +variable "scanner_source_ipv4_cidrs" { + description = "Exact scanner NAT EIP /32s allowed to reach the canary listener." + type = list(string) + + validation { + condition = alltrue([ + for cidr in var.scanner_source_ipv4_cidrs : + cidr != "0.0.0.0/0" && + can(cidrnetmask(cidr)) && + try(tonumber(split("/", cidr)[1]) == 32, false) && + try(cidrhost(cidr, 0) == split("/", cidr)[0], false) + ]) + error_message = "scanner_source_ipv4_cidrs must contain only canonical IPv4 /32s." + } +} + +variable "instance_type" { + description = "Small ARM instance type used only for the harmless listener." + type = string + default = "t4g.nano" +} + +variable "listener_port" { + description = "Single TCP port exposed by the managed canary." + type = number + default = 18080 + + validation { + condition = var.listener_port == 18080 + error_message = "The managed canary listener is intentionally fixed to TCP 18080." + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +data "aws_ami" "al2023" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["al2023-ami-2023.*-arm64"] + } + + filter { + name = "architecture" + values = ["arm64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +locals { + inventory_tag_key = "service" + inventory_tag_value = "${var.name_prefix}-managed-canary" + canary_tags = { + Name = "${var.name_prefix}-managed-canary" + application = "portscanner" + service = local.inventory_tag_value + } +} + +resource "terraform_data" "validation" { + input = var.name_prefix + + lifecycle { + precondition { + condition = length(var.scanner_source_ipv4_cidrs) > 0 + error_message = "At least one scanner NAT EIP /32 is required for the managed canary." + } + + precondition { + condition = length(data.aws_availability_zones.available.names) > 0 + error_message = "The managed canary requires one available availability zone." + } + } +} + +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_hostnames = false + enable_dns_support = true + + tags = { + Name = "${var.name_prefix}-managed-canary" + } +} + +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + + tags = { + Name = "${var.name_prefix}-managed-canary" + } +} + +resource "aws_subnet" "public" { + vpc_id = aws_vpc.this.id + availability_zone = data.aws_availability_zones.available.names[0] + cidr_block = var.vpc_cidr + map_public_ip_on_launch = false + + tags = { + Name = "${var.name_prefix}-managed-canary" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + + tags = { + Name = "${var.name_prefix}-managed-canary" + } +} + +resource "aws_route" "internet" { + route_table_id = aws_route_table.public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.this.id +} + +resource "aws_route_table_association" "public" { + route_table_id = aws_route_table.public.id + subnet_id = aws_subnet.public.id +} + +resource "aws_security_group" "listener" { + name_prefix = "${substr(var.name_prefix, 0, 28)}-canary-" + description = "Managed canary listener with no outbound rules" + vpc_id = aws_vpc.this.id + revoke_rules_on_delete = true + egress = [] + + tags = { + Name = "${var.name_prefix}-managed-canary" + } +} + +resource "aws_vpc_security_group_ingress_rule" "scanner" { + count = length(var.scanner_source_ipv4_cidrs) + + security_group_id = aws_security_group.listener.id + description = "TCP 18080 from scanner NAT EIP" + cidr_ipv4 = var.scanner_source_ipv4_cidrs[count.index] + from_port = var.listener_port + to_port = var.listener_port + ip_protocol = "tcp" +} + +resource "aws_network_interface" "this" { + subnet_id = aws_subnet.public.id + security_groups = [aws_security_group.listener.id] + source_dest_check = true + + tags = local.canary_tags +} + +resource "aws_instance" "this" { + ami = data.aws_ami.al2023.id + instance_type = var.instance_type + + network_interface { + network_interface_id = aws_network_interface.this.id + device_index = 0 + } + + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + instance_metadata_tags = "disabled" + } + + root_block_device { + delete_on_termination = true + encrypted = true + volume_size = 8 + volume_type = "gp3" + } + + instance_initiated_shutdown_behavior = "stop" + monitoring = false + user_data_replace_on_change = true + user_data = <<-USER_DATA + #!/bin/bash + set -euo pipefail + + cat >/usr/local/bin/portscanner-canary.py <<'PYTHON' + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"portscanner managed canary\n" + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + return + + + ThreadingHTTPServer(("0.0.0.0", ${var.listener_port}), Handler).serve_forever() + PYTHON + chmod 0755 /usr/local/bin/portscanner-canary.py + + cat >/etc/systemd/system/portscanner-canary.service <<'UNIT' + [Unit] + Description=Portscanner managed canary HTTP listener + After=network.target + + [Service] + Type=simple + ExecStart=/usr/bin/python3 /usr/local/bin/portscanner-canary.py + Restart=always + RestartSec=2 + DynamicUser=yes + NoNewPrivileges=yes + PrivateDevices=yes + ProtectHome=yes + ProtectSystem=strict + + [Install] + WantedBy=multi-user.target + UNIT + + systemctl daemon-reload + systemctl enable --now portscanner-canary.service + USER_DATA + + tags = local.canary_tags + volume_tags = local.canary_tags + + depends_on = [ + aws_route_table_association.public, + terraform_data.validation, + ] +} + +resource "aws_eip" "this" { + domain = "vpc" + + tags = { + Name = "${var.name_prefix}-managed-canary" + application = "portscanner" + service = local.inventory_tag_value + } + + depends_on = [aws_internet_gateway.this] +} + +resource "aws_eip_association" "this" { + allocation_id = aws_eip.this.id + network_interface_id = aws_network_interface.this.id + + depends_on = [aws_instance.this] +} + +output "vpc_id" { + value = aws_vpc.this.id +} + +output "subnet_id" { + value = aws_subnet.public.id +} + +output "security_group_id" { + value = aws_security_group.listener.id +} + +output "eip_allocation_id" { + value = aws_eip.this.allocation_id +} + +output "public_ip" { + value = aws_eip.this.public_ip +} + +output "public_cidr" { + value = "${aws_eip.this.public_ip}/32" +} + +output "network_interface_id" { + value = aws_network_interface.this.id +} + +output "private_ip" { + value = aws_network_interface.this.private_ip +} + +output "instance_id" { + value = aws_instance.this.id +} + +output "instance_state" { + value = aws_instance.this.instance_state +} + +output "listener_port" { + value = var.listener_port +} + +output "inventory_tag_key" { + value = local.inventory_tag_key +} + +output "inventory_tag_value" { + value = local.inventory_tag_value +} diff --git a/terraform/aws/modules/managed-canary/main.tftest.hcl b/terraform/aws/modules/managed-canary/main.tftest.hcl new file mode 100644 index 0000000..e37675f --- /dev/null +++ b/terraform/aws/modules/managed-canary/main.tftest.hcl @@ -0,0 +1,85 @@ +mock_provider "aws" { + mock_data "aws_availability_zones" { + defaults = { + names = ["us-east-1a"] + } + } + + mock_data "aws_ami" { + defaults = { + id = "ami-0123456789abcdef0" + } + } +} + +variables { + name_prefix = "test-scan" + scanner_source_ipv4_cidrs = [ + "198.51.100.10/32", + "203.0.113.20/32", + ] +} + +run "creates_isolated_minimal_target" { + command = plan + + assert { + condition = ( + aws_subnet.public.map_public_ip_on_launch == false && + aws_route.internet.destination_cidr_block == "0.0.0.0/0" && + length(aws_vpc_security_group_ingress_rule.scanner) == 2 + ) + error_message = "The canary must use its own public subnet and one ingress rule per scanner NAT EIP." + } + + assert { + condition = alltrue([ + for rule in aws_vpc_security_group_ingress_rule.scanner : + rule.from_port == 18080 && + rule.to_port == 18080 && + rule.ip_protocol == "tcp" + ]) + error_message = "The managed canary must expose only TCP 18080." + } + + assert { + condition = ( + length(aws_security_group.listener.egress) == 0 && + aws_instance.this.metadata_options[0].http_tokens == "required" && + aws_instance.this.metadata_options[0].http_put_response_hop_limit == 1 && + aws_instance.this.root_block_device[0].encrypted && + aws_instance.this.root_block_device[0].volume_type == "gp3" + ) + error_message = "The target must have no egress, require IMDSv2, and use an encrypted root disk." + } + + assert { + condition = ( + aws_network_interface.this.tags["service"] == "test-scan-managed-canary" && + output.listener_port == 18080 && + output.inventory_tag_key == "service" && + output.inventory_tag_value == "test-scan-managed-canary" + ) + error_message = "Inventory must receive a deterministic managed-canary identity." + } +} + +run "rejects_missing_scanner_nat_scope" { + command = plan + + variables { + scanner_source_ipv4_cidrs = [] + } + + expect_failures = [terraform_data.validation] +} + +run "rejects_non_exact_scanner_scope" { + command = plan + + variables { + scanner_source_ipv4_cidrs = ["198.51.100.0/24"] + } + + expect_failures = [var.scanner_source_ipv4_cidrs] +} diff --git a/terraform/aws/modules/signals/main.tf b/terraform/aws/modules/signals/main.tf index ca898fa..97acc4f 100644 --- a/terraform/aws/modules/signals/main.tf +++ b/terraform/aws/modules/signals/main.tf @@ -82,6 +82,7 @@ variable "cloudtrail_name" { variable "cloudtrail_bucket_name" { description = "Secure S3 bucket for a created management trail." type = string + default = null } variable "create_central_event_bus" { @@ -172,7 +173,8 @@ locals { account = [data.aws_caller_identity.current.account_id] }) if( ( - length(var.authorized_account_ids) == 0 || + length(var.authorized_account_ids) == 0 ? + true : contains(var.authorized_account_ids, data.aws_caller_identity.current.account_id) ) && (kind != "cloudtrail" || var.cloudtrail_mode != "disabled") @@ -232,6 +234,14 @@ resource "terraform_data" "signal_validation" { error_message = "existing_cloudtrail_arn is required only when cloudtrail_mode is existing." } + precondition { + condition = var.cloudtrail_mode != "create" || try( + length(var.cloudtrail_bucket_name) > 0, + false + ) + error_message = "cloudtrail_bucket_name is required when cloudtrail_mode is create." + } + precondition { condition = alltrue([ for account_id in setunion( diff --git a/terraform/aws/modules/signals/main.tftest.hcl b/terraform/aws/modules/signals/main.tftest.hcl index 3319662..ac0be60 100644 --- a/terraform/aws/modules/signals/main.tftest.hcl +++ b/terraform/aws/modules/signals/main.tftest.hcl @@ -61,8 +61,9 @@ run "disabled_cloudtrail_keeps_only_native_local_state_hint" { command = plan variables { - config_mode = "disabled" - cloudtrail_mode = "disabled" + config_mode = "disabled" + cloudtrail_mode = "disabled" + cloudtrail_bucket_name = null } assert { diff --git a/terraform/aws/modules/storage/main.tf b/terraform/aws/modules/storage/main.tf index f98b15e..a6d5d73 100644 --- a/terraform/aws/modules/storage/main.tf +++ b/terraform/aws/modules/storage/main.tf @@ -94,6 +94,18 @@ variable "enable_config_delivery" { default = false } +variable "finding_export_enabled" { + description = "Create finding S3/SQS integration storage and notifications." + type = bool + default = true +} + +variable "enable_cloudtrail_storage" { + description = "Create the CloudTrail bucket only for a trail managed by this stack." + type = bool + default = true +} + variable "signal_event_rule_arns" { description = "Exact EventBridge rule ARNs permitted to send to the central signal queue." type = list(string) @@ -144,8 +156,15 @@ data "aws_partition" "current" {} data "aws_region" "current" {} locals { - bucket_purposes = toset(["events", "results", "findings", "cloudtrail"]) - queue_purposes = toset(["signal", "priority", "coverage", "target-event", "result", "finding"]) + bucket_purposes = setunion( + toset(["events", "results"]), + var.finding_export_enabled ? toset(["findings"]) : toset([]), + var.enable_cloudtrail_storage ? toset(["cloudtrail"]) : toset([]) + ) + queue_purposes = setunion( + toset(["signal", "priority", "coverage", "target-event", "result"]), + var.finding_export_enabled ? toset(["finding"]) : toset([]) + ) config_source_arn = "arn:${data.aws_partition.current.partition}:config:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:*" } @@ -160,7 +179,7 @@ resource "terraform_data" "storage_validation" { precondition { condition = alltrue([for purpose in local.bucket_purposes : lookup(var.bucket_expiration_days, purpose, 0) > 0]) - error_message = "bucket_expiration_days must contain a positive value for events, results, findings, and cloudtrail." + error_message = "bucket_expiration_days must contain a positive value for every enabled bucket." } } } @@ -644,6 +663,8 @@ resource "aws_s3_bucket_notification" "results" { } resource "aws_s3_bucket_notification" "findings" { + count = var.finding_export_enabled ? 1 : 0 + bucket = aws_s3_bucket.data["findings"].id queue { @@ -784,6 +805,26 @@ output "bucket_policy_ids" { value = { for purpose, policy in aws_s3_bucket_policy.data : purpose => policy.id } } +output "finding_bucket_name" { + value = try(aws_s3_bucket.data["findings"].id, null) +} + +output "finding_bucket_arn" { + value = try(aws_s3_bucket.data["findings"].arn, null) +} + +output "finding_queue_url" { + value = try(aws_sqs_queue.main["finding"].url, null) +} + +output "finding_queue_arn" { + value = try(aws_sqs_queue.main["finding"].arn, null) +} + +output "cloudtrail_bucket_name" { + value = try(aws_s3_bucket.data["cloudtrail"].id, null) +} + output "alarm_names" { description = "CloudWatch storage alarm names keyed by alarm class and queue purpose." value = { diff --git a/terraform/aws/modules/storage/main.tftest.hcl b/terraform/aws/modules/storage/main.tftest.hcl index c78d2da..8f55797 100644 --- a/terraform/aws/modules/storage/main.tftest.hcl +++ b/terraform/aws/modules/storage/main.tftest.hcl @@ -60,3 +60,50 @@ run "reject_visibility_below_lambda_multiplier" { expect_failures = [terraform_data.storage_validation] } + +run "database_boundary_omits_finding_integration" { + command = plan + + variables { + finding_export_enabled = false + } + + assert { + condition = ( + !contains(keys(aws_s3_bucket.data), "findings") && + !contains(keys(aws_sqs_queue.main), "finding") && + !contains(keys(aws_sqs_queue.dead_letter), "finding") && + !contains(keys(aws_cloudwatch_metric_alarm.queue_age), "finding") && + !contains(keys(aws_cloudwatch_metric_alarm.dead_letter_messages), "finding") && + length(aws_s3_bucket_notification.findings) == 0 + ) + error_message = "Database-only mode must omit all finding export storage, notifications, and alarms." + } + + assert { + condition = ( + output.finding_bucket_name == null && + output.finding_bucket_arn == null && + output.finding_queue_url == null && + output.finding_queue_arn == null + ) + error_message = "Disabled finding export outputs must be safely nullable." + } +} + +run "disabled_cloudtrail_storage_is_nullable" { + command = plan + + variables { + enable_cloudtrail_storage = false + cloudtrail_source_arns = [] + } + + assert { + condition = ( + !contains(keys(aws_s3_bucket.data), "cloudtrail") && + output.cloudtrail_bucket_name == null + ) + error_message = "Disabled CloudTrail mode must not retain an unused CloudTrail bucket." + } +} diff --git a/terraform/aws/scripts/bootstrap.sh b/terraform/aws/scripts/bootstrap.sh new file mode 100755 index 0000000..b927410 --- /dev/null +++ b/terraform/aws/scripts/bootstrap.sh @@ -0,0 +1,723 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" +AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd -P)" +REPOSITORY_ROOT="$(CDPATH= cd -- "${AWS_DIR}/../.." && pwd -P)" +DEPLOYMENT_ROOT="${AWS_DIR}/deployment" +STATE_BOOTSTRAP_ROOT="${AWS_DIR}/state-bootstrap" + +usage() { + echo "usage: $0" >&2 + exit 2 +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +log() { + echo "$*" >&2 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" +} + +confirm_apply() { + local phrase="$1" + if [[ "${PORTSCANNER_AUTO_APPROVE:-false}" == "true" ]]; then + return 0 + fi + printf 'Type %s to apply this saved plan: ' "${phrase}" >&2 + read -r confirmation + [[ "${confirmation}" == "${phrase}" ]] || + die "stage not confirmed; saved plan discarded" +} + +verify_mode_0600() { + python3 - "$1" <<'PY' +import os +import pathlib +import stat +import sys + +path = pathlib.Path(sys.argv[1]) +if not path.is_file() or stat.S_IMODE(path.stat().st_mode) != 0o600: + raise SystemExit(f"{path} must be a mode-0600 regular file") +PY +} + +[[ $# -eq 0 ]] || usage + +for command_name in terraform aws docker jq python3 git tar; do + require_command "${command_name}" +done +docker buildx version >/dev/null 2>&1 || die "Docker buildx is required" +TRIVY_NAME="${TRIVY:-trivy}" +require_command "${TRIVY_NAME}" + +if ! TERRAFORM_VERSION_JSON="$(terraform version -json)"; then + die "Terraform version could not be determined" +fi +if ! jq -e '.terraform_version == "1.7.4"' <<<"${TERRAFORM_VERSION_JSON}" >/dev/null; then + die "Terraform 1.7.4 is required" +fi + +if ! RESOLVED_REPOSITORY_ROOT="$(git -C "${REPOSITORY_ROOT}" rev-parse --show-toplevel 2>/dev/null)"; then + die "repository root could not be resolved" +fi +RESOLVED_REPOSITORY_ROOT="$(CDPATH= cd -- "${RESOLVED_REPOSITORY_ROOT}" && pwd -P)" +[[ "${RESOLVED_REPOSITORY_ROOT}" == "${REPOSITORY_ROOT}" ]] || + die "bootstrap must run from the expected repository checkout" +if ! SOURCE_COMMIT="$(git -C "${REPOSITORY_ROOT}" rev-parse --verify 'HEAD^{commit}' 2>/dev/null)"; then + die "the checkout has no resolvable HEAD commit" +fi +[[ "${SOURCE_COMMIT}" =~ ^[0-9a-f]{40,64}$ ]] || + die "HEAD is not a supported Git object ID" +[[ -z "$(git -C "${REPOSITORY_ROOT}" status --porcelain=v1 --untracked-files=all)" ]] || + die "source checkout is not clean; review and commit it before bootstrapping or publishing images" + +for cli_args_name in TF_CLI_ARGS TF_CLI_ARGS_plan TF_CLI_ARGS_apply; do + cli_args_value="${!cli_args_name-}" + for forbidden_arg in -target -destroy -refresh-only -replace; do + if [[ "${cli_args_value}" == *"${forbidden_arg}"* ]]; then + die "${cli_args_name} must not inject ${forbidden_arg} into bootstrap" + fi + done +done + +CONFIG_FILE="${PORTSCANNER_CONFIG_FILE:-${DEPLOYMENT_ROOT}/environment.auto.tfvars.json}" +if [[ "${CONFIG_FILE}" != /* ]]; then + CONFIG_FILE="$(pwd -P)/${CONFIG_FILE}" +fi +[[ -f "${CONFIG_FILE}" ]] || die "environment configuration not found: ${CONFIG_FILE}" +[[ ! -L "${CONFIG_FILE}" ]] || die "environment configuration must not be a symbolic link" + +if ! CONFIG_SUMMARY="$(python3 - "${CONFIG_FILE}" <<'PY' +import ipaddress +import json +import pathlib +import re +import sys + +path = pathlib.Path(sys.argv[1]) +try: + document = json.loads(path.read_text(encoding="utf-8")) +except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise SystemExit(f"invalid environment JSON: {exc}") from exc + +def exact_object(value, expected, label): + if not isinstance(value, dict): + raise SystemExit(f"{label} must be an object") + actual = set(value) + expected_set = set(expected) + if actual != expected_set: + missing = sorted(expected_set - actual) + extra = sorted(actual - expected_set) + raise SystemExit(f"{label} keys are invalid (missing={missing}, extra={extra})") + +def integer(value, label, minimum, maximum): + if isinstance(value, bool) or not isinstance(value, int): + raise SystemExit(f"{label} must be an integer") + if not minimum <= value <= maximum: + raise SystemExit(f"{label} must be between {minimum} and {maximum}") + +def boolean(value, label): + if not isinstance(value, bool): + raise SystemExit(f"{label} must be true or false") + +def string(value, label): + if not isinstance(value, str): + raise SystemExit(f"{label} must be a string") + +def string_list(value, label): + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise SystemExit(f"{label} must be an array of strings") + +def ipv4_network(value, label, exact_prefix=None): + string(value, label) + try: + network = ipaddress.ip_network(value, strict=True) + except ValueError as exc: + raise SystemExit(f"{label} must be a canonical IP network: {exc}") from exc + if network.version != 4: + raise SystemExit(f"{label} must be IPv4") + if exact_prefix is not None and network.prefixlen != exact_prefix: + raise SystemExit(f"{label} must use /{exact_prefix}") + return network + +exact_object(document, ["environment"], "configuration") +environment = document["environment"] +exact_object( + environment, + [ + "aws", + "integrations", + "managed_canary", + "name", + "network", + "retention", + "runner", + "scanner", + ], + "environment", +) + +name = environment["name"] +string(name, "environment.name") +if re.fullmatch(r"[a-z][a-z0-9-]{1,19}", name) is None: + raise SystemExit("environment.name must be 2-20 lowercase alphanumeric or hyphen characters") + +aws = environment["aws"] +exact_object(aws, ["account_id", "region"], "environment.aws") +account_id = aws["account_id"] +region = aws["region"] +string(account_id, "environment.aws.account_id") +string(region, "environment.aws.region") +if re.fullmatch(r"[0-9]{12}", account_id) is None: + raise SystemExit("environment.aws.account_id must be a 12-digit AWS account ID") +if re.fullmatch(r"[a-z]{2}(?:-[a-z0-9]+)+-[0-9]+", region) is None: + raise SystemExit("environment.aws.region must be an AWS Region name") + +network = environment["network"] +exact_object( + network, + ["availability_zones", "az_count", "nat_gateway_mode", "vpc_cidr"], + "environment.network", +) +vpc = ipv4_network(network["vpc_cidr"], "environment.network.vpc_cidr") +if not 16 <= vpc.prefixlen <= 24: + raise SystemExit("environment.network.vpc_cidr must use /16 through /24") +string_list(network["availability_zones"], "environment.network.availability_zones") +integer(network["az_count"], "environment.network.az_count", 2, 3) +if network["nat_gateway_mode"] not in {"single", "one_per_az"}: + raise SystemExit("environment.network.nat_gateway_mode must be single or one_per_az") + +runner = environment["runner"] +exact_object( + runner, + ["eks_installer_principal_arn", "restricted_public_cidr"], + "environment.runner", +) +principal = runner["eks_installer_principal_arn"] +string(principal, "environment.runner.eks_installer_principal_arn") +if re.fullmatch( + rf"arn:[^:]+:iam::{re.escape(account_id)}:(?:role|user)/.+", + principal, +) is None: + raise SystemExit("EKS installer principal must be a same-account IAM role or user ARN") +runner_network = ipv4_network( + runner["restricted_public_cidr"], + "environment.runner.restricted_public_cidr", + 32, +) +if runner_network == ipaddress.ip_network("0.0.0.0/32"): + raise SystemExit("environment.runner.restricted_public_cidr must not be 0.0.0.0/32") + +retention = environment["retention"] +exact_object( + retention, + [ + "bucket_expiration_days", + "database_backup_days", + "destroy_data_on_teardown", + "disposable", + "ecr_untagged_image_days", + "lambda_log_days", + ], + "environment.retention", +) +boolean(retention["disposable"], "environment.retention.disposable") +boolean( + retention["destroy_data_on_teardown"], + "environment.retention.destroy_data_on_teardown", +) +if retention["destroy_data_on_teardown"] and not retention["disposable"]: + raise SystemExit("destroy_data_on_teardown requires disposable=true") +integer(retention["database_backup_days"], "environment.retention.database_backup_days", 1, 35) +integer(retention["lambda_log_days"], "environment.retention.lambda_log_days", 1, 3653) +integer( + retention["ecr_untagged_image_days"], + "environment.retention.ecr_untagged_image_days", + 1, + 36500, +) +bucket_days = retention["bucket_expiration_days"] +exact_object( + bucket_days, + ["cloudtrail", "events", "findings", "results"], + "environment.retention.bucket_expiration_days", +) +for bucket_name, days in bucket_days.items(): + integer(days, f"environment.retention.bucket_expiration_days.{bucket_name}", 1, 36500) + +scanner = environment["scanner"] +exact_object( + scanner, + [ + "architecture", + "max_concurrent_pods", + "max_jobs", + "max_rate", + "min_rate", + "node_instance_type", + "operator_max_concurrent_reconciles", + ], + "environment.scanner", +) +integer( + scanner["operator_max_concurrent_reconciles"], + "environment.scanner.operator_max_concurrent_reconciles", + 1, + 32, +) +integer(scanner["max_concurrent_pods"], "environment.scanner.max_concurrent_pods", 1, 100) +integer(scanner["max_jobs"], "environment.scanner.max_jobs", 1, 1000) +integer(scanner["min_rate"], "environment.scanner.min_rate", 1, 2000) +integer(scanner["max_rate"], "environment.scanner.max_rate", 1, 5000) +if scanner["min_rate"] > scanner["max_rate"]: + raise SystemExit("environment.scanner.min_rate must not exceed max_rate") +architecture = scanner["architecture"] +node_instance_type = scanner["node_instance_type"] +string(architecture, "environment.scanner.architecture") +string(node_instance_type, "environment.scanner.node_instance_type") +if architecture == "arm64": + expected_node_prefix = "t4g." + node_ami_type = "AL2023_ARM_64_STANDARD" +elif architecture == "x86_64": + expected_node_prefix = "t3." + node_ami_type = "AL2023_x86_64_STANDARD" +else: + raise SystemExit("environment.scanner.architecture must be arm64 or x86_64") +if not node_instance_type.startswith(expected_node_prefix): + raise SystemExit("node_instance_type does not match the selected architecture") + +managed_canary = environment["managed_canary"] +exact_object( + managed_canary, + [ + "enabled", + "expected_finding_severity", + "instance_type", + "listen_port", + "vpc_cidr", + ], + "environment.managed_canary", +) +boolean(managed_canary["enabled"], "environment.managed_canary.enabled") +ipv4_network( + managed_canary["vpc_cidr"], + "environment.managed_canary.vpc_cidr", + 28, +) +string(managed_canary["instance_type"], "environment.managed_canary.instance_type") +if not managed_canary["instance_type"].startswith("t4g."): + raise SystemExit("managed_canary.instance_type must be an ARM t4g instance") +integer(managed_canary["listen_port"], "environment.managed_canary.listen_port", 18080, 18080) +if managed_canary["expected_finding_severity"] != "low": + raise SystemExit("managed_canary.expected_finding_severity must be low") + +integrations = environment["integrations"] +exact_object( + integrations, + [ + "allowed_target_cidrs", + "allowed_eni_interface_types", + "authorized_account_ids", + "cloudtrail_mode", + "config_mode", + "denied_target_cidrs", + "existing_cloudtrail_arn", + "existing_config_aggregator_name", + "finding_export_enabled", + "recurring_inventory_enabled", + "required_target_tag_key", + "required_target_tag_value", + "signal_hints_enabled", + "snapshot_regions", + ], + "environment.integrations", +) +for key in ( + "finding_export_enabled", + "recurring_inventory_enabled", + "signal_hints_enabled", +): + boolean(integrations[key], f"environment.integrations.{key}") +for key in ("config_mode", "cloudtrail_mode"): + if integrations[key] not in {"create", "existing", "disabled"}: + raise SystemExit(f"environment.integrations.{key} is invalid") +for key in ("existing_config_aggregator_name", "existing_cloudtrail_arn"): + string(integrations[key], f"environment.integrations.{key}") +if integrations["config_mode"] == "existing" and not integrations["existing_config_aggregator_name"]: + raise SystemExit("existing config mode requires existing_config_aggregator_name") +if integrations["cloudtrail_mode"] == "existing" and not integrations["existing_cloudtrail_arn"]: + raise SystemExit("existing cloudtrail mode requires existing_cloudtrail_arn") +string_list(integrations["snapshot_regions"], "environment.integrations.snapshot_regions") +for snapshot_region in integrations["snapshot_regions"]: + if re.fullmatch(r"[a-z]{2}(?:-[a-z0-9]+)+-[0-9]+", snapshot_region) is None: + raise SystemExit("environment.integrations.snapshot_regions contains an invalid Region") +for key in ("authorized_account_ids", "allowed_target_cidrs", "denied_target_cidrs"): + string_list(integrations[key], f"environment.integrations.{key}") +string_list( + integrations["allowed_eni_interface_types"], + "environment.integrations.allowed_eni_interface_types", +) +if not integrations["allowed_eni_interface_types"] or not all( + re.fullmatch(r"[a-z0-9-]+", value) + for value in integrations["allowed_eni_interface_types"] +): + raise SystemExit("allowed_eni_interface_types must contain supported lowercase names") +for key in ("required_target_tag_key", "required_target_tag_value"): + string(integrations[key], f"environment.integrations.{key}") +if integrations["required_target_tag_key"] not in { + "application", + "environment", + "name", + "service", +}: + raise SystemExit("required_target_tag_key is outside the shared AWS contract") +if not integrations["required_target_tag_value"]: + raise SystemExit("required_target_tag_value must not be empty") +for authorized_account in integrations["authorized_account_ids"]: + if re.fullmatch(r"[0-9]{12}", authorized_account) is None: + raise SystemExit("authorized_account_ids contains an invalid AWS account ID") +for key in ("allowed_target_cidrs", "denied_target_cidrs"): + for cidr in integrations[key]: + ipv4_network(cidr, f"environment.integrations.{key}") + +print( + json.dumps( + { + "account_id": account_id, + "architecture": architecture, + "destroy_data_on_teardown": retention["destroy_data_on_teardown"], + "disposable": retention["disposable"], + "environment_name": name, + "node_ami_type": node_ami_type, + "node_instance_type": node_instance_type, + "region": region, + }, + sort_keys=True, + separators=(",", ":"), + ) +) +PY +)"; then + die "environment configuration validation failed" +fi + +ENVIRONMENT_NAME="$(jq -er '.environment_name' <<<"${CONFIG_SUMMARY}")" +EXPECTED_ACCOUNT_ID="$(jq -er '.account_id' <<<"${CONFIG_SUMMARY}")" +EXPECTED_REGION="$(jq -er '.region' <<<"${CONFIG_SUMMARY}")" +ARCHITECTURE="$(jq -er '.architecture' <<<"${CONFIG_SUMMARY}")" +EXPECTED_NODE_AMI_TYPE="$(jq -er '.node_ami_type' <<<"${CONFIG_SUMMARY}")" +EXPECTED_NODE_INSTANCE_TYPE="$(jq -er '.node_instance_type' <<<"${CONFIG_SUMMARY}")" + +for region_variable in AWS_REGION AWS_DEFAULT_REGION; do + configured_region="${!region_variable-}" + if [[ -n "${configured_region}" && "${configured_region}" != "${EXPECTED_REGION}" ]]; then + die "${region_variable} mismatch: configuration requires ${EXPECTED_REGION}, got ${configured_region}" + fi +done + +if ! AWS_IDENTITY_JSON="$( + aws sts get-caller-identity \ + --region "${EXPECTED_REGION}" \ + --output json +)"; then + die "AWS identity is unavailable in configured Region ${EXPECTED_REGION}" +fi +ACTUAL_ACCOUNT_ID="$(jq -er '.Account | select(test("^[0-9]{12}$"))' <<<"${AWS_IDENTITY_JSON}")" || + die "AWS identity response has no valid account ID" +AWS_IDENTITY_ARN="$(jq -er '.Arn | select(type == "string" and length > 0)' <<<"${AWS_IDENTITY_JSON}")" || + die "AWS identity response has no ARN" +[[ "${ACTUAL_ACCOUNT_ID}" == "${EXPECTED_ACCOUNT_ID}" ]] || + die "AWS identity mismatch: expected account ${EXPECTED_ACCOUNT_ID}, got ${ACTUAL_ACCOUNT_ID}" + +export AWS_REGION="${EXPECTED_REGION}" +export AWS_DEFAULT_REGION="${EXPECTED_REGION}" +export PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID="${EXPECTED_ACCOUNT_ID}" +export PORTSCANNER_EXPECTED_AWS_REGION="${EXPECTED_REGION}" +export PORTSCANNER_CONFIG_FILE="${CONFIG_FILE}" + +ARTIFACT_ROOT="${PORTSCANNER_ARTIFACT_ROOT:-${REPOSITORY_ROOT}/.portscanner}" +if [[ "${ARTIFACT_ROOT}" != /* ]]; then + ARTIFACT_ROOT="$(pwd -P)/${ARTIFACT_ROOT}" +fi +[[ ! -L "${ARTIFACT_ROOT}" ]] || die "artifact root must not be a symbolic link" +ARTIFACT_DIR="${ARTIFACT_ROOT}/${ENVIRONMENT_NAME}/${EXPECTED_ACCOUNT_ID}/${EXPECTED_REGION}" +BACKEND_KEY="environments/${ENVIRONMENT_NAME}/${EXPECTED_ACCOUNT_ID}/${EXPECTED_REGION}/deployment.tfstate" +if [[ ! "${BACKEND_KEY}" =~ ^environments/[a-z][a-z0-9-]{1,19}/[0-9]{12}/[a-z]{2}(-[a-z0-9]+)+-[0-9]+/deployment\.tfstate$ ]] || + [[ "${BACKEND_KEY}" == /* || "${BACKEND_KEY}" == *".."* || "${BACKEND_KEY}" == *"//"* ]]; then + die "derived backend key is unsafe: ${BACKEND_KEY}" +fi + +umask 077 +mkdir -p "${ARTIFACT_DIR}" +[[ ! -L "${ARTIFACT_DIR}" ]] || die "artifact directory must not be a symbolic link" + +BOOTSTRAP_STATE="${ARTIFACT_DIR}/state-bootstrap.tfstate" +BOOTSTRAP_BACKUP="${BOOTSTRAP_STATE}.backup" +BACKEND_CONFIG="${ARTIFACT_DIR}/backend.hcl" +IMAGE_INPUTS="${ARTIFACT_DIR}/images.tfvars" +STATE_DATA_DIR="${ARTIFACT_DIR}/state-bootstrap-data" +STATE_PREFIX="${ENVIRONMENT_NAME}-state" +EXPECTED_LOCK_TABLE="${STATE_PREFIX}-terraform-locks" +EXPECTED_BUCKET_PREFIX="${STATE_PREFIX}-tfstate-" + +if [[ ! -f "${BOOTSTRAP_STATE}" && -e "${BACKEND_CONFIG}" ]]; then + die "backend configuration exists without its bootstrap state; refusing a possible collision" +fi +if [[ ! -f "${BOOTSTRAP_STATE}" && -e "${BOOTSTRAP_BACKUP}" ]]; then + die "bootstrap state backup exists without primary state; restore or review it before continuing" +fi +if [[ ! -f "${BOOTSTRAP_STATE}" ]]; then + if ! EXISTING_LOCK_TABLES="$( + aws dynamodb list-tables \ + --region "${EXPECTED_REGION}" \ + --query TableNames \ + --output json + )"; then + die "could not check for an existing state lock table" + fi + if ! jq -e 'type == "array" and all(.[]; type == "string")' \ + <<<"${EXISTING_LOCK_TABLES}" >/dev/null; then + die "DynamoDB returned a malformed state lock-table inventory" + fi + if jq -e --arg table "${EXPECTED_LOCK_TABLE}" 'index($table) != null' \ + <<<"${EXISTING_LOCK_TABLES}" >/dev/null; then + die "bootstrap state collision: lock table ${EXPECTED_LOCK_TABLE} already exists without matching local bootstrap state" + fi +fi + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-bootstrap.XXXXXX")" +BACKEND_CANDIDATE="" +IMAGE_CANDIDATE="" +cleanup() { + [[ -z "${BACKEND_CANDIDATE}" ]] || rm -f -- "${BACKEND_CANDIDATE}" + [[ -z "${IMAGE_CANDIDATE}" ]] || rm -f -- "${IMAGE_CANDIDATE}" + rm -rf -- "${WORK_DIR}" +} +handle_signal() { + exit 130 +} +trap cleanup EXIT +trap handle_signal HUP INT TERM + +mkdir -p "${STATE_DATA_DIR}" +TF_DATA_DIR="${STATE_DATA_DIR}" terraform "-chdir=${STATE_BOOTSTRAP_ROOT}" init \ + -backend=false \ + -input=false + +read_bootstrap_configuration() { + TF_DATA_DIR="${STATE_DATA_DIR}" terraform "-chdir=${STATE_BOOTSTRAP_ROOT}" output \ + -state="${BOOTSTRAP_STATE}" \ + -json backend_configuration +} + +validate_bootstrap_configuration() { + local configuration_json="$1" + local bucket + local region + local lock_table + local table_arn + + if ! jq -e ' + type == "object" and + keys == ["bucket", "dynamodb_table", "encrypt", "region"] and + (.bucket | type == "string") and + (.dynamodb_table | type == "string") and + (.region | type == "string") and + .encrypt == true + ' <<<"${configuration_json}" >/dev/null; then + die "state-bootstrap output is malformed" + fi + bucket="$(jq -er '.bucket | select(test("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"))' <<<"${configuration_json}")" || + die "state-bootstrap bucket output is unsafe" + region="$(jq -er '.region' <<<"${configuration_json}")" + lock_table="$(jq -er '.dynamodb_table | select(test("^[A-Za-z0-9_.-]{3,255}$"))' <<<"${configuration_json}")" || + die "state-bootstrap lock-table output is unsafe" + [[ "${bucket}" == "${EXPECTED_BUCKET_PREFIX}"* ]] || + die "bootstrap state collision: bucket ${bucket} does not match ${EXPECTED_BUCKET_PREFIX}*" + [[ "${region}" == "${EXPECTED_REGION}" ]] || + die "bootstrap state collision: expected Region ${EXPECTED_REGION}, got ${region}" + [[ "${lock_table}" == "${EXPECTED_LOCK_TABLE}" ]] || + die "bootstrap state collision: expected lock table ${EXPECTED_LOCK_TABLE}, got ${lock_table}" + + aws s3api head-bucket \ + --bucket "${bucket}" \ + --expected-bucket-owner "${EXPECTED_ACCOUNT_ID}" \ + --region "${EXPECTED_REGION}" >/dev/null || + die "state bucket is unavailable or is not owned by the configured account" + table_arn="$( + aws dynamodb describe-table \ + --table-name "${lock_table}" \ + --region "${EXPECTED_REGION}" \ + --query Table.TableArn \ + --output text + )" || die "state lock table is unavailable" + [[ "${table_arn}" =~ ^arn:[^:]+:dynamodb:${EXPECTED_REGION}:${EXPECTED_ACCOUNT_ID}:table/${EXPECTED_LOCK_TABLE}$ ]] || + die "state lock table identity does not match the configured account and Region" +} + +if [[ -f "${BOOTSTRAP_STATE}" ]]; then + EXISTING_BACKEND_JSON="$(read_bootstrap_configuration)" || + die "existing bootstrap state is unreadable; refusing to overwrite it" + validate_bootstrap_configuration "${EXISTING_BACKEND_JSON}" + log "verified resumable state bootstrap for ${ENVIRONMENT_NAME}" +fi + +BOOTSTRAP_PLAN="${WORK_DIR}/state-bootstrap.tfplan" +TF_DATA_DIR="${STATE_DATA_DIR}" terraform "-chdir=${STATE_BOOTSTRAP_ROOT}" plan \ + -input=false \ + -state="${BOOTSTRAP_STATE}" \ + -out="${BOOTSTRAP_PLAN}" \ + "-var=aws_region=${EXPECTED_REGION}" \ + "-var=expected_deployment_account_id=${EXPECTED_ACCOUNT_ID}" \ + "-var=name_prefix=${STATE_PREFIX}" +TF_DATA_DIR="${STATE_DATA_DIR}" terraform "-chdir=${STATE_BOOTSTRAP_ROOT}" show "${BOOTSTRAP_PLAN}" +confirm_apply "state-bootstrap" +TF_DATA_DIR="${STATE_DATA_DIR}" terraform "-chdir=${STATE_BOOTSTRAP_ROOT}" apply \ + -input=false \ + -state="${BOOTSTRAP_STATE}" \ + -backup="${BOOTSTRAP_BACKUP}" \ + "${BOOTSTRAP_PLAN}" +chmod 600 "${BOOTSTRAP_STATE}" +[[ ! -f "${BOOTSTRAP_BACKUP}" ]] || chmod 600 "${BOOTSTRAP_BACKUP}" +verify_mode_0600 "${BOOTSTRAP_STATE}" + +BACKEND_JSON="$(read_bootstrap_configuration)" || + die "state-bootstrap did not produce backend_configuration" +validate_bootstrap_configuration "${BACKEND_JSON}" +STATE_BUCKET="$(jq -er .bucket <<<"${BACKEND_JSON}")" +LOCK_TABLE="$(jq -er .dynamodb_table <<<"${BACKEND_JSON}")" + +BACKEND_CANDIDATE="$(mktemp "${ARTIFACT_DIR}/.backend.hcl.XXXXXX")" +cat >"${BACKEND_CANDIDATE}" </dev/null)"; then + if ! jq -e ' + type == "object" and + keys == [ + "automatic_inventory_enabled", + "canary_mode", + "dispatch_enabled", + "finding_export_enabled", + "managed_canary_enabled", + "migration_run", + "operator_installed", + "periodic_coverage_enabled", + "periodic_snapshots_enabled", + "processor_reconciliation_enabled", + "runtime_created", + "signal_hints_enabled" + ] and + all(.[]; type == "boolean") + ' <<<"${CURRENT_STATE}" >/dev/null; then + die "canonical deployment_state output is malformed" + fi +fi +if [[ -n "${CURRENT_STATE}" ]] && jq -e ' + .runtime_created or .migration_run or .operator_installed or + .dispatch_enabled or .automatic_inventory_enabled or .canary_mode or + .periodic_snapshots_enabled or .periodic_coverage_enabled or + .signal_hints_enabled or .processor_reconciliation_enabled +' <<<"${CURRENT_STATE}" >/dev/null; then + log "canonical foundation already exists at a later stage; no rollback attempted" +else + "${DEPLOY_SCRIPT}" foundation +fi + +BUILD_IMAGES_SCRIPT="${PORTSCANNER_BUILD_IMAGES_SCRIPT:-${SCRIPT_DIR}/build-images.sh}" +[[ -x "${BUILD_IMAGES_SCRIPT}" ]] || die "image build helper is not executable: ${BUILD_IMAGES_SCRIPT}" +IMAGE_CANDIDATE="$(mktemp "${ARTIFACT_DIR}/.images.tfvars.XXXXXX")" +"${BUILD_IMAGES_SCRIPT}" "${DEPLOYMENT_ROOT}" "${ARCHITECTURE}" >"${IMAGE_CANDIDATE}" + +python3 - \ + "${IMAGE_CANDIDATE}" \ + "${ARCHITECTURE}" \ + "${EXPECTED_NODE_AMI_TYPE}" \ + "${EXPECTED_NODE_INSTANCE_TYPE}" <<'PY' +import json +import pathlib +import re +import sys + +path = pathlib.Path(sys.argv[1]) +expected_architecture = sys.argv[2] +expected_node_ami = sys.argv[3] +expected_node_type = sys.argv[4] +text = path.read_text(encoding="utf-8") +match = re.fullmatch( + r'image_digests = \{\n(?P(?: [a-z]+ +=' + r' "sha256:[0-9a-f]{64}"\n)+)\}\n\n' + r'migration_checksum = "(?P[0-9a-f]{64})"\n' + r'lambda_architecture = "(?Parm64|x86_64)"\n' + r'node_ami_type = "(?P[A-Za-z0-9_]+)"\n' + r'node_instance_types = (?P\[[^\n]+\])\n', + text, +) +if match is None: + raise SystemExit("generated image input has an unexpected shape") +components = re.findall(r"^ ([a-z]+) +=", match.group("digests"), re.MULTILINE) +if components != [ + "inventory", + "generator", + "parser", + "processor", + "migrator", + "operator", + "scanner", +]: + raise SystemExit("generated image input does not contain the seven ordered components") +try: + nodes = json.loads(match.group("nodes")) +except json.JSONDecodeError as exc: + raise SystemExit("generated node_instance_types is invalid JSON") from exc +if match.group("architecture") != expected_architecture: + raise SystemExit("generated image architecture does not match environment configuration") +if match.group("ami") != expected_node_ami: + raise SystemExit("generated node AMI does not match environment configuration") +if nodes != [expected_node_type]: + raise SystemExit("generated node types do not match environment configuration") +PY + +chmod 600 "${IMAGE_CANDIDATE}" +mv "${IMAGE_CANDIDATE}" "${IMAGE_INPUTS}" +verify_mode_0600 "${IMAGE_INPUTS}" + +log "Bootstrap complete for ${ENVIRONMENT_NAME} using ${AWS_IDENTITY_ARN} in ${EXPECTED_REGION}." +log "Backend and generated image inputs are private artifacts under ${ARTIFACT_DIR}." +log "Run ${SCRIPT_DIR}/deploy.sh ready to create the paused runtime." diff --git a/terraform/aws/scripts/build-images.sh b/terraform/aws/scripts/build-images.sh index 01b6506..fd8e8a8 100755 --- a/terraform/aws/scripts/build-images.sh +++ b/terraform/aws/scripts/build-images.sh @@ -78,7 +78,7 @@ ROOT="$(CDPATH= cd -- "${ROOT_CANDIDATE}" && pwd -P)" case "${ROOT}" in "${AWS_DIR}/application" | \ - "${AWS_DIR}/examples/created-vpc" | \ + "${AWS_DIR}/deployment" | \ "${AWS_DIR}/examples/existing-vpc" | \ "${AWS_DIR}/examples/multi-account-central") ;; @@ -86,7 +86,7 @@ case "${ROOT}" in die "member-account is not a central image deployment root" ;; *) - die "Terraform root must be application, created-vpc, existing-vpc, or multi-account-central" + die "Terraform root must be application, deployment, existing-vpc, or multi-account-central" ;; esac [[ -f "${ROOT}/main.tf" ]] || die "${ROOT} is not a Terraform root with main.tf" @@ -199,18 +199,50 @@ if ! jq -e --argjson expected "${EXPECTED_COMPONENTS_JSON}" \ fi if ! jq -e ' type == "object" - and keys == ["automatic_inventory_enabled", "canary_mode", "dispatch_enabled", "migration_run", "operator_installed", "runtime_created"] + and keys == [ + "automatic_inventory_enabled", + "canary_mode", + "dispatch_enabled", + "finding_export_enabled", + "managed_canary_enabled", + "migration_run", + "operator_installed", + "periodic_coverage_enabled", + "periodic_snapshots_enabled", + "processor_reconciliation_enabled", + "runtime_created", + "signal_hints_enabled" + ] and all(.[]; type == "boolean") and ((.migration_run == false) or .runtime_created) and ((.operator_installed == false) or .migration_run) and ((.dispatch_enabled == false) or .operator_installed) and ((.automatic_inventory_enabled == false) or .dispatch_enabled) - and ((.canary_mode == false) or (.operator_installed and (.automatic_inventory_enabled == false))) + and ((.periodic_snapshots_enabled == false) or .dispatch_enabled) + and ((.periodic_coverage_enabled == false) or .dispatch_enabled) + and ((.signal_hints_enabled == false) or .dispatch_enabled) + and ((.processor_reconciliation_enabled == false) or (.runtime_created and .migration_run)) + and ( + (.canary_mode == false) or ( + .operator_installed + and (.automatic_inventory_enabled == false) + and (.periodic_snapshots_enabled == false) + and (.periodic_coverage_enabled == false) + and (.signal_hints_enabled == false) + and (.processor_reconciliation_enabled == false) + ) + ) ' <<<"${DEPLOYMENT_STATE_JSON}" >/dev/null; then die "deployment_state is malformed or violates staged deployment ordering" fi DEPLOYMENT_STAGE="$(jq -r ' - if .automatic_inventory_enabled then "active" + if ( + .automatic_inventory_enabled or + .periodic_snapshots_enabled or + .periodic_coverage_enabled or + .signal_hints_enabled or + .processor_reconciliation_enabled + ) then "active" elif (.canary_mode and .dispatch_enabled) then "canary" elif .canary_mode then "canary-paused" elif .dispatch_enabled then "dispatch-only" diff --git a/terraform/aws/scripts/deploy.sh b/terraform/aws/scripts/deploy.sh index 17dd3ff..70c36c8 100755 --- a/terraform/aws/scripts/deploy.sh +++ b/terraform/aws/scripts/deploy.sh @@ -1,329 +1,849 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" -AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd)" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" +AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd -P)" +REPOSITORY_ROOT="$(CDPATH= cd -- "${AWS_DIR}/../.." && pwd -P)" +CANONICAL_ROOT="${AWS_DIR}/deployment" usage() { - echo "usage: $0 [variables.tfvars ...]" >&2 + cat >&2 < + $0 [variables.tfvars ...] +EOF exit 2 } -[[ $# -ge 2 ]] || usage -command -v terraform >/dev/null 2>&1 || { - echo "terraform is required" >&2 +die() { + echo "error: $*" >&2 exit 1 } -command -v aws >/dev/null 2>&1 || { - echo "AWS CLI is required for deployment identity checks" >&2 - exit 1 + +log() { + echo "$*" >&2 } -EXPECTED_ACCOUNT_ID="${PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID:-}" -EXPECTED_REGION="${PORTSCANNER_EXPECTED_AWS_REGION:-}" -[[ "${EXPECTED_ACCOUNT_ID}" =~ ^[0-9]{12}$ ]] || { - echo "PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID must be a 12-digit AWS account ID" >&2 - exit 1 +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" } -[[ "${EXPECTED_REGION}" =~ ^[a-z]{2}(-[a-z0-9]+)+-[0-9]+$ ]] || { - echo "PORTSCANNER_EXPECTED_AWS_REGION must be an AWS Region name" >&2 - exit 1 + +is_stage() { + case "$1" in + foundation | runtime | migrate | canary | activate | pause | pause-canary | ready | evaluate | retire-canary | destroy) + return 0 + ;; + *) + return 1 + ;; + esac } -ACTUAL_ACCOUNT_ID="$( - aws sts get-caller-identity \ - --region "${EXPECTED_REGION}" \ - --query Account \ - --output text -)" -[[ "${ACTUAL_ACCOUNT_ID}" == "${EXPECTED_ACCOUNT_ID}" ]] || { - echo "AWS identity mismatch: expected account ${EXPECTED_ACCOUNT_ID}, got ${ACTUAL_ACCOUNT_ID}" >&2 - exit 1 +file_mode() { + local path="$1" + local mode + if mode="$(stat -f '%Lp' "${path}" 2>/dev/null)"; then + printf '%s' "${mode}" + else + stat -c '%a' "${path}" + fi } -ROOT_ARG="$1" -STAGE="$2" +require_private_generated_file() { + local path="$1" + local label="$2" + [[ -f "${path}" && ! -L "${path}" ]] || + die "${label} must be a regular generated file: ${path}" + [[ "$(file_mode "${path}")" == "600" ]] || + die "${label} must have mode 0600: ${path}" +} -if [[ "${ROOT_ARG}" = /* ]]; then - ROOT="${ROOT_ARG}" +require_command terraform +require_command aws + +[[ $# -ge 1 ]] || usage +CANONICAL_MODE=false +CONFIG_FILE="" +ENVIRONMENT_NAME="" +DISPOSABLE=false +DESTROY_DATA_ON_TEARDOWN=false + +if is_stage "$1"; then + [[ $# -eq 1 ]] || usage + CANONICAL_MODE=true + ROOT="${CANONICAL_ROOT}" + STAGE="$1" else - ROOT="$(pwd)/${ROOT_ARG}" + [[ $# -ge 2 ]] || usage + ROOT_ARG="$1" + STAGE="$2" + is_stage "${STAGE}" || usage + case "${STAGE}" in + ready | evaluate | retire-canary | destroy) + die "${STAGE} is available only with the canonical no-root invocation" + ;; + esac + if [[ "${ROOT_ARG}" = /* ]]; then + ROOT_CANDIDATE="${ROOT_ARG}" + else + ROOT_CANDIDATE="$(pwd -P)/${ROOT_ARG}" + fi + [[ -d "${ROOT_CANDIDATE}" ]] || die "Terraform root does not exist: ${ROOT_ARG}" + ROOT="$(CDPATH= cd -- "${ROOT_CANDIDATE}" && pwd -P)" fi -ROOT="$(CDPATH= cd -- "${ROOT}" && pwd)" case "${ROOT}/" in "${AWS_DIR}/"*) ;; - *) - echo "terraform root must remain under ${AWS_DIR}" >&2 - exit 1 - ;; + *) die "terraform root must remain under ${AWS_DIR}" ;; esac +[[ -f "${ROOT}/main.tf" ]] || die "${ROOT} is not a Terraform root with main.tf" -[[ -f "${ROOT}/main.tf" ]] || { - echo "${ROOT} is not a Terraform root with main.tf" >&2 - exit 1 -} +PLAN_ARGS=() +if [[ "${CANONICAL_MODE}" == "true" ]]; then + require_command jq + CONFIG_FILE="${PORTSCANNER_CONFIG_FILE:-${CANONICAL_ROOT}/environment.auto.tfvars.json}" + if [[ "${CONFIG_FILE}" != /* ]]; then + CONFIG_FILE="$(pwd -P)/${CONFIG_FILE}" + fi + [[ -f "${CONFIG_FILE}" && ! -L "${CONFIG_FILE}" ]] || + die "canonical environment configuration not found: ${CONFIG_FILE}; copy environment.auto.tfvars.json.example" + + if ! CONFIG_SUMMARY="$(jq -ce ' + . as $document + | .environment as $environment + | if ( + ($document | type == "object" and keys == ["environment"]) and + ($environment | type == "object" and keys == [ + "aws", + "integrations", + "managed_canary", + "name", + "network", + "retention", + "runner", + "scanner" + ]) and + ($environment.name | type == "string" and test("^[a-z][a-z0-9-]{1,19}$")) and + ($environment.aws | type == "object" and keys == ["account_id", "region"]) and + ($environment.aws.account_id | type == "string" and test("^[0-9]{12}$")) and + ($environment.aws.region | type == "string" and test("^[a-z]{2}(-[a-z0-9]+)+-[0-9]+$")) and + ($environment.network | type == "object" and keys == ["availability_zones", "az_count", "nat_gateway_mode", "vpc_cidr"]) and + ($environment.runner | type == "object" and keys == ["eks_installer_principal_arn", "restricted_public_cidr"]) and + ($environment.runner.eks_installer_principal_arn | type == "string") and + ($environment.runner.restricted_public_cidr | type == "string" and test("/32$") and . != "0.0.0.0/0") and + ($environment.retention | type == "object" and keys == [ + "bucket_expiration_days", + "database_backup_days", + "destroy_data_on_teardown", + "disposable", + "ecr_untagged_image_days", + "lambda_log_days" + ]) and + ($environment.retention.disposable | type == "boolean") and + ($environment.retention.destroy_data_on_teardown | type == "boolean") and + (($environment.retention.destroy_data_on_teardown | not) or $environment.retention.disposable) and + ($environment.scanner | type == "object" and keys == [ + "architecture", + "max_concurrent_pods", + "max_jobs", + "max_rate", + "min_rate", + "node_instance_type", + "operator_max_concurrent_reconciles" + ]) and + ($environment.scanner.architecture == "arm64" or $environment.scanner.architecture == "x86_64") and + ($environment.managed_canary | type == "object" and keys == [ + "enabled", + "expected_finding_severity", + "instance_type", + "listen_port", + "vpc_cidr" + ]) and + ($environment.managed_canary.enabled | type == "boolean") and + ($environment.integrations | type == "object" and keys == [ + "allowed_eni_interface_types", + "allowed_target_cidrs", + "authorized_account_ids", + "cloudtrail_mode", + "config_mode", + "denied_target_cidrs", + "existing_cloudtrail_arn", + "existing_config_aggregator_name", + "finding_export_enabled", + "recurring_inventory_enabled", + "required_target_tag_key", + "required_target_tag_value", + "signal_hints_enabled", + "snapshot_regions" + ]) + ) then + { + account_id: $environment.aws.account_id, + architecture: $environment.scanner.architecture, + destroy_data_on_teardown: $environment.retention.destroy_data_on_teardown, + disposable: $environment.retention.disposable, + environment_name: $environment.name, + region: $environment.aws.region + } + else + error("canonical environment configuration is malformed") + end + ' "${CONFIG_FILE}")"; then + die "canonical environment configuration validation failed" + fi + + ENVIRONMENT_NAME="$(jq -er .environment_name <<<"${CONFIG_SUMMARY}")" + EXPECTED_ACCOUNT_ID="$(jq -er .account_id <<<"${CONFIG_SUMMARY}")" + EXPECTED_REGION="$(jq -er .region <<<"${CONFIG_SUMMARY}")" + DISPOSABLE="$(jq -er '.disposable | tostring' <<<"${CONFIG_SUMMARY}")" + DESTROY_DATA_ON_TEARDOWN="$(jq -er '.destroy_data_on_teardown | tostring' <<<"${CONFIG_SUMMARY}")" + + if [[ -n "${PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID:-}" && + "${PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID}" != "${EXPECTED_ACCOUNT_ID}" ]]; then + die "PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID conflicts with canonical environment configuration" + fi + if [[ -n "${PORTSCANNER_EXPECTED_AWS_REGION:-}" && + "${PORTSCANNER_EXPECTED_AWS_REGION}" != "${EXPECTED_REGION}" ]]; then + die "PORTSCANNER_EXPECTED_AWS_REGION conflicts with canonical environment configuration" + fi + + ARTIFACT_ROOT="${PORTSCANNER_ARTIFACT_ROOT:-${REPOSITORY_ROOT}/.portscanner}" + if [[ "${ARTIFACT_ROOT}" != /* ]]; then + ARTIFACT_ROOT="$(pwd -P)/${ARTIFACT_ROOT}" + fi + ARTIFACT_DIR="${ARTIFACT_ROOT}/${ENVIRONMENT_NAME}/${EXPECTED_ACCOUNT_ID}/${EXPECTED_REGION}" + DERIVED_BACKEND_CONFIG="${ARTIFACT_DIR}/backend.hcl" + IMAGE_INPUTS="${PORTSCANNER_IMAGE_VARS_FILE:-${ARTIFACT_DIR}/images.tfvars}" + BACKEND_CONFIG="${TF_BACKEND_CONFIG:-${DERIVED_BACKEND_CONFIG}}" + if [[ "${BACKEND_CONFIG}" != /* ]]; then + BACKEND_CONFIG="$(pwd -P)/${BACKEND_CONFIG}" + fi + if [[ "${IMAGE_INPUTS}" != /* ]]; then + IMAGE_INPUTS="$(pwd -P)/${IMAGE_INPUTS}" + fi + + require_private_generated_file "${BACKEND_CONFIG}" "backend configuration" + BACKEND_KEY="environments/${ENVIRONMENT_NAME}/${EXPECTED_ACCOUNT_ID}/${EXPECTED_REGION}/deployment.tfstate" + [[ "${BACKEND_KEY}" != /* && "${BACKEND_KEY}" != *".."* && "${BACKEND_KEY}" != *"//"* ]] || + die "derived backend key is unsafe" + BACKEND_CONTENT="$(<"${BACKEND_CONFIG}")" + [[ "${BACKEND_CONTENT}" == *"key = \"${BACKEND_KEY}\""* ]] || + die "backend configuration key does not match the canonical environment identity" + [[ "${BACKEND_CONTENT}" == *"region = \"${EXPECTED_REGION}\""* ]] || + die "backend configuration Region does not match the canonical environment identity" + [[ "${BACKEND_CONTENT}" == *"encrypt = true"* ]] || + die "backend configuration must enable encryption" + backend_bucket_fields=0 + backend_key_fields=0 + backend_region_fields=0 + backend_lock_fields=0 + backend_encrypt_fields=0 + while IFS= read -r backend_line; do + [[ -n "${backend_line}" && "${backend_line}" == *=* ]] || + die "backend configuration has an unexpected line" + backend_field="${backend_line%%=*}" + backend_field="${backend_field//[[:space:]]/}" + case "${backend_field}" in + bucket) backend_bucket_fields=$((backend_bucket_fields + 1)) ;; + key) backend_key_fields=$((backend_key_fields + 1)) ;; + region) backend_region_fields=$((backend_region_fields + 1)) ;; + dynamodb_table) backend_lock_fields=$((backend_lock_fields + 1)) ;; + encrypt) backend_encrypt_fields=$((backend_encrypt_fields + 1)) ;; + access_key | secret_key | token | profile | shared_credentials_file) + die "backend configuration must not contain credentials or a credential profile" + ;; + *) die "backend configuration contains unsupported field ${backend_field}" ;; + esac + done <<<"${BACKEND_CONTENT}" + [[ "${backend_bucket_fields}" -eq 1 && + "${backend_key_fields}" -eq 1 && + "${backend_region_fields}" -eq 1 && + "${backend_lock_fields}" -eq 1 && + "${backend_encrypt_fields}" -eq 1 ]] || + die "backend configuration must contain each generated field exactly once" + + PLAN_ARGS+=("-var-file=${CONFIG_FILE}") +else + EXPECTED_ACCOUNT_ID="${PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID:-}" + EXPECTED_REGION="${PORTSCANNER_EXPECTED_AWS_REGION:-}" + [[ "${EXPECTED_ACCOUNT_ID}" =~ ^[0-9]{12}$ ]] || + die "PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID must be a 12-digit AWS account ID" + [[ "${EXPECTED_REGION}" =~ ^[a-z]{2}(-[a-z0-9]+)+-[0-9]+$ ]] || + die "PORTSCANNER_EXPECTED_AWS_REGION must be an AWS Region name" + + for VAR_FILE in "${@:3}"; do + if [[ "${VAR_FILE}" != /* ]]; then + VAR_FILE="${ROOT}/${VAR_FILE}" + fi + [[ -f "${VAR_FILE}" ]] || die "variable file not found: ${VAR_FILE}" + PLAN_ARGS+=("-var-file=${VAR_FILE}") + done + PLAN_ARGS+=( + "-var=aws_region=${EXPECTED_REGION}" + "-var=expected_deployment_account_id=${EXPECTED_ACCOUNT_ID}" + ) + BACKEND_CONFIG="${TF_BACKEND_CONFIG:-}" + if [[ -n "${BACKEND_CONFIG}" && "${BACKEND_CONFIG}" != /* ]]; then + BACKEND_CONFIG="$(pwd -P)/${BACKEND_CONFIG}" + fi + if [[ -n "${BACKEND_CONFIG}" && ! -f "${BACKEND_CONFIG}" ]]; then + die "backend configuration not found: ${BACKEND_CONFIG}" + fi +fi for cli_args_name in TF_CLI_ARGS TF_CLI_ARGS_plan TF_CLI_ARGS_apply; do cli_args_value="${!cli_args_name-}" for forbidden_arg in -target -destroy -refresh-only -replace; do if [[ "${cli_args_value}" == *"${forbidden_arg}"* ]]; then - echo "${cli_args_name} must not inject ${forbidden_arg} into staged deployment" >&2 - exit 1 + die "${cli_args_name} must not inject ${forbidden_arg} into staged deployment" fi done done -PLAN_ARGS=() -for VAR_FILE in "${@:3}"; do - if [[ "${VAR_FILE}" != /* ]]; then - VAR_FILE="${ROOT}/${VAR_FILE}" - fi - [[ -f "${VAR_FILE}" ]] || { - echo "variable file not found: ${VAR_FILE}" >&2 - exit 1 - } - PLAN_ARGS+=("-var-file=${VAR_FILE}") -done -PLAN_ARGS+=( - "-var=aws_region=${EXPECTED_REGION}" - "-var=expected_deployment_account_id=${EXPECTED_ACCOUNT_ID}" -) +ACTUAL_ACCOUNT_ID="$( + aws sts get-caller-identity \ + --region "${EXPECTED_REGION}" \ + --query Account \ + --output text +)" +[[ "${ACTUAL_ACCOUNT_ID}" == "${EXPECTED_ACCOUNT_ID}" ]] || + die "AWS identity mismatch: expected account ${EXPECTED_ACCOUNT_ID}, got ${ACTUAL_ACCOUNT_ID}" + +export PORTSCANNER_EXPECTED_AWS_ACCOUNT_ID="${EXPECTED_ACCOUNT_ID}" +export PORTSCANNER_EXPECTED_AWS_REGION="${EXPECTED_REGION}" INIT_ARGS=("-input=false") -if [[ -n "${TF_BACKEND_CONFIG:-}" ]]; then - BACKEND_CONFIG="${TF_BACKEND_CONFIG}" - if [[ "${BACKEND_CONFIG}" != /* ]]; then - BACKEND_CONFIG="$(pwd)/${BACKEND_CONFIG}" - fi - [[ -f "${BACKEND_CONFIG}" ]] || { - echo "backend configuration not found: ${BACKEND_CONFIG}" >&2 - exit 1 - } +if [[ -n "${BACKEND_CONFIG}" ]]; then INIT_ARGS+=("-backend-config=${BACKEND_CONFIG}") fi - -case "${STAGE}" in - foundation) - STAGE_ARGS=( - "-var=deploy_runtime=false" - "-var=run_migration=false" - "-var=install_operator=false" - "-var=enable_event_dispatch=false" - "-var=enable_automatic_inventory=false" - "-var=canary_mode=false" - ) - ;; - runtime) - STAGE_ARGS=( - "-var=deploy_runtime=true" - "-var=run_migration=false" - "-var=install_operator=false" - "-var=enable_event_dispatch=false" - "-var=enable_automatic_inventory=false" - "-var=canary_mode=false" - ) - ;; - migrate | pause) - STAGE_ARGS=( - "-var=deploy_runtime=true" - "-var=run_migration=true" - "-var=install_operator=true" - "-var=enable_event_dispatch=false" - "-var=enable_automatic_inventory=false" - "-var=canary_mode=false" - ) - ;; - pause-canary) - STAGE_ARGS=( - "-var=deploy_runtime=true" - "-var=run_migration=true" - "-var=install_operator=true" - "-var=enable_event_dispatch=false" - "-var=enable_automatic_inventory=false" - "-var=canary_mode=true" - ) - ;; - canary) - STAGE_ARGS=( - "-var=deploy_runtime=true" - "-var=run_migration=true" - "-var=install_operator=true" - "-var=enable_event_dispatch=true" - "-var=enable_automatic_inventory=false" - "-var=canary_mode=true" - ) - ;; - activate) - STAGE_ARGS=( - "-var=deploy_runtime=true" - "-var=run_migration=true" - "-var=install_operator=true" - "-var=enable_event_dispatch=true" - "-var=enable_automatic_inventory=true" - "-var=canary_mode=false" - ) - ;; - *) - usage - ;; -esac +terraform "-chdir=${ROOT}" init "${INIT_ARGS[@]}" WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portable-portscanner.XXXXXX")" -trap 'rm -rf "${WORK_DIR}"' EXIT -PLAN_FILE="${WORK_DIR}/${STAGE}.tfplan" +PLAN_COUNTER=0 +IMAGE_ARGS_ADDED=false +EVALUATION_NEEDS_PAUSE=false +EVALUATION_PAUSED=false -terraform "-chdir=${ROOT}" init "${INIT_ARGS[@]}" +cleanup() { + local status=$? + local pause_status=0 + trap - EXIT HUP INT TERM + if [[ "${EVALUATION_NEEDS_PAUSE}" == "true" && "${EVALUATION_PAUSED}" != "true" ]]; then + log "Evaluation did not complete cleanly; applying pause-canary before exit." + set +e + apply_stage pause-canary + pause_status=$? + set -e + if [[ "${pause_status}" -ne 0 ]]; then + log "ERROR: automatic pause-canary failed; run deploy.sh pause-canary immediately." + status="${pause_status}" + fi + fi + rm -rf -- "${WORK_DIR}" + exit "${status}" +} +handle_signal() { + exit 130 +} +trap cleanup EXIT +trap handle_signal HUP INT TERM + +ensure_image_inputs() { + if [[ "${CANONICAL_MODE}" != "true" || "${IMAGE_ARGS_ADDED}" == "true" ]]; then + return 0 + fi + require_private_generated_file "${IMAGE_INPUTS}" "generated image input" + PLAN_ARGS+=("-var-file=${IMAGE_INPUTS}") + IMAGE_ARGS_ADDED=true +} + +set_stage_args() { + case "$1" in + foundation) + STAGE_ARGS=( + "-var=deploy_runtime=false" + "-var=run_migration=false" + "-var=install_operator=false" + "-var=enable_event_dispatch=false" + "-var=enable_automatic_inventory=false" + "-var=canary_mode=false" + ) + ;; + runtime) + STAGE_ARGS=( + "-var=deploy_runtime=true" + "-var=run_migration=false" + "-var=install_operator=false" + "-var=enable_event_dispatch=false" + "-var=enable_automatic_inventory=false" + "-var=canary_mode=false" + ) + ;; + migrate | pause) + STAGE_ARGS=( + "-var=deploy_runtime=true" + "-var=run_migration=true" + "-var=install_operator=true" + "-var=enable_event_dispatch=false" + "-var=enable_automatic_inventory=false" + "-var=canary_mode=false" + ) + ;; + pause-canary) + STAGE_ARGS=( + "-var=deploy_runtime=true" + "-var=run_migration=true" + "-var=install_operator=true" + "-var=enable_event_dispatch=false" + "-var=enable_automatic_inventory=false" + "-var=canary_mode=true" + ) + ;; + canary) + STAGE_ARGS=( + "-var=deploy_runtime=true" + "-var=run_migration=true" + "-var=install_operator=true" + "-var=enable_event_dispatch=true" + "-var=enable_automatic_inventory=false" + "-var=canary_mode=true" + ) + ;; + activate) + STAGE_ARGS=( + "-var=deploy_runtime=true" + "-var=run_migration=true" + "-var=install_operator=true" + "-var=enable_event_dispatch=true" + "-var=enable_automatic_inventory=true" + "-var=canary_mode=false" + ) + ;; + *) + die "unsupported Terraform stage: $1" + ;; + esac +} + +refresh_state_list() { + STATE_LIST="$(terraform "-chdir=${ROOT}" state list 2>/dev/null || true)" +} -STATE_LIST="$(terraform "-chdir=${ROOT}" state list 2>/dev/null || true)" state_contains() { [[ "${STATE_LIST}" == *"$1"* ]] } -case "${STAGE}" in - foundation) - if state_contains "aws_lambda_function.this" || state_contains "aws_lambda_invocation.migration" || state_contains "helm_release.operator"; then - echo "refusing to roll an existing runtime back to foundation; use a separately reviewed Terraform plan" >&2 - exit 1 - fi - ;; - runtime) - state_contains "aws_ecr_repository.this" || { - echo "foundation state was not found; apply the foundation stage first" >&2 - exit 1 - } - if state_contains "aws_lambda_invocation.migration" || state_contains "helm_release.operator"; then - echo "refusing to roll a migrated installation back to runtime-paused state" >&2 - exit 1 - fi - ;; - migrate) - state_contains "aws_lambda_function.this" || { - echo "paused runtime state was not found; apply the runtime stage first" >&2 - exit 1 - } - ;; - canary | activate) - state_contains "aws_lambda_invocation.migration" || { - echo "migration state was not found; apply the migrate stage first" >&2 - exit 1 - } - state_contains "helm_release.operator" || { - echo "operator release state was not found; apply the migrate stage first" >&2 - exit 1 - } - if [[ "${STAGE}" == "activate" ]]; then - command -v jq >/dev/null 2>&1 || { - echo "jq is required to verify the canary stage before activation" >&2 - exit 1 - } - if ! CURRENT_DEPLOYMENT_STATE="$( - terraform "-chdir=${ROOT}" output -json deployment_state - )"; then - echo "deployment_state output is unavailable; apply and verify the canary stage first" >&2 - exit 1 +validate_stage_progression() { + local stage="$1" + refresh_state_list + case "${stage}" in + foundation) + if state_contains "aws_lambda_function.this" || + state_contains "aws_lambda_invocation.migration" || + state_contains "helm_release.operator"; then + die "refusing to roll an existing runtime back to foundation; use a separately reviewed Terraform plan" + fi + ;; + runtime) + state_contains "aws_ecr_repository.this" || + die "foundation state was not found; apply the foundation stage first" + if state_contains "aws_lambda_invocation.migration" || state_contains "helm_release.operator"; then + die "refusing to roll a migrated installation back to runtime-paused state" + fi + ;; + migrate) + state_contains "aws_lambda_function.this" || + die "paused runtime state was not found; apply the runtime stage first" + ;; + canary | activate) + state_contains "aws_lambda_invocation.migration" || + die "migration state was not found; apply the migrate stage first" + state_contains "helm_release.operator" || + die "operator release state was not found; apply the migrate stage first" + if [[ "${stage}" == "activate" ]]; then + require_command jq + if ! CURRENT_DEPLOYMENT_STATE="$( + terraform "-chdir=${ROOT}" output -json deployment_state + )"; then + die "deployment_state output is unavailable; apply and verify the canary stage first" + fi + if ! jq -e ' + type == "object" and + keys == [ + "automatic_inventory_enabled", + "canary_mode", + "dispatch_enabled", + "finding_export_enabled", + "managed_canary_enabled", + "migration_run", + "operator_installed", + "periodic_coverage_enabled", + "periodic_snapshots_enabled", + "processor_reconciliation_enabled", + "runtime_created", + "signal_hints_enabled" + ] and + all(.[]; type == "boolean") and + .runtime_created and + .migration_run and + .operator_installed and + .canary_mode and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) + ' <<<"${CURRENT_DEPLOYMENT_STATE}" >/dev/null; then + die "activation requires a previously applied canary or pause-canary state" + fi fi + ;; + pause | pause-canary) + state_contains "aws_lambda_invocation.migration" || + die "migration state was not found; apply the migrate stage first" + state_contains "helm_release.operator" || + die "operator release state was not found; apply the migrate stage first" + ;; + esac +} + +validate_pause_plan() { + local stage="$1" + local plan_file="$2" + local plan_json="${WORK_DIR}/${stage}-${PLAN_COUNTER}.json" + require_command jq + terraform "-chdir=${ROOT}" show -json "${plan_file}" >"${plan_json}" + if ! jq -e ' + def disables_mapping_only: + .type == "aws_lambda_event_source_mapping" and + .change.before.enabled == true and + .change.after.enabled == false and + ((.change.before | del(.enabled)) == (.change.after | del(.enabled))); + def disables_rule_only: + .type == "aws_cloudwatch_event_rule" and + .change.before.state == "ENABLED" and + .change.after.state == "DISABLED" and + ((.change.before | del(.state)) == (.change.after | del(.state))); + [ + .resource_changes[]? + | select(.mode == "managed") + | select(.change.actions != ["no-op"]) + ] + | all(.[]; + (.change.actions == ["update"]) and + (disables_mapping_only or disables_rule_only) + ) + ' "${plan_json}" >/dev/null; then + die "refusing pause plan: it changes resources outside dispatch mappings and rules" + fi +} + +confirm_stage() { + local stage="$1" + if [[ "${PORTSCANNER_AUTO_APPROVE:-false}" == "true" ]]; then + return 0 + fi + printf 'Type %s to apply this saved plan: ' "${stage}" >&2 + read -r confirmation + [[ "${confirmation}" == "${stage}" ]] || + die "stage not confirmed; saved plan discarded" +} + +stage_success_message() { + case "$1" in + foundation) + echo "Foundation applied with all runtime dispatch disabled." + echo "Build and scan all seven images before running the runtime stage." + ;; + runtime) + echo "Digest-pinned runtime created with mappings, schedules, and EventBridge dispatch paused." + ;; + migrate) + echo "Keyed migration completed and the namespace-scoped operator was installed; dispatch remains paused." + ;; + canary) + echo "One-target queue and stream processing are active; automatic inventory remains paused." + ;; + activate) + echo "Queue and stream processing, schedules, and filtered EC2 hint forwarding are active." + ;; + pause | pause-canary) + echo "Event mappings, schedules, and filtered EC2 hint forwarding are paused; runtime and evidence are retained." + ;; + esac +} + +apply_stage() { + local stage="$1" + local plan_file + local paused_state + + case "${stage}" in + runtime | migrate | canary | activate | pause | pause-canary) + ensure_image_inputs + ;; + esac + set_stage_args "${stage}" + validate_stage_progression "${stage}" + PLAN_COUNTER=$((PLAN_COUNTER + 1)) + plan_file="${WORK_DIR}/${PLAN_COUNTER}-${stage}.tfplan" + + terraform "-chdir=${ROOT}" plan \ + -input=false \ + -out="${plan_file}" \ + "${PLAN_ARGS[@]}" \ + "${STAGE_ARGS[@]}" + + case "${stage}" in + pause | pause-canary) + validate_pause_plan "${stage}" "${plan_file}" + ;; + esac + + confirm_stage "${stage}" + terraform "-chdir=${ROOT}" apply "${plan_file}" + case "${stage}" in + pause | pause-canary) + paused_state="$(current_deployment_state)" if ! jq -e ' - type == "object" and - keys == ["automatic_inventory_enabled", "canary_mode", "dispatch_enabled", "migration_run", "operator_installed", "runtime_created"] and - all(.[]; type == "boolean") and - .runtime_created and - .migration_run and - .operator_installed and - .canary_mode and - (.automatic_inventory_enabled == false) - ' <<<"${CURRENT_DEPLOYMENT_STATE}" >/dev/null; then - echo "activation requires a previously applied canary or pause-canary state" >&2 - exit 1 + (.dispatch_enabled == false) and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) + ' <<<"${paused_state}" >/dev/null; then + die "${stage} applied but one or more dispatch/reconciliation controls remain enabled" fi - fi - ;; - pause | pause-canary) - state_contains "aws_lambda_invocation.migration" || { - echo "migration state was not found; apply the migrate stage first" >&2 - exit 1 - } - state_contains "helm_release.operator" || { - echo "operator release state was not found; apply the migrate stage first" >&2 - exit 1 - } - ;; -esac + ;; + esac + stage_success_message "${stage}" +} + +current_deployment_state() { + local state_json + require_command jq + if ! state_json="$(terraform "-chdir=${ROOT}" output -json deployment_state)"; then + die "deployment_state output is unavailable; run bootstrap.sh first" + fi + if ! jq -e ' + type == "object" and + keys == [ + "automatic_inventory_enabled", + "canary_mode", + "dispatch_enabled", + "finding_export_enabled", + "managed_canary_enabled", + "migration_run", + "operator_installed", + "periodic_coverage_enabled", + "periodic_snapshots_enabled", + "processor_reconciliation_enabled", + "runtime_created", + "signal_hints_enabled" + ] and + all(.[]; type == "boolean") and + ((.migration_run | not) or .runtime_created) and + ((.operator_installed | not) or .migration_run) and + ((.dispatch_enabled | not) or .operator_installed) and + ((.automatic_inventory_enabled | not) or .dispatch_enabled) and + ((.periodic_snapshots_enabled | not) or .dispatch_enabled) and + ((.periodic_coverage_enabled | not) or .dispatch_enabled) and + ((.signal_hints_enabled | not) or .dispatch_enabled) and + ((.processor_reconciliation_enabled | not) or (.runtime_created and .migration_run)) and + ( + (.canary_mode | not) or ( + .operator_installed and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) + ) + ) + ' <<<"${state_json}" >/dev/null; then + die "deployment_state is malformed or violates staged deployment ordering" + fi + printf '%s' "${state_json}" +} -terraform "-chdir=${ROOT}" plan \ - -input=false \ - -out="${PLAN_FILE}" \ - "${PLAN_ARGS[@]}" \ - "${STAGE_ARGS[@]}" +run_ready() { + local state_json + local pause_stage + state_json="$(current_deployment_state)" -case "${STAGE}" in - pause | pause-canary) - command -v jq >/dev/null 2>&1 || { - echo "jq is required to validate a pause plan" >&2 - exit 1 - } - PLAN_JSON="${WORK_DIR}/${STAGE}.json" - terraform "-chdir=${ROOT}" show -json "${PLAN_FILE}" >"${PLAN_JSON}" - if ! jq -e ' - def disables_mapping_only: - .type == "aws_lambda_event_source_mapping" and - .change.before.enabled == true and - .change.after.enabled == false and - ((.change.before | del(.enabled)) == (.change.after | del(.enabled))); - def disables_rule_only: - .type == "aws_cloudwatch_event_rule" and - .change.before.state == "ENABLED" and - .change.after.state == "DISABLED" and - ((.change.before | del(.state)) == (.change.after | del(.state))); - [ - .resource_changes[]? - | select(.mode == "managed") - | select(.change.actions != ["no-op"]) - ] - | all(.[]; - (.change.actions == ["update"]) and - (disables_mapping_only or disables_rule_only) - ) - ' "${PLAN_JSON}" >/dev/null; then - echo "refusing pause plan: it changes resources outside dispatch mappings and rules" >&2 - exit 1 + if ! jq -e '.runtime_created' <<<"${state_json}" >/dev/null; then + apply_stage runtime + state_json="$(current_deployment_state)" + fi + if ! jq -e '.migration_run and .operator_installed' <<<"${state_json}" >/dev/null; then + apply_stage migrate + state_json="$(current_deployment_state)" + fi + if ! jq -e ' + .runtime_created and + .migration_run and + .operator_installed + ' <<<"${state_json}" >/dev/null; then + die "ready orchestration did not reach installed migrated runtime" + fi + if jq -e ' + .dispatch_enabled or + .automatic_inventory_enabled or + .periodic_snapshots_enabled or + .periodic_coverage_enabled or + .signal_hints_enabled or + .processor_reconciliation_enabled + ' <<<"${state_json}" >/dev/null; then + if jq -e '.canary_mode' <<<"${state_json}" >/dev/null; then + pause_stage="pause-canary" + else + pause_stage="pause" fi - ;; -esac + apply_stage "${pause_stage}" + state_json="$(current_deployment_state)" + fi + if ! jq -e ' + (.dispatch_enabled == false) and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) + ' <<<"${state_json}" >/dev/null; then + die "ready orchestration did not leave dispatch paused" + fi + echo "Runtime, migration, and operator are ready; dispatch is paused." +} -if [[ "${PORTSCANNER_AUTO_APPROVE:-false}" != "true" ]]; then - printf 'Type %s to apply this saved plan: ' "${STAGE}" - read -r CONFIRMATION - if [[ "${CONFIRMATION}" != "${STAGE}" ]]; then - echo "stage not confirmed; saved plan discarded" >&2 - exit 1 +ensure_paused_for_destroy() { + local state_json + local pause_stage + state_json="$(current_deployment_state)" + if jq -e ' + .dispatch_enabled or + .automatic_inventory_enabled or + .periodic_snapshots_enabled or + .periodic_coverage_enabled or + .signal_hints_enabled or + .processor_reconciliation_enabled + ' <<<"${state_json}" >/dev/null; then + if jq -e '.canary_mode' <<<"${state_json}" >/dev/null; then + pause_stage="pause-canary" + else + pause_stage="pause" + fi + apply_stage "${pause_stage}" + state_json="$(current_deployment_state)" fi -fi + if ! jq -e ' + (.dispatch_enabled == false) and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) + ' <<<"${state_json}" >/dev/null; then + die "destroy requires a verified paused deployment" + fi + PAUSED_STATE_JSON="${state_json}" +} -terraform "-chdir=${ROOT}" apply "${PLAN_FILE}" +run_destroy() { + local state_json + local destroy_plan + local confirmation + [[ "${CANONICAL_MODE}" == "true" && "${ROOT}" == "${CANONICAL_ROOT}" ]] || + die "destroy is restricted to the canonical deployment root" + [[ "${DISPOSABLE}" == "true" ]] || + die "destroy refused: environment.retention.disposable must explicitly be true" + [[ "${DESTROY_DATA_ON_TEARDOWN}" == "true" ]] || + die "destroy refused: environment.retention.destroy_data_on_teardown must explicitly be true" + + ensure_paused_for_destroy + state_json="${PAUSED_STATE_JSON}" + if jq -e '.runtime_created' <<<"${state_json}" >/dev/null; then + ensure_image_inputs + fi + if jq -e '.operator_installed' <<<"${state_json}" >/dev/null; then + RETIREMENT_READINESS_HELPER="${PORTSCANNER_CANARY_RETIRE_HELPER:-${SCRIPT_DIR}/retire-canary.sh}" + [[ -x "${RETIREMENT_READINESS_HELPER}" ]] || + die "destroy requires the retirement readiness helper: ${RETIREMENT_READINESS_HELPER}" + "${RETIREMENT_READINESS_HELPER}" \ + --verify-only \ + --terraform-root "${ROOT}" \ + --environment-config "${CONFIG_FILE}" \ + --backend-config "${BACKEND_CONFIG}" \ + --image-inputs "${IMAGE_INPUTS}" + fi + if jq -e '.operator_installed' <<<"${state_json}" >/dev/null; then + if jq -e '.canary_mode' <<<"${state_json}" >/dev/null; then + set_stage_args pause-canary + else + set_stage_args pause + fi + elif jq -e '.runtime_created' <<<"${state_json}" >/dev/null; then + set_stage_args runtime + else + set_stage_args foundation + fi + + PLAN_COUNTER=$((PLAN_COUNTER + 1)) + destroy_plan="${WORK_DIR}/${PLAN_COUNTER}-destroy.tfplan" + terraform "-chdir=${ROOT}" plan \ + -destroy \ + -input=false \ + -out="${destroy_plan}" \ + "${PLAN_ARGS[@]}" \ + "${STAGE_ARGS[@]}" + terraform "-chdir=${ROOT}" show "${destroy_plan}" + printf 'Type destroy %s to apply this exact saved destroy plan: ' "${ENVIRONMENT_NAME}" >&2 + read -r confirmation + [[ "${confirmation}" == "destroy ${ENVIRONMENT_NAME}" ]] || + die "destroy not confirmed; saved plan discarded" + terraform "-chdir=${ROOT}" apply "${destroy_plan}" + echo "Canonical application resources were destroyed." + echo "The state-bootstrap bucket, lock table, backend configuration, and bootstrap state were retained." +} case "${STAGE}" in - foundation) - echo "Foundation applied with all runtime dispatch disabled." - echo "Build and scan inventory, generator, parser, processor, migrator, operator, and scanner images externally." - echo "Push them to repository_urls and record immutable sha256 digests before running the runtime stage." + foundation | runtime | migrate | canary | activate | pause | pause-canary) + apply_stage "${STAGE}" ;; - runtime) - echo "Digest-pinned runtime created with mappings, schedules, and EventBridge dispatch paused." - echo "Review the plan and migration checksum before running the migrate stage." + ready) + run_ready ;; - migrate) - echo "Keyed migration completed and the namespace-scoped operator was installed; dispatch remains paused." - echo "Verify EKS and application health before running the canary stage." + evaluate) + run_ready + CANARY_HELPER="${PORTSCANNER_CANARY_HELPER:-${SCRIPT_DIR}/evaluate-canary.sh}" + [[ -x "${CANARY_HELPER}" ]] || + die "evaluate unavailable: managed-canary trigger/status integration is not installed at ${CANARY_HELPER}; ready completed and dispatch remains paused" + EVALUATION_NEEDS_PAUSE=true + apply_stage canary + "${CANARY_HELPER}" \ + --terraform-root "${ROOT}" \ + --environment-config "${CONFIG_FILE}" \ + --backend-config "${BACKEND_CONFIG}" \ + --image-inputs "${IMAGE_INPUTS}" + apply_stage pause-canary + EVALUATION_PAUSED=true + echo "Managed canary evaluation completed and dispatch returned to pause-canary." ;; - canary) - echo "One-target priority/result and stream processing are active; signal/coverage consumers, schedules, and EventBridge inventory hints remain paused." - echo "Invoke the snapshot Lambda once with an exact account_id, region, and target_public_ipv4, then return to the pause-canary stage." + retire-canary) + CANARY_RETIRE_HELPER="${PORTSCANNER_CANARY_RETIRE_HELPER:-${SCRIPT_DIR}/retire-canary.sh}" + [[ -x "${CANARY_RETIRE_HELPER}" ]] || + die "retire-canary unavailable: managed-canary Terraform retirement integration is not installed at ${CANARY_RETIRE_HELPER}" + ensure_paused_for_destroy + "${CANARY_RETIRE_HELPER}" \ + --terraform-root "${ROOT}" \ + --environment-config "${CONFIG_FILE}" \ + --backend-config "${BACKEND_CONFIG}" \ + --image-inputs "${IMAGE_INPUTS}" ;; - activate) - echo "Queue and stream processing, schedules, and filtered EC2 hint forwarding are active." + destroy) + run_destroy ;; - pause | pause-canary) - echo "Event mappings, schedules, and filtered EC2 hint forwarding are paused; runtime and evidence are retained." + *) + usage ;; esac diff --git a/terraform/aws/scripts/evaluate-canary.sh b/terraform/aws/scripts/evaluate-canary.sh new file mode 100755 index 0000000..0049310 --- /dev/null +++ b/terraform/aws/scripts/evaluate-canary.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT +set -euo pipefail + +usage() { + echo "usage: $0 --terraform-root PATH --environment-config PATH --backend-config PATH --image-inputs PATH" >&2 + exit 2 +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" +} + +ROOT="" +CONFIG_FILE="" +BACKEND_CONFIG="" +IMAGE_INPUTS="" +while [[ $# -gt 0 ]]; do + case "$1" in + --terraform-root) + [[ $# -ge 2 ]] || usage + ROOT="$2" + shift 2 + ;; + --environment-config) + [[ $# -ge 2 ]] || usage + CONFIG_FILE="$2" + shift 2 + ;; + --backend-config) + [[ $# -ge 2 ]] || usage + BACKEND_CONFIG="$2" + shift 2 + ;; + --image-inputs) + [[ $# -ge 2 ]] || usage + IMAGE_INPUTS="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +[[ -n "${ROOT}" && -n "${CONFIG_FILE}" && -n "${BACKEND_CONFIG}" && -n "${IMAGE_INPUTS}" ]] || + usage +for path in "${CONFIG_FILE}" "${BACKEND_CONFIG}" "${IMAGE_INPUTS}"; do + [[ -f "${path}" && ! -L "${path}" ]] || die "required generated/configuration file is unavailable: ${path}" +done +[[ -d "${ROOT}" && -f "${ROOT}/main.tf" ]] || die "invalid Terraform root: ${ROOT}" + +for command_name in terraform aws jq mktemp; do + require_command "${command_name}" +done + +ACCOUNT_ID="$(jq -er '.environment.aws.account_id | select(test("^[0-9]{12}$"))' "${CONFIG_FILE}")" || + die "environment configuration has no valid AWS account ID" +REGION="$(jq -er '.environment.aws.region | select(test("^[a-z]{2}(-[a-z0-9]+)+-[0-9]+$"))' "${CONFIG_FILE}")" || + die "environment configuration has no valid AWS Region" +EXPECTED_SEVERITY="$(jq -er '.environment.managed_canary.expected_finding_severity' "${CONFIG_FILE}")" || + die "environment configuration has no managed-canary severity" +[[ "${EXPECTED_SEVERITY}" == "low" ]] || + die "managed-canary verification currently requires expected_finding_severity=low" + +ACTUAL_ACCOUNT_ID="$( + aws sts get-caller-identity \ + --region "${REGION}" \ + --query Account \ + --output text +)" || die "AWS identity is unavailable" +[[ "${ACTUAL_ACCOUNT_ID}" == "${ACCOUNT_ID}" ]] || + die "AWS identity mismatch: expected ${ACCOUNT_ID}, got ${ACTUAL_ACCOUNT_ID}" + +DEPLOYMENT_STATE="$(terraform "-chdir=${ROOT}" output -json deployment_state)" || + die "deployment_state output is unavailable" +if ! jq -e ' + .runtime_created and + .migration_run and + .operator_installed and + .dispatch_enabled and + .canary_mode and + .managed_canary_enabled and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) +' <<<"${DEPLOYMENT_STATE}" >/dev/null; then + die "managed-canary evaluation requires the active one-target canary stage" +fi + +CANARY="$(terraform "-chdir=${ROOT}" output -json managed_canary)" || + die "managed_canary output is unavailable" +if ! jq -e --arg account "${ACCOUNT_ID}" --arg region "${REGION}" ' + type == "object" and + .account_id == $account and + .region == $region and + (.network_interface_id | type == "string" and test("^eni-[0-9a-f]+$")) and + (.instance_id | type == "string" and test("^i-[0-9a-f]+$")) and + .instance_state == "running" and + (.public_ip | type == "string" and length > 0) and + .public_cidr == (.public_ip + "/32") and + .listener_port == 18080 and + (.inventory_tag_key | type == "string" and length > 0) and + (.inventory_tag_value | type == "string" and length > 0) +' <<<"${CANARY}" >/dev/null; then + die "managed_canary output is malformed or the instance is not running" +fi + +SNAPSHOT_INVOCATION="$( + terraform "-chdir=${ROOT}" output -json managed_canary_snapshot_invocation +)" || die "managed-canary snapshot invocation output is unavailable" +STATUS_INVOCATION="$( + terraform "-chdir=${ROOT}" output -json managed_canary_status_invocation +)" || die "managed-canary status invocation output is unavailable" + +validate_invocation() { + local invocation="$1" + local operation="$2" + jq -e \ + --arg account "${ACCOUNT_ID}" \ + --arg region "${REGION}" \ + --arg operation "${operation}" ' + type == "object" and + keys == ["function_arn", "function_name", "payload"] and + (.function_name | type == "string" and length > 0) and + (.function_arn | type == "string") and + (.function_arn | test( + "^arn:[^:]+:lambda:" + $region + ":" + $account + + ":function:[A-Za-z0-9-_]+$" + )) and + .function_arn == ( + (.function_arn | split(":")[0:6] | join(":")) + ":" + .function_name + ) and + .payload == {operation: $operation} + ' <<<"${invocation}" >/dev/null || + die "managed-canary ${operation} invocation output is malformed" +} +validate_invocation "${SNAPSHOT_INVOCATION}" "managed-canary" +validate_invocation "${STATUS_INVOCATION}" "managed-canary-status" + +TIMEOUT_SECONDS="${PORTSCANNER_CANARY_TIMEOUT_SECONDS:-1800}" +POLL_SECONDS="${PORTSCANNER_CANARY_POLL_SECONDS:-10}" +STABILITY_POLLS="${PORTSCANNER_CANARY_STABILITY_POLLS:-3}" +[[ "${TIMEOUT_SECONDS}" =~ ^[0-9]+$ && "${TIMEOUT_SECONDS}" -ge 60 && "${TIMEOUT_SECONDS}" -le 7200 ]] || + die "PORTSCANNER_CANARY_TIMEOUT_SECONDS must be an integer from 60 through 7200" +[[ "${POLL_SECONDS}" =~ ^[0-9]+$ && "${POLL_SECONDS}" -le 300 ]] || + die "PORTSCANNER_CANARY_POLL_SECONDS must be an integer from 0 through 300" +[[ "${STABILITY_POLLS}" =~ ^[0-9]+$ && "${STABILITY_POLLS}" -ge 1 && "${STABILITY_POLLS}" -le 12 ]] || + die "PORTSCANNER_CANARY_STABILITY_POLLS must be an integer from 1 through 12" + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-canary.XXXXXX")" +cleanup() { + rm -rf -- "${WORK_DIR}" +} +handle_signal() { + exit 130 +} +trap cleanup EXIT +trap handle_signal HUP INT TERM +INVOKE_COUNTER=0 +INVOKE_PAYLOAD="" + +invoke_lambda() { + local invocation="$1" + local label="$2" + local function_name + local request + local response_file + local metadata + + INVOKE_COUNTER=$((INVOKE_COUNTER + 1)) + function_name="$(jq -er .function_name <<<"${invocation}")" + request="$(jq -ce .payload <<<"${invocation}")" + response_file="${WORK_DIR}/${INVOKE_COUNTER}-${label}.json" + metadata="$( + aws lambda invoke \ + --function-name "${function_name}" \ + --invocation-type RequestResponse \ + --cli-binary-format raw-in-base64-out \ + --payload "${request}" \ + --region "${REGION}" \ + --output json \ + "${response_file}" + )" || die "${label} Lambda invocation failed" + if ! jq -e ' + .StatusCode == 200 and + (has("FunctionError") | not) + ' <<<"${metadata}" >/dev/null; then + die "${label} Lambda reported a function error" + fi + jq -e . "${response_file}" >/dev/null 2>&1 || + die "${label} Lambda returned a non-JSON payload" + INVOKE_PAYLOAD="$(<"${response_file}")" +} + +validate_snapshot_response() { + local payload="$1" + jq -e ' + type == "array" and + length == 1 and + .[0] as $summary | + ($summary | type == "object") and + $summary.completion == "complete" and + $summary.targets == 1 and + ($summary.pages | type == "number" and . >= 1) and + ($summary.revalidated | type == "number" and . >= 0) and + ($summary.added | type == "number" and . >= 0) and + ($summary.changed | type == "number" and . >= 0) and + ($summary.noop | type == "number" and . >= 0) and + $summary.removed == 0 and + $summary.race == 0 + ' <<<"${payload}" >/dev/null || + die "managed-canary snapshot did not produce one complete, non-removing target result" +} + +validate_status_response() { + local payload="$1" + jq -e ' + type == "object" and + keys == [ + "attempt_count", + "complete_coverage_count", + "event_count", + "low_finding_count", + "open_exposure_count", + "operation", + "ready", + "status", + "target_count", + "unexpected_high_finding_count" + ] and + .operation == "managed-canary-status" and + (.ready | type == "boolean") and + .status == (if .ready then "ready" else "pending" end) and + all( + .target_count, + .event_count, + .attempt_count, + .complete_coverage_count, + .open_exposure_count, + .low_finding_count, + .unexpected_high_finding_count; + type == "number" and . >= 0 and floor == . + ) and + .target_count <= 1 and + .event_count <= 1 and + .attempt_count <= 1 and + .complete_coverage_count <= 1 and + .open_exposure_count <= 1 and + .low_finding_count <= 1 and + .unexpected_high_finding_count == 0 + ' <<<"${payload}" >/dev/null || + die "managed-canary status returned an invalid or non-idempotent result" +} + +wait_until_ready() { + local deadline=$((SECONDS + TIMEOUT_SECONDS)) + while ((SECONDS <= deadline)); do + invoke_lambda "${STATUS_INVOCATION}" "status" + validate_status_response "${INVOKE_PAYLOAD}" + if jq -e '.ready' <<<"${INVOKE_PAYLOAD}" >/dev/null; then + return 0 + fi + jq -c '{ + status, + target_count, + event_count, + attempt_count, + complete_coverage_count, + open_exposure_count, + low_finding_count + }' <<<"${INVOKE_PAYLOAD}" >&2 + sleep "${POLL_SECONDS}" + done + die "managed-canary finding did not become ready within ${TIMEOUT_SECONDS} seconds" +} + +invoke_lambda "${SNAPSHOT_INVOCATION}" "snapshot" +validate_snapshot_response "${INVOKE_PAYLOAD}" +wait_until_ready + +# Replay the same trusted snapshot and require a stable one-event/one-attempt +# result. This proves the public evaluation path is idempotent before pause. +invoke_lambda "${SNAPSHOT_INVOCATION}" "snapshot-replay" +validate_snapshot_response "${INVOKE_PAYLOAD}" +for ((poll = 1; poll <= STABILITY_POLLS; poll++)); do + if ((POLL_SECONDS > 0)); then + sleep "${POLL_SECONDS}" + fi + invoke_lambda "${STATUS_INVOCATION}" "status-stability-${poll}" + validate_status_response "${INVOKE_PAYLOAD}" + jq -e ' + .ready and + .target_count == 1 and + .event_count == 1 and + .attempt_count == 1 and + .complete_coverage_count == 1 and + .open_exposure_count == 1 and + .low_finding_count == 1 and + .unexpected_high_finding_count == 0 + ' <<<"${INVOKE_PAYLOAD}" >/dev/null || + die "managed-canary replay changed the effective target, event, attempt, exposure, or finding" +done + +echo "Managed canary verified: one targeted TCP 18080 attempt and one low PostgreSQL finding." diff --git a/terraform/aws/scripts/retire-canary.sh b/terraform/aws/scripts/retire-canary.sh new file mode 100755 index 0000000..9f0dfc9 --- /dev/null +++ b/terraform/aws/scripts/retire-canary.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT +set -euo pipefail + +usage() { + echo "usage: $0 [--verify-only] --terraform-root PATH --environment-config PATH --backend-config PATH --image-inputs PATH" >&2 + exit 2 +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" +} + +file_mode() { + if stat -f '%Lp' "$1" 2>/dev/null; then + return + fi + stat -c '%a' "$1" +} + +ROOT="" +CONFIG_FILE="" +BACKEND_CONFIG="" +IMAGE_INPUTS="" +VERIFY_ONLY=false +while [[ $# -gt 0 ]]; do + case "$1" in + --verify-only) + VERIFY_ONLY=true + shift + ;; + --terraform-root) + [[ $# -ge 2 ]] || usage + ROOT="$2" + shift 2 + ;; + --environment-config) + [[ $# -ge 2 ]] || usage + CONFIG_FILE="$2" + shift 2 + ;; + --backend-config) + [[ $# -ge 2 ]] || usage + BACKEND_CONFIG="$2" + shift 2 + ;; + --image-inputs) + [[ $# -ge 2 ]] || usage + IMAGE_INPUTS="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +[[ -n "${ROOT}" && -n "${CONFIG_FILE}" && -n "${BACKEND_CONFIG}" && -n "${IMAGE_INPUTS}" ]] || + usage +[[ -d "${ROOT}" && -f "${ROOT}/main.tf" ]] || die "invalid Terraform root: ${ROOT}" +[[ -f "${CONFIG_FILE}" && ! -L "${CONFIG_FILE}" ]] || + die "environment configuration is unavailable" +for generated in "${BACKEND_CONFIG}" "${IMAGE_INPUTS}"; do + [[ -f "${generated}" && ! -L "${generated}" && "$(file_mode "${generated}")" == "600" ]] || + die "generated deployment input must be a mode-0600 regular file: ${generated}" +done + +for command_name in terraform aws jq kubectl mktemp; do + require_command "${command_name}" +done + +ACCOUNT_ID="$(jq -er '.environment.aws.account_id | select(test("^[0-9]{12}$"))' "${CONFIG_FILE}")" || + die "environment configuration has no valid AWS account ID" +REGION="$(jq -er '.environment.aws.region | select(test("^[a-z]{2}(-[a-z0-9]+)+-[0-9]+$"))' "${CONFIG_FILE}")" || + die "environment configuration has no valid AWS Region" +ENVIRONMENT_NAME="$(jq -er '.environment.name | select(test("^[a-z][a-z0-9-]{1,19}$"))' "${CONFIG_FILE}")" || + die "environment configuration has no valid environment name" +if [[ "${VERIFY_ONLY}" != "true" && + "$(jq -er '.environment.managed_canary.enabled | tostring' "${CONFIG_FILE}")" != "false" ]]; then + die "set environment.managed_canary.enabled=false in the environment file before retirement" +fi + +ACTUAL_ACCOUNT_ID="$( + aws sts get-caller-identity \ + --region "${REGION}" \ + --query Account \ + --output text +)" || die "AWS identity is unavailable" +[[ "${ACTUAL_ACCOUNT_ID}" == "${ACCOUNT_ID}" ]] || + die "AWS identity mismatch: expected ${ACCOUNT_ID}, got ${ACTUAL_ACCOUNT_ID}" + +terraform "-chdir=${ROOT}" init \ + -input=false \ + "-backend-config=${BACKEND_CONFIG}" >/dev/null + +STATE="$(terraform "-chdir=${ROOT}" output -json deployment_state)" || + die "deployment_state output is unavailable" +if ! jq -e ' + .runtime_created and + .migration_run and + .operator_installed and + (.dispatch_enabled == false) and + (.automatic_inventory_enabled == false) and + (.periodic_snapshots_enabled == false) and + (.periodic_coverage_enabled == false) and + (.signal_hints_enabled == false) and + (.processor_reconciliation_enabled == false) +' <<<"${STATE}" >/dev/null; then + die "managed-canary retirement requires a fully paused installed deployment" +fi +if [[ "${VERIFY_ONLY}" != "true" ]] && + jq -e '.managed_canary_enabled == false' <<<"${STATE}" >/dev/null; then + echo "Managed canary is already retired." + exit 0 +fi + +CURRENT_FINGERPRINT="$( + terraform "-chdir=${ROOT}" output -raw retirement_configuration_fingerprint +)" || die "retirement configuration fingerprint is unavailable" +[[ "${CURRENT_FINGERPRINT}" =~ ^[0-9a-f]{64}$ ]] || + die "current retirement configuration fingerprint is malformed" + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-retire-canary.XXXXXX")" +cleanup() { + rm -rf -- "${WORK_DIR}" +} +handle_signal() { + exit 130 +} +trap cleanup EXIT +trap handle_signal HUP INT TERM + +QUEUE_URLS="$(terraform "-chdir=${ROOT}" output -json queue_urls)" || + die "queue_urls output is unavailable" +if ! jq -e 'type == "object" and all(.[]; type == "string" and length > 0)' \ + <<<"${QUEUE_URLS}" >/dev/null; then + die "queue_urls output is malformed" +fi + +QUEUE_STABILITY_POLLS="${PORTSCANNER_RETIRE_QUEUE_POLLS:-2}" +QUEUE_POLL_SECONDS="${PORTSCANNER_RETIRE_QUEUE_POLL_SECONDS:-5}" +[[ "${QUEUE_STABILITY_POLLS}" =~ ^[0-9]+$ && + "${QUEUE_STABILITY_POLLS}" -ge 2 && + "${QUEUE_STABILITY_POLLS}" -le 12 ]] || + die "PORTSCANNER_RETIRE_QUEUE_POLLS must be an integer from 2 through 12" +[[ "${QUEUE_POLL_SECONDS}" =~ ^[0-9]+$ && "${QUEUE_POLL_SECONDS}" -le 300 ]] || + die "PORTSCANNER_RETIRE_QUEUE_POLL_SECONDS must be an integer from 0 through 300" + +for ((poll = 1; poll <= QUEUE_STABILITY_POLLS; poll++)); do + while IFS= read -r queue_url; do + ATTRIBUTES="$( + aws sqs get-queue-attributes \ + --queue-url "${queue_url}" \ + --attribute-names \ + ApproximateNumberOfMessages \ + ApproximateNumberOfMessagesNotVisible \ + ApproximateNumberOfMessagesDelayed \ + --region "${REGION}" \ + --query Attributes \ + --output json + )" || die "could not inspect queue readiness" + if ! jq -e ' + type == "object" and + ((.ApproximateNumberOfMessages // "0") | tonumber) == 0 and + ((.ApproximateNumberOfMessagesNotVisible // "0") | tonumber) == 0 and + ((.ApproximateNumberOfMessagesDelayed // "0") | tonumber) == 0 + ' <<<"${ATTRIBUTES}" >/dev/null; then + die "all queues must be drained before releasing the managed-canary EIP" + fi + done < <(jq -r '.[]' <<<"${QUEUE_URLS}") + if ((poll < QUEUE_STABILITY_POLLS && QUEUE_POLL_SECONDS > 0)); then + sleep "${QUEUE_POLL_SECONDS}" + fi +done + +CLUSTER_NAME="$(terraform "-chdir=${ROOT}" output -raw eks_cluster_name)" || + die "eks_cluster_name output is unavailable" +NAMESPACE="$(terraform "-chdir=${ROOT}" output -raw operator_namespace)" || + die "operator_namespace output is unavailable" +[[ "${CLUSTER_NAME}" =~ ^[A-Za-z0-9][A-Za-z0-9_-]+$ ]] || + die "EKS cluster name is malformed" +[[ "${NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || + die "operator namespace is malformed" + +KUBECONFIG_FILE="${WORK_DIR}/kubeconfig" +aws eks update-kubeconfig \ + --name "${CLUSTER_NAME}" \ + --region "${REGION}" \ + --alias portscanner-retire-canary \ + --kubeconfig "${KUBECONFIG_FILE}" >/dev/null || + die "could not create the temporary EKS kubeconfig" +chmod 600 "${KUBECONFIG_FILE}" + +JOBS="$( + kubectl \ + --kubeconfig "${KUBECONFIG_FILE}" \ + --namespace "${NAMESPACE}" \ + get jobs \ + --selector app.kubernetes.io/name=portscanner,app.kubernetes.io/component=scanner \ + --output json +)" || die "could not inspect scanner Jobs" +if ! jq -e ' + type == "object" and + (.items | type == "array") and + all( + .items[]; + ((.status.active // 0) == 0) and + any( + .status.conditions[]?; + (.status == "True") and (.type == "Complete" or .type == "Failed") + ) + ) +' <<<"${JOBS}" >/dev/null; then + die "all scanner Jobs must be terminal before releasing the managed-canary EIP" +fi + +SCANNERS="$( + kubectl \ + --kubeconfig "${KUBECONFIG_FILE}" \ + --namespace "${NAMESPACE}" \ + get scanners.scanning.portscanner.io \ + --output json +)" || die "could not inspect Scanner resources" +if ! jq -e ' + type == "object" and + (.items | type == "array") and + all( + .items[]; + (.status.outcome // "") as $outcome | + $outcome == "Succeeded" or + $outcome == "Failed" or + $outcome == "Cancelled" or + $outcome == "Expired" + ) +' <<<"${SCANNERS}" >/dev/null; then + die "all Scanner resources must be terminal before releasing the managed-canary EIP" +fi + +if [[ "${VERIFY_ONLY}" == "true" ]]; then + echo "Retirement readiness verified: queues are drained and scanner work is terminal." + exit 0 +fi + +PLAN_FILE="${WORK_DIR}/retire-canary.tfplan" +PLAN_JSON="${WORK_DIR}/retire-canary.json" +terraform "-chdir=${ROOT}" plan \ + -input=false \ + -out="${PLAN_FILE}" \ + "-var-file=${CONFIG_FILE}" \ + "-var-file=${IMAGE_INPUTS}" \ + -var=deploy_runtime=true \ + -var=run_migration=true \ + -var=install_operator=true \ + -var=enable_event_dispatch=false \ + -var=enable_automatic_inventory=false \ + -var=canary_mode=false +terraform "-chdir=${ROOT}" show -json "${PLAN_FILE}" >"${PLAN_JSON}" + +PLANNED_FINGERPRINT="$( + jq -er '.planned_values.outputs.retirement_configuration_fingerprint.value' "${PLAN_JSON}" +)" || die "retirement plan has no configuration fingerprint" +[[ "${PLANNED_FINGERPRINT}" == "${CURRENT_FINGERPRINT}" ]] || + die "retirement plan changes environment settings other than managed_canary.enabled" + +if ! jq -e ' + [ + .resource_changes[]? + | select(.mode == "managed") + | select(.change.actions != ["no-op"]) + ] as $changes + | ($changes | length) > 0 + and any( + $changes[]; + .address == "module.portscanner.module.managed_canary[0].aws_eip.this" + and .change.actions == ["delete"] + ) + and any( + $changes[]; + .address == "module.portscanner.module.managed_canary[0].aws_instance.this" + and .change.actions == ["delete"] + ) + and all( + $changes[]; + if .change.actions == ["delete"] then + (.address | startswith("module.portscanner.module.managed_canary[0].")) + elif .change.actions == ["update"] then + ( + .address == "terraform_data.canonical_configuration" or + .address == "module.portscanner.terraform_data.deployment_validation" or + (.address | startswith("module.portscanner.module.functions.aws_lambda_function.this[")) or + .address == "module.portscanner.module.eks.helm_release.operator[0]" + ) + else + false + end + ) + and ( + .planned_values.outputs.deployment_state.value + | .managed_canary_enabled == false + and .canary_mode == false + and .dispatch_enabled == false + and .automatic_inventory_enabled == false + and .periodic_snapshots_enabled == false + and .periodic_coverage_enabled == false + and .signal_hints_enabled == false + and .processor_reconciliation_enabled == false + ) +' "${PLAN_JSON}" >/dev/null; then + die "retirement plan contains a change outside the reviewed managed-canary boundary" +fi + +terraform "-chdir=${ROOT}" show "${PLAN_FILE}" +if [[ "${PORTSCANNER_AUTO_APPROVE:-false}" != "true" ]]; then + printf 'Type retire-canary %s to apply this exact saved plan: ' "${ENVIRONMENT_NAME}" >&2 + read -r confirmation + [[ "${confirmation}" == "retire-canary ${ENVIRONMENT_NAME}" ]] || + die "managed-canary retirement not confirmed; saved plan discarded" +fi +terraform "-chdir=${ROOT}" apply "${PLAN_FILE}" + +RETIRED_STATE="$(terraform "-chdir=${ROOT}" output -json deployment_state)" || + die "deployment_state output is unavailable after retirement" +RETIRED_CANARY="$(terraform "-chdir=${ROOT}" output -json managed_canary)" || + die "managed_canary output is unavailable after retirement" +jq -e ' + .managed_canary_enabled == false and + .canary_mode == false and + .dispatch_enabled == false +' <<<"${RETIRED_STATE}" >/dev/null || + die "managed-canary retirement did not leave the deployment paused" +jq -e '. == null' <<<"${RETIRED_CANARY}" >/dev/null || + die "managed-canary resources remain in Terraform output after retirement" + +echo "Managed canary retired after queue, Job, Scanner, and saved-plan verification." diff --git a/terraform/aws/scripts/tests/bootstrap-test.sh b/terraform/aws/scripts/tests/bootstrap-test.sh new file mode 100755 index 0000000..12b6604 --- /dev/null +++ b/terraform/aws/scripts/tests/bootstrap-test.sh @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +set -euo pipefail + +TEST_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" +SCRIPT_DIR="$(CDPATH= cd -- "${TEST_DIR}/.." && pwd -P)" +BOOTSTRAP_SCRIPT="${SCRIPT_DIR}/bootstrap.sh" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-bootstrap-test.XXXXXX")" +cleanup() { + rm -rf -- "${WORK_DIR}" +} +trap cleanup EXIT HUP INT TERM + +FAKE_BIN="${WORK_DIR}/bin" +CONFIG_FILE="${WORK_DIR}/environment.auto.tfvars.json" +ARTIFACT_ROOT="${WORK_DIR}/artifacts" +TF_CALLS="${WORK_DIR}/terraform-calls" +AWS_CALLS="${WORK_DIR}/aws-calls" +DEPLOY_CALLS="${WORK_DIR}/deploy-calls" +BUILD_CALLS="${WORK_DIR}/build-calls" +mkdir -p "${FAKE_BIN}" +: >"${TF_CALLS}" +: >"${AWS_CALLS}" +: >"${DEPLOY_CALLS}" +: >"${BUILD_CALLS}" + +ENVIRONMENT_NAME="test-eval" +ACCOUNT_ID="123456789012" +REGION="us-east-1" +ARTIFACT_DIR="${ARTIFACT_ROOT}/${ENVIRONMENT_NAME}/${ACCOUNT_ID}/${REGION}" +BOOTSTRAP_STATE="${ARTIFACT_DIR}/state-bootstrap.tfstate" +BACKEND_CONFIG="${ARTIFACT_DIR}/backend.hcl" +IMAGE_INPUTS="${ARTIFACT_DIR}/images.tfvars" +BACKEND_KEY="environments/${ENVIRONMENT_NAME}/${ACCOUNT_ID}/${REGION}/deployment.tfstate" + +write_config() { + local environment_name="${1:-${ENVIRONMENT_NAME}}" + local installer_arn="arn:" + installer_arn+="aws:iam::${ACCOUNT_ID}:role/test-installer" + cat >"${CONFIG_FILE}" <"${FAKE_BIN}/terraform" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >>"${TF_CALLS:?}" +if [[ "${1:-}" == "version" && "${2:-}" == "-json" ]]; then + printf '%s\n' '{"terraform_version":"1.7.4"}' + exit 0 +fi + +root="${1#-chdir=}" +operation="${2:-}" +case "${operation}" in + init) + ;; + plan) + plan_file="" + for argument in "$@"; do + if [[ "${argument}" == -out=* ]]; then + plan_file="${argument#-out=}" + fi + done + [[ -n "${plan_file}" ]] || exit 91 + : >"${plan_file}" + ;; + show) + printf '%s\n' "saved bootstrap plan" + ;; + apply) + state_file="" + backup_file="" + for argument in "$@"; do + case "${argument}" in + -state=*) state_file="${argument#-state=}" ;; + -backup=*) backup_file="${argument#-backup=}" ;; + esac + done + [[ -n "${state_file}" ]] || exit 92 + if [[ -f "${state_file}" && -n "${backup_file}" ]]; then + cp "${state_file}" "${backup_file}" + fi + printf '%s\n' '{"version":4}' >"${state_file}" + ;; + output) + if [[ "${root}" == */state-bootstrap ]]; then + state_file="" + for argument in "$@"; do + case "${argument}" in + -state=*) state_file="${argument#-state=}" ;; + esac + done + [[ -f "${state_file}" ]] || exit 93 + printf '{"bucket":"%s","dynamodb_table":"%s","encrypt":true,"region":"%s"}\n' \ + "${FAKE_STATE_BUCKET:-test-eval-state-tfstate-abc123}" \ + "${FAKE_LOCK_TABLE:-test-eval-state-terraform-locks}" \ + "${FAKE_STATE_REGION:-us-east-1}" + else + [[ "${3:-}" == "-json" && "${4:-}" == "deployment_state" ]] || exit 94 + printf '%s\n' '{"runtime_created":false,"migration_run":false,"operator_installed":false,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":false}' + fi + ;; + *) + echo "unexpected terraform invocation: $*" >&2 + exit 95 + ;; +esac +EOF + +cat >"${FAKE_BIN}/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${AWS_CALLS:?}" +case "${1:-}/${2:-}" in + sts/get-caller-identity) + identity_arn="arn:aws:iam::123456789012:role/test-installer" + printf '{"Account":"%s","Arn":"%s","UserId":"test"}\n' \ + "${FAKE_AWS_ACCOUNT_ID:-123456789012}" \ + "${identity_arn}" + ;; + s3api/head-bucket) + [[ "${FAKE_BUCKET_FAILURE:-false}" != "true" ]] + ;; + dynamodb/list-tables) + if [[ "${FAKE_TABLE_EXISTS:-false}" == "true" ]]; then + printf '%s\n' '["test-eval-state-terraform-locks"]' + else + printf '%s\n' '[]' + fi + ;; + dynamodb/describe-table) + default_table_arn="arn:aws:dynamodb:us-east-1:123456789012:table/test-eval-state-terraform-locks" + printf '%s\n' "${FAKE_TABLE_ARN:-${default_table_arn}}" + ;; + *) + exit 96 + ;; +esac +EOF + +cat >"${FAKE_BIN}/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "${1:-}" == "buildx" && "${2:-}" == "version" ]] +EOF + +cat >"${FAKE_BIN}/trivy" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + +cat >"${FAKE_BIN}/git" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "${1:-}" == "-C" && "${2:-}" == "${EXPECTED_REPOSITORY_ROOT:?}" ]] +case "${3:-}/${4:-}" in + rev-parse/--show-toplevel) + printf '%s\n' "${EXPECTED_REPOSITORY_ROOT}" + ;; + rev-parse/--verify) + [[ "${5:-}" == "HEAD^{commit}" ]] || exit 97 + printf '%s\n' 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + ;; + status/--porcelain=v1) + [[ "${5:-}" == "--untracked-files=all" ]] || exit 97 + [[ "${FAKE_GIT_DIRTY:-false}" != "true" ]] || printf '%s\n' ' M changed-file' + ;; + *) + exit 97 + ;; +esac +EOF + +cat >"${FAKE_BIN}/fake-deploy" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${DEPLOY_CALLS:?}" +[[ "$*" == "foundation" ]] +EOF + +cat >"${FAKE_BIN}/fake-build-images" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${BUILD_CALLS:?}" +if [[ "${FAKE_BUILD_OUTPUT:-valid}" == "invalid" ]]; then + printf '%s\n' 'invalid generated input' + exit 0 +fi +DIGEST="sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +cat <"${WORK_DIR}/first-output" 2>"${WORK_DIR}/first-error" || + fail "valid first bootstrap failed" + +for generated_file in "${BOOTSTRAP_STATE}" "${BACKEND_CONFIG}" "${IMAGE_INPUTS}"; do + [[ -f "${generated_file}" ]] || fail "bootstrap did not generate ${generated_file}" + GENERATED_MODE="$(stat -f '%Lp' "${generated_file}" 2>/dev/null || stat -c '%a' "${generated_file}")" + [[ "${GENERATED_MODE}" == "600" ]] || + fail "generated file was not mode 0600: ${generated_file}" +done +BACKEND_CONTENT="$(<"${BACKEND_CONFIG}")" +[[ "${BACKEND_CONTENT}" == *"key = \"${BACKEND_KEY}\""* ]] || + fail "backend key was not deterministically derived from environment/account/Region" +for credential_field in access_key secret_key token profile shared_credentials_file; do + [[ "${BACKEND_CONTENT}" != *"${credential_field}"* ]] || + fail "backend configuration contains credential material" +done +[[ "$(<"${DEPLOY_CALLS}")" == "foundation" ]] || + fail "bootstrap did not apply the paused foundation through deploy.sh" +[[ "$(<"${BUILD_CALLS}")" == *"/terraform/aws/deployment arm64"* ]] || + fail "bootstrap did not invoke build-images for the canonical root and architecture" +TF_LOG="$(<"${TF_CALLS}")" +[[ "${TF_LOG}" == *"-chdir="*"/terraform/aws/deployment init -input=false -reconfigure -backend-config=${BACKEND_CONFIG}"* ]] || + fail "bootstrap did not reconfigure the canonical S3 backend" +[[ "${TF_LOG}" != *"-target"* ]] || + fail "bootstrap used a targeted Terraform operation" + +FIRST_BACKEND_CONTENT="$(<"${BACKEND_CONFIG}")" +run_bootstrap >/dev/null 2>"${WORK_DIR}/resume-error" || + fail "resumable bootstrap failed" +[[ "$(<"${BACKEND_CONFIG}")" == "${FIRST_BACKEND_CONTENT}" ]] || + fail "resumable bootstrap rewrote backend identity" +[[ "$(<"${WORK_DIR}/resume-error")" == *"verified resumable state bootstrap"* ]] || + fail "resumable bootstrap did not report state verification" + +rm "${BACKEND_CONFIG}" +run_bootstrap >/dev/null 2>&1 || + fail "bootstrap did not safely regenerate a missing backend file from verified bootstrap state" +[[ "$(<"${BACKEND_CONFIG}")" == "${FIRST_BACKEND_CONTENT}" ]] || + fail "regenerated backend file did not match the original identity" + +printf '%s\n' 'key = "unsafe/../collision.tfstate"' >"${BACKEND_CONFIG}" +chmod 600 "${BACKEND_CONFIG}" +if run_bootstrap >/dev/null 2>"${WORK_DIR}/backend-collision-error"; then + fail "bootstrap overwrote a colliding backend configuration" +fi +[[ "$(<"${BACKEND_CONFIG}")" == 'key = "unsafe/../collision.tfstate"' ]] || + fail "bootstrap modified a colliding backend configuration" +[[ "$(<"${WORK_DIR}/backend-collision-error")" == *"backend configuration collision"* ]] || + fail "backend collision failure was not explicit" +printf '%s' "${FIRST_BACKEND_CONTENT}" >"${BACKEND_CONFIG}" +chmod 600 "${BACKEND_CONFIG}" + +if FAKE_LOCK_TABLE="other-environment-locks" run_bootstrap >/dev/null 2>"${WORK_DIR}/state-collision-error"; then + fail "bootstrap accepted state from a colliding environment" +fi +[[ "$(<"${WORK_DIR}/state-collision-error")" == *"bootstrap state collision"* ]] || + fail "bootstrap state collision failure was not explicit" + +AWS_CALL_COUNT_BEFORE="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +jq '.environment.credentials = {"access_key": "must-not-parse"}' \ + "${CONFIG_FILE}" >"${WORK_DIR}/invalid-config.json" +mv "${WORK_DIR}/invalid-config.json" "${CONFIG_FILE}" +if run_bootstrap >/dev/null 2>"${WORK_DIR}/config-error"; then + fail "bootstrap accepted unknown credential fields in environment JSON" +fi +AWS_CALL_COUNT_AFTER="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +[[ "${AWS_CALL_COUNT_AFTER}" == "${AWS_CALL_COUNT_BEFORE}" ]] || + fail "bootstrap contacted AWS before rejecting malformed configuration" +[[ "$(<"${WORK_DIR}/config-error")" == *"environment keys are invalid"* ]] || + fail "configuration parsing failure was not explicit" + +write_config +if FAKE_AWS_ACCOUNT_ID="222222222222" run_bootstrap >/dev/null 2>"${WORK_DIR}/identity-error"; then + fail "bootstrap accepted an unexpected AWS identity" +fi +[[ "$(<"${WORK_DIR}/identity-error")" == *"AWS identity mismatch"* ]] || + fail "bootstrap identity mismatch was not explicit" + +AWS_CALL_COUNT_BEFORE="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +if AWS_REGION="us-west-2" run_bootstrap >/dev/null 2>"${WORK_DIR}/region-error"; then + fail "bootstrap accepted an AWS Region that conflicts with environment configuration" +fi +AWS_CALL_COUNT_AFTER="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +[[ "${AWS_CALL_COUNT_AFTER}" == "${AWS_CALL_COUNT_BEFORE}" ]] || + fail "bootstrap called STS before rejecting the configured Region mismatch" +[[ "$(<"${WORK_DIR}/region-error")" == *"AWS_REGION mismatch"* ]] || + fail "bootstrap Region mismatch was not explicit" + +AWS_CALL_COUNT_BEFORE="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +if FAKE_GIT_DIRTY=true run_bootstrap >/dev/null 2>"${WORK_DIR}/dirty-error"; then + fail "bootstrap accepted a dirty source checkout before image publication" +fi +AWS_CALL_COUNT_AFTER="$(wc -l <"${AWS_CALLS}" | tr -d ' ')" +[[ "${AWS_CALL_COUNT_AFTER}" == "${AWS_CALL_COUNT_BEFORE}" ]] || + fail "bootstrap contacted AWS before rejecting a dirty checkout" +[[ "$(<"${WORK_DIR}/dirty-error")" == *"source checkout is not clean"* ]] || + fail "dirty checkout failure was not explicit" + +write_config "../unsafe" +if run_bootstrap >/dev/null 2>"${WORK_DIR}/unsafe-path-error"; then + fail "bootstrap accepted an environment name that could escape artifact/backend paths" +fi +[[ "$(<"${WORK_DIR}/unsafe-path-error")" == *"environment.name must be"* ]] || + fail "unsafe environment path failure was not explicit" +write_config + +ORPHAN_ARTIFACT_ROOT="${WORK_DIR}/orphan-artifacts" +ORPHAN_DIR="${ORPHAN_ARTIFACT_ROOT}/${ENVIRONMENT_NAME}/${ACCOUNT_ID}/${REGION}" +mkdir -p "${ORPHAN_DIR}" +printf '%s\n' 'orphan backend' >"${ORPHAN_DIR}/backend.hcl" +chmod 600 "${ORPHAN_DIR}/backend.hcl" +if run_bootstrap "${ORPHAN_ARTIFACT_ROOT}" >/dev/null 2>"${WORK_DIR}/orphan-error"; then + fail "bootstrap accepted a backend file without local bootstrap state" +fi +[[ "$(<"${WORK_DIR}/orphan-error")" == *"backend configuration exists without its bootstrap state"* ]] || + fail "orphan backend collision failure was not explicit" + +LOST_STATE_ARTIFACT_ROOT="${WORK_DIR}/lost-state-artifacts" +if FAKE_TABLE_EXISTS=true run_bootstrap "${LOST_STATE_ARTIFACT_ROOT}" \ + >/dev/null 2>"${WORK_DIR}/lost-state-error"; then + fail "bootstrap accepted existing remote state resources without matching local bootstrap state" +fi +[[ "$(<"${WORK_DIR}/lost-state-error")" == *"lock table"*"already exists"* ]] || + fail "lost bootstrap-state collision failure was not explicit" + +printf '%s' "${FIRST_BACKEND_CONTENT}" >"${BACKEND_CONFIG}" +chmod 600 "${BACKEND_CONFIG}" +ORIGINAL_IMAGES="$(<"${IMAGE_INPUTS}")" +if FAKE_BUILD_OUTPUT=invalid run_bootstrap >/dev/null 2>"${WORK_DIR}/invalid-images-error"; then + fail "bootstrap accepted malformed generated image inputs" +fi +[[ "$(<"${IMAGE_INPUTS}")" == "${ORIGINAL_IMAGES}" ]] || + fail "failed image generation replaced the previous atomic image input" + +echo "bootstrap helper tests passed" diff --git a/terraform/aws/scripts/tests/build-images-test.sh b/terraform/aws/scripts/tests/build-images-test.sh index e757c87..0360f8c 100755 --- a/terraform/aws/scripts/tests/build-images-test.sh +++ b/terraform/aws/scripts/tests/build-images-test.sh @@ -5,8 +5,9 @@ TEST_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" SCRIPT_DIR="$(CDPATH= cd -- "${TEST_DIR}/.." && pwd -P)" AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd -P)" BUILD_SCRIPT="${SCRIPT_DIR}/build-images.sh" -TF_ROOT="${AWS_DIR}/examples/created-vpc" +TF_ROOT="${AWS_DIR}/deployment" APPLICATION_ROOT="${AWS_DIR}/application" +DEPLOYMENT_ROOT="${AWS_DIR}/deployment" EXISTING_ROOT="${AWS_DIR}/examples/existing-vpc" CENTRAL_ROOT="${AWS_DIR}/examples/multi-account-central" MEMBER_ROOT="${AWS_DIR}/examples/member-account" @@ -58,6 +59,12 @@ case "${output_name}" in "operator_installed":false, "dispatch_enabled":false, "automatic_inventory_enabled":false, + "periodic_snapshots_enabled":false, + "periodic_coverage_enabled":false, + "signal_hints_enabled":false, + "processor_reconciliation_enabled":false, + "finding_export_enabled":false, + "managed_canary_enabled":false, "canary_mode":false }' ;; @@ -68,6 +75,12 @@ case "${output_name}" in "operator_installed":true, "dispatch_enabled":true, "automatic_inventory_enabled":true, + "periodic_snapshots_enabled":true, + "periodic_coverage_enabled":true, + "signal_hints_enabled":true, + "processor_reconciliation_enabled":true, + "finding_export_enabled":false, + "managed_canary_enabled":false, "canary_mode":false }' ;; @@ -78,6 +91,12 @@ case "${output_name}" in "operator_installed":true, "dispatch_enabled":true, "automatic_inventory_enabled":false, + "periodic_snapshots_enabled":false, + "periodic_coverage_enabled":false, + "signal_hints_enabled":false, + "processor_reconciliation_enabled":false, + "finding_export_enabled":false, + "managed_canary_enabled":true, "canary_mode":true }' ;; @@ -88,6 +107,12 @@ case "${output_name}" in "operator_installed":true, "dispatch_enabled":false, "automatic_inventory_enabled":false, + "periodic_snapshots_enabled":false, + "periodic_coverage_enabled":false, + "signal_hints_enabled":false, + "processor_reconciliation_enabled":false, + "finding_export_enabled":false, + "managed_canary_enabled":true, "canary_mode":true }' ;; @@ -98,6 +123,12 @@ case "${output_name}" in "operator_installed":false, "dispatch_enabled":true, "automatic_inventory_enabled":false, + "periodic_snapshots_enabled":false, + "periodic_coverage_enabled":false, + "signal_hints_enabled":false, + "processor_reconciliation_enabled":false, + "finding_export_enabled":false, + "managed_canary_enabled":false, "canary_mode":true }' ;; @@ -108,6 +139,12 @@ case "${output_name}" in "operator_installed":false, "dispatch_enabled":false, "automatic_inventory_enabled":false, + "periodic_snapshots_enabled":false, + "periodic_coverage_enabled":false, + "signal_hints_enabled":false, + "processor_reconciliation_enabled":false, + "finding_export_enabled":false, + "managed_canary_enabled":true, "canary_mode":true }' ;; @@ -176,7 +213,7 @@ done [[ "${DRY_RUN_LOG}" == *"no AWS identity call, registry login, build, push, or digest output"* ]] || fail "dry run did not state its side-effect boundary" -for central_root in "${APPLICATION_ROOT}" "${EXISTING_ROOT}" "${CENTRAL_ROOT}"; do +for central_root in "${APPLICATION_ROOT}" "${DEPLOYMENT_ROOT}" "${EXISTING_ROOT}" "${CENTRAL_ROOT}"; do PATH="${FAKE_BIN}:${PATH}" \ PORTSCANNER_ALLOW_DIRTY=true \ "${BUILD_SCRIPT}" --dry-run "${central_root}" arm64 >"${STDOUT_FILE}" 2>"${STDERR_FILE}" || @@ -241,7 +278,7 @@ MEMBER_ERROR="$(<"${STDERR_FILE}")" fail "member-account rejection was not explicit" if PATH="${FAKE_BIN}:${PATH}" PORTSCANNER_ALLOW_DIRTY=true \ - "${BUILD_SCRIPT}" --dry-run "${TF_ROOT}/../created-vpc" arm64 >/dev/null 2>&1; then + "${BUILD_SCRIPT}" --dry-run "${TF_ROOT}/../deployment" arm64 >/dev/null 2>&1; then fail "root containing a traversal segment was accepted" fi diff --git a/terraform/aws/scripts/tests/canary-helpers-test.sh b/terraform/aws/scripts/tests/canary-helpers-test.sh new file mode 100755 index 0000000..df0de9b --- /dev/null +++ b/terraform/aws/scripts/tests/canary-helpers-test.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../../../.." && pwd -P)" +EVALUATE="${ROOT}/terraform/aws/scripts/evaluate-canary.sh" +RETIRE="${ROOT}/terraform/aws/scripts/retire-canary.sh" +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-canary-helper-tests.XXXXXX")" +cleanup() { + rm -rf -- "${WORK_DIR}" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +write_config() { + local enabled="$1" + cat >"${WORK_DIR}/environment.auto.tfvars.json" <"${WORK_DIR}/bin/terraform" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +command_name="${2:-}" +case "${command_name}" in + output) + format="${3:-}" + name="${4:-}" + if [[ "${format}" == "-json" ]]; then + case "${name}" in + deployment_state) + if [[ -f "${FAKE_APPLIED:-/nonexistent}" ]]; then + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":false,"canary_mode":false}' + elif [[ "${FAKE_RETIRE_MODE:-false}" == "true" ]]; then + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":true}' + else + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":true,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":true}' + fi + ;; + managed_canary) + if [[ -f "${FAKE_APPLIED:-/nonexistent}" ]]; then + printf '%s\n' 'null' + else + printf '%s\n' '{"account_id":"123456789012","region":"us-east-1","network_interface_id":"eni-0123456789abcdef0","instance_id":"i-0123456789abcdef0","instance_state":"running","public_ip":"203.0.113.20","public_cidr":"203.0.113.20/32","listener_port":18080,"inventory_tag_key":"service","inventory_tag_value":"test-eval-managed-canary"}' + fi + ;; + managed_canary_snapshot_invocation) + printf '%s\n' '{"function_name":"test-snapshot","function_arn":"arn:aws:lambda:us-east-1:123456789012:function:test-snapshot","payload":{"operation":"managed-canary"}}' + ;; + managed_canary_status_invocation) + printf '%s\n' '{"function_name":"test-processor","function_arn":"arn:aws:lambda:us-east-1:123456789012:function:test-processor","payload":{"operation":"managed-canary-status"}}' + ;; + queue_urls) + printf '%s\n' '{"priority":"https://sqs.us-east-1.amazonaws.com/123456789012/priority","result":"https://sqs.us-east-1.amazonaws.com/123456789012/result"}' + ;; + *) + exit 90 + ;; + esac + elif [[ "${format}" == "-raw" ]]; then + case "${name}" in + retirement_configuration_fingerprint) + printf '%064d\n' 0 + ;; + eks_cluster_name) + printf '%s\n' 'test-cluster' + ;; + operator_namespace) + printf '%s\n' 'portscanner-system' + ;; + *) + exit 91 + ;; + esac + else + exit 92 + fi + ;; + init) + ;; + plan) + for argument in "$@"; do + case "${argument}" in + -out=*) + : >"${argument#-out=}" + ;; + esac + done + ;; + show) + if [[ "${3:-}" == "-json" ]]; then + fingerprint="$(printf '%064d' 0)" + [[ "${FAKE_FINGERPRINT_MISMATCH:-false}" != "true" ]] || fingerprint="$(printf '%064d' 1)" + cat <"${FAKE_APPLIED:?}" + ;; + *) + echo "unexpected terraform invocation: $*" >&2 + exit 93 + ;; +esac +EOF +chmod 755 "${WORK_DIR}/bin/terraform" + +cat >"${WORK_DIR}/bin/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${1:-} ${2:-}" in + "sts get-caller-identity") + printf '%s\n' '123456789012' + ;; + "lambda invoke") + function_name="" + previous="" + for argument in "$@"; do + if [[ "${previous}" == "--function-name" ]]; then + function_name="${argument}" + fi + previous="${argument}" + done + response_file="${!#}" + if [[ "${function_name}" == "test-snapshot" ]]; then + printf '%s\n' '[{"completion":"complete","targets":1,"pages":1,"revalidated":0,"added":1,"changed":0,"noop":0,"removed":0,"race":0}]' >"${response_file}" + elif [[ "${function_name}" == "test-processor" ]]; then + attempt_count=1 + [[ "${FAKE_DUPLICATE_STATUS:-false}" != "true" ]] || attempt_count=2 + printf '{"operation":"managed-canary-status","status":"ready","ready":true,"target_count":1,"event_count":1,"attempt_count":%s,"complete_coverage_count":1,"open_exposure_count":1,"low_finding_count":1,"unexpected_high_finding_count":0}\n' "${attempt_count}" >"${response_file}" + else + exit 80 + fi + printf '%s\n' '{"StatusCode":200}' + ;; + "sqs get-queue-attributes") + printf '%s\n' '{"ApproximateNumberOfMessages":"0","ApproximateNumberOfMessagesNotVisible":"0","ApproximateNumberOfMessagesDelayed":"0"}' + ;; + "eks update-kubeconfig") + previous="" + for argument in "$@"; do + if [[ "${previous}" == "--kubeconfig" ]]; then + printf '%s\n' 'apiVersion: v1' >"${argument}" + fi + previous="${argument}" + done + ;; + *) + echo "unexpected aws invocation: $*" >&2 + exit 81 + ;; +esac +EOF +chmod 755 "${WORK_DIR}/bin/aws" + +cat >"${WORK_DIR}/bin/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ " $* " == *" get jobs "* ]]; then + printf '%s\n' '{"items":[{"status":{"active":0,"conditions":[{"type":"Complete","status":"True"}]}}]}' +elif [[ " $* " == *" get scanners.scanning.portscanner.io "* ]]; then + printf '%s\n' '{"items":[{"status":{"outcome":"Succeeded"}}]}' +else + exit 70 +fi +EOF +chmod 755 "${WORK_DIR}/bin/kubectl" + +write_config true +PATH="${WORK_DIR}/bin:${PATH}" \ +PORTSCANNER_CANARY_TIMEOUT_SECONDS=60 \ +PORTSCANNER_CANARY_POLL_SECONDS=0 \ +PORTSCANNER_CANARY_STABILITY_POLLS=1 \ + "${EVALUATE}" \ + --terraform-root "${WORK_DIR}/root" \ + --environment-config "${WORK_DIR}/environment.auto.tfvars.json" \ + --backend-config "${WORK_DIR}/backend.hcl" \ + --image-inputs "${WORK_DIR}/images.tfvars" \ + >"${WORK_DIR}/evaluate.out" +[[ "$(<"${WORK_DIR}/evaluate.out")" == *"Managed canary verified"* ]] || + fail "successful evaluation did not report verification" + +if PATH="${WORK_DIR}/bin:${PATH}" \ + FAKE_DUPLICATE_STATUS=true \ + PORTSCANNER_CANARY_TIMEOUT_SECONDS=60 \ + PORTSCANNER_CANARY_POLL_SECONDS=0 \ + PORTSCANNER_CANARY_STABILITY_POLLS=1 \ + "${EVALUATE}" \ + --terraform-root "${WORK_DIR}/root" \ + --environment-config "${WORK_DIR}/environment.auto.tfvars.json" \ + --backend-config "${WORK_DIR}/backend.hcl" \ + --image-inputs "${WORK_DIR}/images.tfvars" \ + >"${WORK_DIR}/duplicate.out" 2>"${WORK_DIR}/duplicate.err"; then + fail "evaluation accepted a duplicate effective attempt" +fi +[[ "$(<"${WORK_DIR}/duplicate.err")" == *"non-idempotent"* ]] || + fail "duplicate evaluation failure was not explicit" + +PATH="${WORK_DIR}/bin:${PATH}" \ +FAKE_RETIRE_MODE=true \ +PORTSCANNER_RETIRE_QUEUE_POLLS=2 \ +PORTSCANNER_RETIRE_QUEUE_POLL_SECONDS=0 \ + "${RETIRE}" \ + --verify-only \ + --terraform-root "${WORK_DIR}/root" \ + --environment-config "${WORK_DIR}/environment.auto.tfvars.json" \ + --backend-config "${WORK_DIR}/backend.hcl" \ + --image-inputs "${WORK_DIR}/images.tfvars" \ + >"${WORK_DIR}/verify-only.out" +[[ "$(<"${WORK_DIR}/verify-only.out")" == *"Retirement readiness verified"* ]] || + fail "verify-only retirement readiness did not complete" + +write_config false +FAKE_APPLIED="${WORK_DIR}/applied" +export FAKE_APPLIED +PATH="${WORK_DIR}/bin:${PATH}" \ +FAKE_RETIRE_MODE=true \ +PORTSCANNER_AUTO_APPROVE=true \ +PORTSCANNER_RETIRE_QUEUE_POLLS=2 \ +PORTSCANNER_RETIRE_QUEUE_POLL_SECONDS=0 \ + "${RETIRE}" \ + --terraform-root "${WORK_DIR}/root" \ + --environment-config "${WORK_DIR}/environment.auto.tfvars.json" \ + --backend-config "${WORK_DIR}/backend.hcl" \ + --image-inputs "${WORK_DIR}/images.tfvars" \ + >"${WORK_DIR}/retire.out" +[[ "$(<"${WORK_DIR}/retire.out")" == *"Managed canary retired"* ]] || + fail "successful retirement did not report completion" + +rm -f "${FAKE_APPLIED}" +if PATH="${WORK_DIR}/bin:${PATH}" \ + FAKE_RETIRE_MODE=true \ + FAKE_FINGERPRINT_MISMATCH=true \ + PORTSCANNER_AUTO_APPROVE=true \ + PORTSCANNER_RETIRE_QUEUE_POLLS=2 \ + PORTSCANNER_RETIRE_QUEUE_POLL_SECONDS=0 \ + "${RETIRE}" \ + --terraform-root "${WORK_DIR}/root" \ + --environment-config "${WORK_DIR}/environment.auto.tfvars.json" \ + --backend-config "${WORK_DIR}/backend.hcl" \ + --image-inputs "${WORK_DIR}/images.tfvars" \ + >"${WORK_DIR}/fingerprint.out" 2>"${WORK_DIR}/fingerprint.err"; then + fail "retirement accepted an unrelated environment change" +fi +[[ "$(<"${WORK_DIR}/fingerprint.err")" == *"other than managed_canary.enabled"* ]] || + fail "retirement fingerprint failure was not explicit" + +echo "canary helper tests passed" diff --git a/terraform/aws/scripts/tests/deploy-test.sh b/terraform/aws/scripts/tests/deploy-test.sh index f7a9f7f..80395a7 100755 --- a/terraform/aws/scripts/tests/deploy-test.sh +++ b/terraform/aws/scripts/tests/deploy-test.sh @@ -5,7 +5,7 @@ TEST_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" SCRIPT_DIR="$(CDPATH= cd -- "${TEST_DIR}/.." && pwd -P)" AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd -P)" DEPLOY_SCRIPT="${SCRIPT_DIR}/deploy.sh" -TF_ROOT="${AWS_DIR}/examples/created-vpc" +TF_ROOT="${AWS_DIR}/deployment" fail() { echo "FAIL: $*" >&2 @@ -70,13 +70,30 @@ case "${operation}" in output) [[ "${3:-}" == "-json" && "${4:-}" == "deployment_state" ]] || exit 95 if [[ "${FAKE_TF_DEPLOYMENT_STATE:-canary}" == "migrated" ]]; then - printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"canary_mode":false}' + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":false,"canary_mode":false}' else - printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":true,"automatic_inventory_enabled":false,"canary_mode":true}' + applied_stage="" + [[ ! -f "${TF_CALLS:?}.stage" ]] || applied_stage="$(<"${TF_CALLS}.stage")" + case "${applied_stage}" in + pause) + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":false,"canary_mode":false}' + ;; + pause-canary) + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":true}' + ;; + *) + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":true,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":true}' + ;; + esac fi ;; apply) [[ -f "${3:-}" ]] || exit 93 + case "${3:-}" in + *pause-canary.tfplan) printf '%s\n' pause-canary >"${TF_CALLS:?}.stage" ;; + *pause.tfplan) printf '%s\n' pause >"${TF_CALLS:?}.stage" ;; + *canary.tfplan) printf '%s\n' canary >"${TF_CALLS:?}.stage" ;; + esac ;; *) echo "unexpected terraform invocation: $*" >&2 @@ -254,4 +271,409 @@ if PATH="${FAKE_BIN}:${PATH}" \ fail "unexpected AWS account was accepted" fi +CANONICAL_CONFIG="${WORK_DIR}/environment.auto.tfvars.json" +CANONICAL_ARTIFACT_ROOT="${WORK_DIR}/artifacts" +CANONICAL_STATE="${WORK_DIR}/canonical-state" +CANONICAL_APPLIES="${WORK_DIR}/canonical-applies" +CANONICAL_ENVIRONMENT="test-eval" +CANONICAL_ACCOUNT="123456789012" +CANONICAL_REGION="us-east-1" +CANONICAL_ARTIFACT_DIR="${CANONICAL_ARTIFACT_ROOT}/${CANONICAL_ENVIRONMENT}/${CANONICAL_ACCOUNT}/${CANONICAL_REGION}" +CANONICAL_BACKEND="${CANONICAL_ARTIFACT_DIR}/backend.hcl" +CANONICAL_IMAGES="${CANONICAL_ARTIFACT_DIR}/images.tfvars" +mkdir -p "${CANONICAL_ARTIFACT_DIR}" + +write_canonical_config() { + local disposable="$1" + local destroy_data="$2" + local account_id="${3:-${CANONICAL_ACCOUNT}}" + local installer_arn="arn:" + installer_arn+="aws:iam::${account_id}:role/test-installer" + cat >"${CANONICAL_CONFIG}" <"${CANONICAL_BACKEND}" <"${FAKE_BIN}/terraform" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >>"${TF_CALLS:?}" +operation="${2:-}" +state="$(<"${CANONICAL_STATE:?}")" +case "${operation}" in + init) + ;; + state) + [[ "${3:-}" == "list" ]] || exit 91 + case "${state}" in + foundation) + printf '%s\n' "module.portscanner.module.repositories.aws_ecr_repository.this[\"inventory\"]" + ;; + runtime) + printf '%s\n' \ + "module.portscanner.module.repositories.aws_ecr_repository.this[\"inventory\"]" \ + "module.portscanner.module.functions.aws_lambda_function.this[\"inventory\"]" + ;; + ready | active) + printf '%s\n' \ + "module.portscanner.module.repositories.aws_ecr_repository.this[\"inventory\"]" \ + "module.portscanner.module.functions.aws_lambda_function.this[\"inventory\"]" \ + "module.portscanner.module.functions.aws_lambda_invocation.migration[0]" \ + "module.portscanner.module.eks.helm_release.operator[0]" + ;; + destroyed) + ;; + *) + exit 92 + ;; + esac + ;; + output) + [[ "${3:-}" == "-json" && "${4:-}" == "deployment_state" ]] || exit 93 + case "${state}" in + foundation) + printf '%s\n' '{"runtime_created":false,"migration_run":false,"operator_installed":false,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":false}' + ;; + runtime) + printf '%s\n' '{"runtime_created":true,"migration_run":false,"operator_installed":false,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":false}' + ;; + ready) + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":false,"automatic_inventory_enabled":false,"periodic_snapshots_enabled":false,"periodic_coverage_enabled":false,"signal_hints_enabled":false,"processor_reconciliation_enabled":false,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":false}' + ;; + active) + printf '%s\n' '{"runtime_created":true,"migration_run":true,"operator_installed":true,"dispatch_enabled":true,"automatic_inventory_enabled":true,"periodic_snapshots_enabled":true,"periodic_coverage_enabled":true,"signal_hints_enabled":true,"processor_reconciliation_enabled":true,"finding_export_enabled":false,"managed_canary_enabled":true,"canary_mode":false}' + ;; + *) + exit 94 + ;; + esac + ;; + plan) + plan_file="" + for argument in "$@"; do + if [[ "${argument}" == -out=* ]]; then + plan_file="${argument#-out=}" + fi + done + [[ -n "${plan_file}" ]] || exit 95 + : >"${plan_file}" + ;; + show) + if [[ "${3:-}" == "-json" ]]; then + printf '%s\n' '{"resource_changes":[{"mode":"managed","type":"aws_lambda_event_source_mapping","change":{"actions":["update"],"before":{"enabled":true},"after":{"enabled":false}}}]}' + else + printf '%s\n' "saved destroy plan" + fi + ;; + apply) + plan_file="${3:-}" + [[ -f "${plan_file}" ]] || exit 96 + printf '%s\n' "${plan_file}" >>"${CANONICAL_APPLIES:?}" + case "${plan_file}" in + *-runtime.tfplan) printf '%s\n' runtime >"${CANONICAL_STATE}" ;; + *-migrate.tfplan | *-pause.tfplan | *-pause-canary.tfplan) + printf '%s\n' ready >"${CANONICAL_STATE}" + ;; + *-destroy.tfplan) printf '%s\n' destroyed >"${CANONICAL_STATE}" ;; + esac + ;; + *) + echo "unexpected terraform invocation: $*" >&2 + exit 97 + ;; +esac +EOF + +cat >"${FAKE_BIN}/aws" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "sts" && "${2:-}" == "get-caller-identity" ]]; then + printf '%s\n' "${FAKE_AWS_ACCOUNT_ID:-123456789012}" + exit 0 +fi +exit 98 +EOF +chmod +x "${FAKE_BIN}/terraform" "${FAKE_BIN}/aws" + +write_canonical_config true true +write_backend +: >"${CANONICAL_IMAGES}" +chmod 600 "${CANONICAL_IMAGES}" +printf '%s\n' foundation >"${CANONICAL_STATE}" +: >"${CANONICAL_APPLIES}" +: >"${TF_CALLS}" + +PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null + +[[ "$(<"${CANONICAL_STATE}")" == "ready" ]] || + fail "ready orchestration did not reach the paused ready state" +CANONICAL_CALLS="$(<"${TF_CALLS}")" +[[ "${CANONICAL_CALLS}" == *"-chdir=${AWS_DIR}/deployment init"* ]] || + fail "canonical invocation did not default to the deployment root" +[[ "${CANONICAL_CALLS}" == *"-backend-config=${CANONICAL_BACKEND}"* ]] || + fail "canonical invocation did not use the derived backend" +[[ "${CANONICAL_CALLS}" == *"-var-file=${CANONICAL_CONFIG}"* ]] || + fail "canonical invocation did not use the one environment configuration" +[[ "${CANONICAL_CALLS}" == *"-var-file=${CANONICAL_IMAGES}"* ]] || + fail "ready did not use generated image inputs" +[[ "${CANONICAL_CALLS}" == *"-runtime.tfplan"* && "${CANONICAL_CALLS}" == *"-migrate.tfplan"* ]] || + fail "ready did not orchestrate runtime then migrate" +[[ "${CANONICAL_CALLS}" != *" -target"* ]] || + fail "ready used a targeted Terraform operation" + +: >"${TF_CALLS}" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_CANARY_HELPER="${WORK_DIR}/missing-canary-helper" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" evaluate >"${WORK_DIR}/evaluate-output" 2>&1; then + fail "evaluate pretended to succeed without managed-canary/status integration" +fi +EVALUATE_OUTPUT="$(<"${WORK_DIR}/evaluate-output")" +[[ "${EVALUATE_OUTPUT}" == *"managed-canary trigger/status integration is not installed"* ]] || + fail "evaluate missing-integration failure was not precise" +[[ "$(<"${CANONICAL_STATE}")" == "ready" ]] || + fail "evaluate without runtime integration did not remain paused" +[[ "$(<"${TF_CALLS}")" != *"-canary.tfplan"* ]] || + fail "evaluate entered canary before checking the future helper" + +CANARY_HELPER_CALLS="${WORK_DIR}/canary-helper-calls" +cat >"${WORK_DIR}/canary-helper" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >"${CANARY_HELPER_CALLS:?}" +EOF +chmod 755 "${WORK_DIR}/canary-helper" +: >"${TF_CALLS}" +PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + CANARY_HELPER_CALLS="${CANARY_HELPER_CALLS}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_CANARY_HELPER="${WORK_DIR}/canary-helper" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" evaluate >/dev/null +[[ "$(<"${TF_CALLS}")" == *"-canary.tfplan"* && + "$(<"${TF_CALLS}")" == *"-pause-canary.tfplan"* ]] || + fail "evaluate did not bracket the canary helper with guarded stages" +[[ "$(<"${CANARY_HELPER_CALLS}")" == *"--terraform-root ${AWS_DIR}/deployment"* && + "$(<"${CANARY_HELPER_CALLS}")" == *"--environment-config ${CANONICAL_CONFIG}"* ]] || + fail "evaluate did not pass canonical inputs to the canary helper" + +printf '%s\n' active >"${CANONICAL_STATE}" +: >"${TF_CALLS}" +PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null +[[ "$(<"${CANONICAL_STATE}")" == "ready" ]] || + fail "ready did not safely pause an already active deployment" +[[ "$(<"${TF_CALLS}")" == *"-pause.tfplan"* ]] || + fail "ready did not use the guarded pause stage for active state" + +write_backend "unsafe/../collision.tfstate" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null 2>&1; then + fail "canonical deploy accepted a colliding backend key" +fi +write_backend + +printf '%s\n' 'profile = "must-not-be-embedded"' >>"${CANONICAL_BACKEND}" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null 2>&1; then + fail "canonical deploy accepted credentials or a profile in backend configuration" +fi +write_backend + +chmod 644 "${CANONICAL_BACKEND}" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null 2>&1; then + fail "canonical deploy accepted a non-private backend file" +fi +chmod 600 "${CANONICAL_BACKEND}" + +printf '%s\n' foundation >"${CANONICAL_STATE}" +chmod 644 "${CANONICAL_IMAGES}" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null 2>&1; then + fail "ready accepted non-private generated image inputs" +fi +chmod 600 "${CANONICAL_IMAGES}" + +write_canonical_config true true "222222222222" +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" ready >/dev/null 2>&1; then + fail "canonical deploy accepted an AWS identity mismatch" +fi +write_canonical_config true true + +write_canonical_config false false +if PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" destroy >/dev/null 2>&1; then + fail "destroy accepted an environment without disposable retention acknowledgement" +fi + +write_canonical_config true true +printf '%s\n' foundation >"${CANONICAL_STATE}" +: >"${CANONICAL_APPLIES}" +if printf '%s\n' wrong-confirmation | PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" destroy >/dev/null 2>&1; then + fail "destroy accepted an inexact confirmation" +fi +[[ ! -s "${CANONICAL_APPLIES}" ]] || + fail "destroy applied after an inexact confirmation" + +printf '%s\n' foundation >"${CANONICAL_STATE}" +: >"${CANONICAL_APPLIES}" +: >"${TF_CALLS}" +printf '%s\n' "destroy ${CANONICAL_ENVIRONMENT}" | PATH="${FAKE_BIN}:${PATH}" \ + TF_CALLS="${TF_CALLS}" \ + CANONICAL_STATE="${CANONICAL_STATE}" \ + CANONICAL_APPLIES="${CANONICAL_APPLIES}" \ + PORTSCANNER_CONFIG_FILE="${CANONICAL_CONFIG}" \ + PORTSCANNER_ARTIFACT_ROOT="${CANONICAL_ARTIFACT_ROOT}" \ + PORTSCANNER_AUTO_APPROVE=true \ + "${DEPLOY_SCRIPT}" destroy >/dev/null +[[ "$(<"${CANONICAL_STATE}")" == "destroyed" ]] || + fail "exactly confirmed canonical destroy did not apply" +DESTROY_CALLS="$(<"${TF_CALLS}")" +[[ "${DESTROY_CALLS}" == *"plan -destroy -input=false"* ]] || + fail "destroy did not save an explicit destroy plan" +[[ "${DESTROY_CALLS}" == *"show "*"destroy.tfplan"* ]] || + fail "destroy did not show the exact saved plan before applying" +[[ "${DESTROY_CALLS}" != *"/state-bootstrap"* ]] || + fail "destroy attempted to operate on state-bootstrap" + echo "deploy helper tests passed" diff --git a/terraform/aws/scripts/tests/emergency-pause-test.sh b/terraform/aws/scripts/tests/emergency-pause-test.sh index 7c1263c..07ac1e5 100755 --- a/terraform/aws/scripts/tests/emergency-pause-test.sh +++ b/terraform/aws/scripts/tests/emergency-pause-test.sh @@ -5,7 +5,7 @@ TEST_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" SCRIPT_DIR="$(CDPATH= cd -- "${TEST_DIR}/.." && pwd -P)" AWS_DIR="$(CDPATH= cd -- "${SCRIPT_DIR}/.." && pwd -P)" PAUSE_SCRIPT="${SCRIPT_DIR}/emergency-pause.sh" -TF_ROOT="${AWS_DIR}/examples/created-vpc" +TF_ROOT="${AWS_DIR}/deployment" fail() { echo "FAIL: $*" >&2 diff --git a/terraform/aws/scripts/validate.sh b/terraform/aws/scripts/validate.sh index e21df20..bc22d8a 100755 --- a/terraform/aws/scripts/validate.sh +++ b/terraform/aws/scripts/validate.sh @@ -24,8 +24,8 @@ mkdir -p "${TF_PLUGIN_CACHE_DIR}" ROOTS=( "${AWS_DIR}/state-bootstrap" "${AWS_DIR}/application" + "${AWS_DIR}/deployment" "${AWS_DIR}/member-account" - "${AWS_DIR}/examples/created-vpc" "${AWS_DIR}/examples/existing-vpc" "${AWS_DIR}/examples/multi-account-central" "${AWS_DIR}/examples/member-account" diff --git a/terraform/aws/state-bootstrap/README.md b/terraform/aws/state-bootstrap/README.md deleted file mode 100644 index 652380f..0000000 --- a/terraform/aws/state-bootstrap/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# State bootstrap - -Apply this root once with ordinary AWS environment credentials; no profile is embedded: - -```sh -terraform init -terraform apply \ - -var='aws_region=us-east-1' \ - -var='expected_deployment_account_id=REPLACE_WITH_12_DIGIT_ACCOUNT_ID' -terraform output backend_configuration -``` - -The AWS provider refuses credentials from any other account. - -Store the output values in an out-of-band backend configuration file, add a unique `key` such as `environments/example-central.tfstate` for every root, and initialize with `terraform init -backend-config=...`. Do not put credentials in that file or reuse a key between roots. - -The generated S3 bucket has public access blocked, bucket-owner-enforced ownership, TLS-only policy, encryption, versioning, and noncurrent-version retention. The DynamoDB lock table uses on-demand capacity, encryption, and PITR. - -Destruction warning: both resources have `prevent_destroy`, and the bucket does not force-delete. Removing those controls risks losing the source of truth for every managed resource and must be a separately reviewed retirement operation. Retained versions and the lock table incur small ongoing costs. diff --git a/tools/sanitize-policy.toml b/tools/sanitize-policy.toml index 347a8cd..f885e74 100644 --- a/tools/sanitize-policy.toml +++ b/tools/sanitize-policy.toml @@ -7,7 +7,7 @@ max_binary_bytes = 524288 # Binary files are denied by default. An intentionally distributable binary must be # reviewed and listed as an exact repository-relative path plus lowercase SHA-256: -# { path = "docs/example.png", sha256 = "<64 lowercase hex characters>" } +# { path = "examples/approved-diagram.png", sha256 = "<64 lowercase hex characters>" } binary_allowlist = [] # These are the only account-shaped fixtures accepted in fixture-scoped paths. @@ -20,13 +20,13 @@ synthetic_fixture_paths = [ "**/tests/**", "operator/**/*_test.go", "operator/config/samples/**", - "docs/**", "examples/**", "terraform/aws/**/*.tftest.hcl", "terraform/aws/examples/**", "tools/sanitize-policy.toml", ] synthetic_aws_resource_ids = [ + "ami-0123456789abcdef0", "eipalloc-cccccccc", "eipassoc-dddddddd", "eni-00000000", @@ -95,6 +95,8 @@ synthetic_aws_arns = [ "arn:aws:iam::123456789012:role/test-signals", "arn:aws:iam::123456789012:role/test-snapshot", "arn:aws:iam::123456789012:role/test-target-projector", + "arn:aws:s3:::test-events", + "arn:aws:s3:::test-results", "arn:aws:sqs:us-east-1:123456789012:test-coverage", "arn:aws:sqs:us-east-1:123456789012:test-priority", "arn:aws:sqs:us-east-1:123456789012:test-result", @@ -102,6 +104,13 @@ synthetic_aws_arns = [ "arn:aws:sqs:us-east-1:123456789012:test-signal-dlq", "arn:aws:sqs:us-east-1:123456789012:test-target-event", "arn:aws:dynamodb:us-east-1:123456789012:table/test-inventory/stream/2026-08-07T00:00:00.000", + "arn:aws:dynamodb:us-east-1:123456789012:table/test-inventory", + "arn:aws:dynamodb:us-east-1:123456789012:table/test-dispatch", + "arn:aws:dynamodb:us-east-1:123456789012:table/test-inventory/stream/test", + "arn:aws:dynamodb:us-east-1:123456789012:table/test-eval-state-terraform-locks", + "arn:aws:eks:us-east-1:123456789012:cluster/test", + "arn:aws:lambda:us-east-1:123456789012:function:test-snapshot", + "arn:aws:lambda:us-east-1:123456789012:function:test-processor", "arn:aws:secretsmanager:us-east-1:123456789012:secret:test-application", "arn:aws:secretsmanager:us-east-1:123456789012:secret:test-master", ] diff --git a/tools/test_migrator_package.sh b/tools/test_migrator_package.sh index 9a3371c..88de9c9 100644 --- a/tools/test_migrator_package.sh +++ b/tools/test_migrator_package.sh @@ -81,7 +81,7 @@ source_files = {path.name: path.read_bytes() for path in source_entries} packaged_files = {path.name: path.read_bytes() for path in packaged_entries} assert packaged_files == source_files assert migration_set_checksum(packaged) == migration_set_checksum(repository) -assert len(discover_migrations(packaged)) == 5 +assert len(discover_migrations(packaged)) == 1 PY "${WORK_DIR}/venv/bin/act-migrate" --help >/dev/null ) diff --git a/tools/tests/test_release_packaging.py b/tools/tests/test_release_packaging.py index 02200e9..162b1a9 100644 --- a/tools/tests/test_release_packaging.py +++ b/tools/tests/test_release_packaging.py @@ -35,6 +35,28 @@ def requirement_records(content: str) -> list[str]: class ReleasePackagingTests(unittest.TestCase): + def test_release_version_is_consistent(self) -> None: + workspace = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + version = workspace["project"]["version"] + self.assertEqual(version, "1.0.0") + + for component in (*COMPONENTS, "contracts"): + with self.subTest(component=component): + metadata = tomllib.loads( + (ROOT / component / "pyproject.toml").read_text(encoding="utf-8") + )["project"] + self.assertEqual(metadata["version"], version) + + scanner_init = (ROOT / "scanner/nmap/src/portscanner_scanner/__init__.py").read_text( + encoding="utf-8" + ) + self.assertIn(f'__version__ = "{version}"', scanner_init) + + chart = (ROOT / "operator/chart/portscanner/Chart.yaml").read_text(encoding="utf-8") + self.assertIn(f"version: {version}", chart) + self.assertIn(f'appVersion: "{version}"', chart) + self.assertIn(f"## [{version}]", (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")) + def test_component_metadata_packages_the_repository_license(self) -> None: repository_license = (ROOT / "LICENSE").read_bytes() for component in (*COMPONENTS, "contracts"): @@ -47,6 +69,22 @@ def test_component_metadata_packages_the_repository_license(self) -> None: self.assertEqual(metadata["license-files"], ["LICENSE"]) self.assertEqual((component_root / "LICENSE").read_bytes(), repository_license) + def test_operator_image_packages_repository_legal_files(self) -> None: + operator_root = ROOT / "operator" + self.assertEqual((operator_root / "LICENSE").read_bytes(), (ROOT / "LICENSE").read_bytes()) + self.assertEqual( + (operator_root / "THIRD_PARTY_NOTICES.md").read_bytes(), + (ROOT / "THIRD_PARTY_NOTICES.md").read_bytes(), + ) + dockerfile = (operator_root / "Dockerfile").read_text(encoding="utf-8") + self.assertIn('org.opencontainers.image.licenses="MIT"', dockerfile) + self.assertIn("COPY --from=builder /workspace/LICENSE /licenses/LICENSE", dockerfile) + self.assertIn( + "COPY --from=builder /workspace/THIRD_PARTY_NOTICES.md " + "/licenses/THIRD_PARTY_NOTICES.md", + dockerfile, + ) + def test_runtime_exports_are_exact_hash_pins(self) -> None: for component in COMPONENTS: with self.subTest(component=component): @@ -84,7 +122,7 @@ def test_lambda_dockerfiles_enforce_lock_and_license_controls(self) -> None: self.assertIn("--no-build-isolation", dockerfile) self.assertIn("--no-deps", dockerfile) self.assertIn( - "COPY LICENSE NOTICE THIRD_PARTY_NOTICES.md /licenses/", + "COPY LICENSE THIRD_PARTY_NOTICES.md /licenses/", dockerfile, ) self.assertIn("USER 65532:65532", dockerfile) @@ -120,6 +158,8 @@ def test_container_workflow_watches_shared_build_inputs(self) -> None: workflow = (ROOT / ".github/workflows/containers.yml").read_text(encoding="utf-8") for path in ( '".dockerignore"', + '"LICENSE"', + '"THIRD_PARTY_NOTICES.md"', '"contracts/**"', '"db/migrations/**"', '"uv.lock"', @@ -128,6 +168,7 @@ def test_container_workflow_watches_shared_build_inputs(self) -> None: ): with self.subTest(path=path): self.assertIn(path, workflow) + self.assertNotIn(' - "NOTICE"', workflow) if __name__ == "__main__": diff --git a/tools/verify_python_image.py b/tools/verify_python_image.py index d29a84c..6a936d4 100644 --- a/tools/verify_python_image.py +++ b/tools/verify_python_image.py @@ -15,7 +15,7 @@ from typing import Final ROOT: Final = Path(__file__).resolve().parents[1] -LEGAL_FILES: Final = ("LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") +LEGAL_FILES: Final = ("LICENSE", "THIRD_PARTY_NOTICES.md") IMAGE_CHECK: Final = r""" import hashlib import importlib diff --git a/uv.lock b/uv.lock index 6ccd2ac..21560b3 100644 --- a/uv.lock +++ b/uv.lock @@ -183,16 +183,28 @@ sdist = { url = "https://files.pythonhosted.org/packages/34/b2/5334340790a87e0c2 wheels = [ { url = "https://files.pythonhosted.org/packages/84/2a/ac6f4d0f14855c5b88744adaf76526676bfeb14147bb95dd6e6939a3f5a5/awslambdaric-4.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:07c3b547cd332b973722187233a921661d81fc5e8b3cc947a2315ccd4f29b3a5", size = 353376, upload-time = "2026-07-30T09:05:53.102Z" }, { url = "https://files.pythonhosted.org/packages/31/85/579fa0dc67f19e1a84c2685540d9782c954d684202dd4f025ff7c3cf3e58/awslambdaric-4.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:eac7f8e72406c51c34f8800fd6c97e44c3486358c3855af822ca4591526f3de6", size = 355300, upload-time = "2026-07-30T09:05:49.19Z" }, + { url = "https://files.pythonhosted.org/packages/51/4a/fa4a44985e4faa3032b8a426a8559d7d45531f62f19a99cbe43775ab8118/awslambdaric-4.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d11da5921737b3bc509225c3a6e93e0bd48c85735b5552e20d25ffa05615687c", size = 1327443, upload-time = "2026-08-10T17:48:56.654Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cf/16f6813739b921ec483199ec20c299ceab7cc65e5d78acc51352ccb26bd6/awslambdaric-4.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b73e49e6f40a117a99205f2818dbeeb0778f43cba5412b8562e0bbf23a578cc5", size = 1383853, upload-time = "2026-08-10T17:57:15.808Z" }, { url = "https://files.pythonhosted.org/packages/df/9d/32991e9def4742031f78d4536f20b98ab6c2bec8d122be901aa0f4b48082/awslambdaric-4.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce0a40940135547d64d5083705a866f934b6ed6ad8f7855de282b368c8e896d5", size = 353370, upload-time = "2026-07-30T09:05:54.893Z" }, { url = "https://files.pythonhosted.org/packages/2b/0a/14ed9d38570e1471c182162381beab9051caed09b6d1ba1f11749421738c/awslambdaric-4.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0071102d613d877113c4ef9868c1865d235cea0acce9def06d52952c31f9f51", size = 355327, upload-time = "2026-07-30T09:05:50.34Z" }, + { url = "https://files.pythonhosted.org/packages/35/0a/503d0dcc9249c9fe735b657d9bff5b7f9d4071c8e3bfe69c6c19e68921ab/awslambdaric-4.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:da4d4cf0e4fe5fcfa9a6fcb7592b117f4445cd01bad0696b2936fed18fcf3b68", size = 1327457, upload-time = "2026-08-10T17:48:58.255Z" }, + { url = "https://files.pythonhosted.org/packages/9a/59/a9470035ead4126062d1bd721238f9217e5d44d8ddcd11fd4b618b3139d6/awslambdaric-4.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:57f30f54f22d595b74e4ebc2aacf80e9c87fdd886d594e351a68e10981fbc594", size = 1384003, upload-time = "2026-08-10T17:57:17.371Z" }, { url = "https://files.pythonhosted.org/packages/bf/f1/4ecca6725ebd95a2913734b3d09c40a156b29aceeaa0cf7b1e3ec2a501a8/awslambdaric-4.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee64b5fb8d5829c4c176a28977dbb86d2ed566aaec5ae2dc6b158546441b2bc3", size = 353512, upload-time = "2026-07-30T09:05:56.463Z" }, { url = "https://files.pythonhosted.org/packages/a9/7e/695942fe096fc25c1759a29a9f401d5ed77d678b72c95ea65dd422e57313/awslambdaric-4.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a87f9ab88084670dcbdd299a5fa6dd35084b734c19c45644ea1a25dc295d5005", size = 355393, upload-time = "2026-07-30T09:05:52.28Z" }, + { url = "https://files.pythonhosted.org/packages/64/8d/92bf2fbc1b90b9e82d47614371974cd2e74d673aff615e9c0b5e57da69f6/awslambdaric-4.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3a48d9e34b5842bebcf048a1faedb1f78da32b346e1313bc0462dc4efadbee7f", size = 1327640, upload-time = "2026-08-10T17:48:59.557Z" }, + { url = "https://files.pythonhosted.org/packages/82/61/9e3d86b1871e0b620c5c3c290ebbbc57542638db106e8ff20662992d844d/awslambdaric-4.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:adb42b7390508f32df8471f08af0170a9349cd6c12f30e28e21954abcd8aed4c", size = 1384014, upload-time = "2026-08-10T17:57:18.969Z" }, { url = "https://files.pythonhosted.org/packages/76/18/e14ff2cafbbf7983e768ade50e72ef9489f0751e2cc89bb6b990337a3878/awslambdaric-4.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:145f6eecd9de9984e6e25271b2ae941ca4899622dfe34bf0f26bb2fc496ba948", size = 354962, upload-time = "2026-07-30T09:05:58.001Z" }, { url = "https://files.pythonhosted.org/packages/26/45/36102f4c72e61ecaf420d168f2d1585484393c0a628f849b572911257fe9/awslambdaric-4.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4116de07b2828279205b255ff5d389b9440d3f807376ffcaf0492ced6cc99378", size = 356419, upload-time = "2026-07-30T09:05:54.007Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e0/ad612e81d95c90dc892b65ed46dd028e6d7cab8d41919f8a3f54854ce59b/awslambdaric-4.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1952feb5f3e7a2d5dea0a18481c1fe5363fcfde0e129b57ce3c947f738b9677e", size = 1329488, upload-time = "2026-08-10T17:49:00.929Z" }, + { url = "https://files.pythonhosted.org/packages/83/82/fbf42e741d827c7830f7b99f071e00a4a57ad862df381be296f5b1b25ef4/awslambdaric-4.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9207703cb8b737bbfddba8c0e22586ef18ffca277ee3360543ce3f053b9ec95e", size = 1385174, upload-time = "2026-08-10T17:57:20.497Z" }, { url = "https://files.pythonhosted.org/packages/56/4f/d004ebdf987b73985bf42036bb89f54d50d99751f2fd563e6f92031129f9/awslambdaric-4.0.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84e3710db039d90f743e6ce8c04c5593a398e9a5e6768f46fd045368cecfd0bb", size = 354000, upload-time = "2026-07-30T09:05:59.779Z" }, { url = "https://files.pythonhosted.org/packages/e6/5a/272f0f0700769ed8b199c2fab70d9cfcaf2164b49791fe9e662e41ca149b/awslambdaric-4.0.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd1858cc1e65e3ac0bad983704838e23ea9e7bc4f3bc083654855b9dcdcaac35", size = 356640, upload-time = "2026-07-30T09:05:55.665Z" }, + { url = "https://files.pythonhosted.org/packages/b6/a1/87bde859a0cf20ee25ae2216433a6b6422ab4e381bfc7600be5af98e13ef/awslambdaric-4.0.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:fc1b9955d12976eca40693fcd06bc663fa0ecf4e74003a61a8b8fc5f3a7f1693", size = 1328253, upload-time = "2026-08-10T17:49:02.485Z" }, + { url = "https://files.pythonhosted.org/packages/2f/4b/d82a9d23213ab01abfb84489206602ba12586f2fb1d869b806fdd1d46ed8/awslambdaric-4.0.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:597253283b449ddcf761c0e895d3e822511b1b336e749a435d4cfec1201099bd", size = 1384660, upload-time = "2026-08-10T17:57:21.959Z" }, { url = "https://files.pythonhosted.org/packages/2a/5d/19de623162204ef35d01f4416e3402e58009da0850eaced4d129df707d02/awslambdaric-4.0.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e895aa069e413c4c6eb32a70c188d0c4cd01debf3330a4272bcef71da46b8372", size = 355286, upload-time = "2026-07-30T09:06:01.143Z" }, { url = "https://files.pythonhosted.org/packages/03/4d/032a10eef23aba3da7e4dcf9e72d248729cbb2c0bd6dab7e19863619872f/awslambdaric-4.0.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:939a45fa096f700c8d7441b2924e0250b98cba6d5bb7e9ebcb5974d1d8c1e318", size = 357696, upload-time = "2026-07-30T09:05:57.242Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/aa400934fb1fe77edfe43e57e83574bb06b0a1df7b0c1c61c5a43fcbb580/awslambdaric-4.0.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7fb5ba045d206a4c0583d4885a2293bfc252be67af4c39e868dabb93bd83db0f", size = 1330023, upload-time = "2026-08-10T17:49:03.806Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a1/a4e3d401ba991bdd53515827caa4e2505d99372cf3d0517654f21d233d4a/awslambdaric-4.0.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:74939ef7b0e47a2801747dda0285aedd02394bfe21a3e4b081dc96dce5cb335e", size = 1385492, upload-time = "2026-08-10T17:57:23.896Z" }, ] [[package]] @@ -909,7 +921,7 @@ wheels = [ [[package]] name = "portscanner-contracts" -version = "0.1.0" +version = "1.0.0" source = { editable = "contracts" } dependencies = [ { name = "pydantic" }, @@ -920,7 +932,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.11,<3" }] [[package]] name = "portscanner-generator" -version = "0.1.0" +version = "1.0.0" source = { editable = "generator" } dependencies = [ { name = "awslambdaric" }, @@ -948,7 +960,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-inventory" -version = "0.1.0" +version = "1.0.0" source = { editable = "inventory" } dependencies = [ { name = "awslambdaric" }, @@ -972,7 +984,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-migrator" -version = "0.1.0" +version = "1.0.0" source = { editable = "db/migrator" } dependencies = [ { name = "awslambdaric" }, @@ -996,7 +1008,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-parser" -version = "0.1.0" +version = "1.0.0" source = { editable = "parser" } dependencies = [ { name = "awslambdaric" }, @@ -1024,7 +1036,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-processor" -version = "0.1.0" +version = "1.0.0" source = { editable = "processor" } dependencies = [ { name = "awslambdaric" }, @@ -1050,7 +1062,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-scanner" -version = "0.1.0" +version = "1.0.0" source = { editable = "scanner/nmap" } dependencies = [ { name = "boto3" }, @@ -1074,7 +1086,7 @@ provides-extras = ["test"] [[package]] name = "portscanner-workspace" -version = "0.1.0" +version = "1.0.0" source = { virtual = "." } [package.dev-dependencies] From 649c97619455d012496995f94b85c5c05926d724 Mon Sep 17 00:00:00 2001 From: Qiancheng Wu Date: Wed, 12 Aug 2026 21:23:00 -0700 Subject: [PATCH 2/4] Trim remaining release surface Remove stale community, workflow, compatibility, and deployment scaffolding while keeping active security and legal controls, and add an architecture overview with the conference talk. Co-authored-by: Cursor --- .dockerignore | 1 + .github/workflows/aws-sandbox.yml | 189 ---------------- .github/workflows/containers.yml | 1 - CHANGELOG.md | 7 +- CODE_OF_CONDUCT.md | 75 ------- CONTRIBUTING.md | 117 ---------- LICENSES/Apache-2.0.txt | 201 ++++++++++++++++++ Makefile | 2 +- README.md | 65 ++++-- THIRD_PARTY_NOTICES.md | 19 +- .../src/portscanner_generator/revalidation.py | 51 +---- generator/tests/fakes.py | 10 +- generator/tests/test_dispatch.py | 6 +- operator/.dockerignore | 53 ----- operator/Dockerfile | 6 +- operator/Makefile | 2 +- operator/PROJECT | 19 -- operator/THIRD_PARTY_NOTICES.md | 21 +- .../portscanner/templates/scanner-rbac.yaml | 51 ----- .../templates/scanner-serviceaccount.yaml | 19 ++ operator/chart/portscanner/values.yaml | 5 - parser/src/act_parser/contracts.py | 54 +---- parser/src/act_parser/models.py | 23 +- parser/tests/test_contracts.py | 90 +------- parser/tests/test_models.py | 8 - scanner/nmap/README.md | 5 +- .../nmap/src/portscanner_scanner/parser.py | 176 --------------- scanner/nmap/tests/test_worker_and_cli.py | 11 - scanner/nmap/tests/test_xml.py | 94 ++++++++ scanner/nmap/tests/test_xml_and_parser.py | 185 ---------------- terraform/aws/scripts/build-images.sh | 2 +- tools/sanitize-policy.toml | 4 - tools/tests/test_release_packaging.py | 26 +-- 33 files changed, 453 insertions(+), 1145 deletions(-) delete mode 100644 .github/workflows/aws-sandbox.yml delete mode 100644 CODE_OF_CONDUCT.md delete mode 100644 CONTRIBUTING.md create mode 100644 LICENSES/Apache-2.0.txt delete mode 100644 operator/.dockerignore delete mode 100644 operator/PROJECT delete mode 100644 operator/chart/portscanner/templates/scanner-rbac.yaml create mode 100644 operator/chart/portscanner/templates/scanner-serviceaccount.yaml delete mode 100644 scanner/nmap/src/portscanner_scanner/parser.py create mode 100644 scanner/nmap/tests/test_xml.py delete mode 100644 scanner/nmap/tests/test_xml_and_parser.py diff --git a/.dockerignore b/.dockerignore index 5cacb9d..4311c29 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ venv **/__pycache__ **/*.py[cod] **/*.egg-info +**/bin **/build **/dist **/.mypy_cache diff --git a/.github/workflows/aws-sandbox.yml b/.github/workflows/aws-sandbox.yml deleted file mode 100644 index cf1efd7..0000000 --- a/.github/workflows/aws-sandbox.yml +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Portscanner contributors -# SPDX-License-Identifier: MIT - -name: AWS sandbox (manual) - -on: - workflow_dispatch: - inputs: - run_live: - description: Run the configured sandbox after environment approval - required: true - type: boolean - default: false - authorization_confirmed: - description: Confirm written authorization and provider-policy review - required: true - type: boolean - default: false - -permissions: - contents: read - id-token: write - -concurrency: - group: aws-sandbox - cancel-in-progress: false - -jobs: - sandbox: - name: Approval-protected sandbox - if: >- - inputs.run_live && - inputs.authorization_confirmed && - vars.AWS_SANDBOX_ENABLED == 'true' - runs-on: ubuntu-latest - timeout-minutes: 120 - environment: - name: aws-sandbox - steps: - - uses: actions/checkout@v7.0.1 - - - name: Validate protected configuration - env: - EXPECTED_ACCOUNT: ${{ vars.AWS_SANDBOX_ACCOUNT_ID }} - REGION: ${{ vars.AWS_SANDBOX_REGION }} - ROLE_REFERENCE: ${{ vars.AWS_SANDBOX_ROLE_ARN }} - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - TFVARS_JSON: ${{ secrets.AWS_SANDBOX_TFVARS_JSON }} - shell: bash - run: | - set -euo pipefail - test -n "${EXPECTED_ACCOUNT}" - test -n "${REGION}" - test -n "${ROLE_REFERENCE}" - test -n "${TF_ROOT}" - test -n "${TFVARS_JSON}" - [[ "${EXPECTED_ACCOUNT}" =~ ^[0-9]{12}$ ]] - [[ "${REGION}" =~ ^[a-z]{2}-[a-z]+-[0-9]+$ ]] - [[ "${TF_ROOT}" =~ ^[A-Za-z0-9_./-]+$ ]] - [[ "${TF_ROOT}" != /* ]] - [[ "${TF_ROOT}" != *".."* ]] - test -f "${TF_ROOT}/main.tf" - jq -e '.force_destroy_buckets == true' <<<"${TFVARS_JSON}" >/dev/null - echo "Protected sandbox configuration is present." - - - name: Obtain short-lived AWS credentials - uses: aws-actions/configure-aws-credentials@v6.2.3 - with: - role-to-assume: ${{ vars.AWS_SANDBOX_ROLE_ARN }} - role-session-name: portscanner-sandbox-${{ github.run_id }} - aws-region: ${{ vars.AWS_SANDBOX_REGION }} - allowed-account-ids: ${{ vars.AWS_SANDBOX_ACCOUNT_ID }} - mask-aws-account-id: true - unset-current-credentials: true - output-credentials: false - - - uses: hashicorp/setup-terraform@v4.0.1 - with: - terraform_version: "1.7.4" - - - name: Prepare isolated local state - env: - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - TFVARS_JSON: ${{ secrets.AWS_SANDBOX_TFVARS_JSON }} - shell: bash - run: | - set -euo pipefail - printf '%s' "${TFVARS_JSON}" > "${RUNNER_TEMP}/sandbox.tfvars.json" - chmod 600 "${RUNNER_TEMP}/sandbox.tfvars.json" - terraform -chdir="${TF_ROOT}" init -backend=false -input=false -no-color - touch "${RUNNER_TEMP}/sandbox-initialized" - - - name: Plan with dispatch disabled - env: - AWS_SANDBOX_ACCOUNT_ID: ${{ vars.AWS_SANDBOX_ACCOUNT_ID }} - AWS_SANDBOX_REGION: ${{ vars.AWS_SANDBOX_REGION }} - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - shell: bash - run: | - set -euo pipefail - terraform -chdir="${TF_ROOT}" plan \ - -input=false \ - -no-color \ - -state="${RUNNER_TEMP}/sandbox.tfstate" \ - -var-file="${RUNNER_TEMP}/sandbox.tfvars.json" \ - -var="aws_region=${AWS_SANDBOX_REGION}" \ - -var="expected_deployment_account_id=${AWS_SANDBOX_ACCOUNT_ID}" \ - -var='deploy_runtime=false' \ - -var='run_migration=false' \ - -var='install_operator=false' \ - -var='enable_event_dispatch=false' \ - -var='enable_automatic_inventory=false' \ - -var='canary_mode=false' \ - -out="${RUNNER_TEMP}/sandbox.plan" - - - name: Apply bounded sandbox - env: - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - shell: bash - run: | - set -euo pipefail - timeout 45m terraform -chdir="${TF_ROOT}" apply \ - -input=false \ - -no-color \ - -state="${RUNNER_TEMP}/sandbox.tfstate" \ - "${RUNNER_TEMP}/sandbox.plan" - touch "${RUNNER_TEMP}/sandbox-applied" - - - name: Run non-scanning smoke checks - env: - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - shell: bash - run: | - set -euo pipefail - test -f "${RUNNER_TEMP}/sandbox-applied" - terraform -chdir="${TF_ROOT}" output \ - -state="${RUNNER_TEMP}/sandbox.tfstate" \ - -json > /dev/null - echo "Sandbox applied with dispatch disabled; no network scan was run." - - - name: Destroy sandbox resources - if: always() - env: - AWS_SANDBOX_ACCOUNT_ID: ${{ vars.AWS_SANDBOX_ACCOUNT_ID }} - AWS_SANDBOX_REGION: ${{ vars.AWS_SANDBOX_REGION }} - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - shell: bash - run: | - set -euo pipefail - if [[ ! -f "${RUNNER_TEMP}/sandbox-initialized" ]]; then - echo "Terraform was not initialized; no Terraform resources were created." - exit 0 - fi - timeout 45m terraform -chdir="${TF_ROOT}" destroy \ - -auto-approve \ - -input=false \ - -no-color \ - -state="${RUNNER_TEMP}/sandbox.tfstate" \ - -var-file="${RUNNER_TEMP}/sandbox.tfvars.json" \ - -var="aws_region=${AWS_SANDBOX_REGION}" \ - -var="expected_deployment_account_id=${AWS_SANDBOX_ACCOUNT_ID}" \ - -var='deploy_runtime=false' \ - -var='run_migration=false' \ - -var='install_operator=false' \ - -var='enable_event_dispatch=false' \ - -var='enable_automatic_inventory=false' \ - -var='canary_mode=false' - - - name: Verify cleanup - if: always() - env: - TF_ROOT: ${{ vars.AWS_TERRAFORM_ROOT }} - shell: bash - run: | - set -euo pipefail - if [[ ! -f "${RUNNER_TEMP}/sandbox.tfstate" ]]; then - exit 0 - fi - if [[ -n "$(terraform -chdir="${TF_ROOT}" state list \ - -state="${RUNNER_TEMP}/sandbox.tfstate")" ]]; then - echo "Sandbox state still contains resources." >&2 - exit 1 - fi - rm -f \ - "${RUNNER_TEMP}/sandbox.plan" \ - "${RUNNER_TEMP}/sandbox.tfstate" \ - "${RUNNER_TEMP}/sandbox.tfstate.backup" \ - "${RUNNER_TEMP}/sandbox.tfvars.json" - echo "Sandbox cleanup verified." diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 2066f98..68a0e43 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -91,7 +91,6 @@ jobs: text=True, ) component_contexts = { - "operator/Dockerfile": "operator", "scanner/nmap/Dockerfile": ".", } python_images = { diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb449d..f150cb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,10 +48,9 @@ use [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and operator startup smoke. - Publication sanitizer with a denylist policy and unit tests. - CI definitions for source/schema tests, generated drift, Terraform, Kubernetes/Helm, - containers, CodeQL, license/REUSE, secret scanning, publication sanitization, and a - disabled manual AWS sandbox. -- Security, contribution, conduct, issue, pull request, dependency update, pre-commit, - and secret-scanning policies. + containers, CodeQL, license/REUSE, secret scanning, and publication sanitization. +- Security, issue, pull request, dependency update, pre-commit, and secret-scanning + policies. - Helm-only Kubernetes packaging, optional finding export, and a PostgreSQL-first default deployment boundary. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index bd5678f..0000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# Code of conduct - -## Our pledge - -We pledge to make participation in this project a harassment-free experience for -everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex -characteristics, gender identity and expression, level of experience, education, -socioeconomic status, nationality, personal appearance, race, religion, or sexual -identity and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, -inclusive, and healthy community. - -## Expected behavior - -Examples of behavior that contributes to a positive community include: - -- showing empathy and respect; -- giving and accepting constructive technical feedback; -- focusing on what is best for the community and safe operation of the project; -- acknowledging mistakes and learning from them; and -- protecting private, security-sensitive, and personal information. - -Examples of unacceptable behavior include: - -- sexualized language or attention; -- trolling, insults, derogatory comments, or personal or political attacks; -- public or private harassment; -- publishing another person's private information without explicit permission; -- encouraging unauthorized scanning or misuse of the project; and -- conduct that would reasonably be considered inappropriate in a professional setting. - -## Responsibilities - -Project maintainers may clarify and enforce these standards and may remove, edit, or -reject comments, commits, code, issues, and other contributions that are inconsistent -with them. Maintainers will communicate moderation reasons when appropriate and will -handle reports with discretion. - -This code applies in project spaces and when an individual officially represents the -project in public spaces. - -## Reporting - -For a private report, use the project's [GitHub private reporting -form](https://github.com/Roblox/portscanner/security/advisories/new) and begin the report -title with `Conduct report`. Do not include unnecessary personal or sensitive details. -Reports will be reviewed by project maintainers, who will avoid conflicts of interest -and share information only as needed to investigate and respond. - -For content hosted by GitHub, you may also use GitHub's block/report controls and -[abuse-reporting process](https://support.github.com/contact/report-abuse). - -## Enforcement - -Consequences may include: - -1. **Correction:** a private or public clarification and request to stop. -2. **Warning:** a formal warning with conditions for continued participation. -3. **Temporary restriction:** a time-limited restriction from project interaction. -4. **Permanent restriction:** removal from project participation for sustained, - repeated, or severe violations. - -Maintainers will consider context, impact, safety, privacy, and patterns of behavior. -Retaliation against a reporter or participant in an investigation is prohibited. - -## Attribution - -This policy is adapted from the [Contributor Covenant, version -2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 2c63521..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,117 +0,0 @@ - - -# Contributing - -Thank you for helping improve Portscanner. Changes should preserve authorization, -ownership, UNKNOWN, idempotency, and data-minimization invariants. - -By participating, you agree to follow [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). Report -vulnerabilities through the private process in [SECURITY.md](SECURITY.md), not a public -issue. - -## Before starting - -- Search existing issues and pull requests. -- Open a design issue before changing contracts, Target identity/generation, - authorization, scan behavior, evidence semantics, migrations, or deployment topology. -- Do not test against a public or third-party host. -- Do not add production data, internal measurements, slide assets, names, account - identifiers, cloud resource IDs, local paths, or company integration claims. - -## Development setup - -Use supported Python and Go versions from CI. Install project-specific dependencies only -from the component you are changing. Install pre-commit and enable it: - -```bash -python3 -m pip install pre-commit -pre-commit install -``` - -Run the public-content checks: - -```bash -python3 -m unittest discover -s tools/tests -p 'test_*.py' -v -python3 tools/sanitize.py --working-tree -pre-commit run --all-files -``` - -The root `Makefile` is the command index for component, schema, generated, -Terraform, Helm, container, and integration checks. Run `make ci` before -requesting review and `make containers` when an image or runtime dependency -changes. - -## Design rules - -- Keep **DISCOVER → PRIORITIZE → VERIFY → ACT** boundaries explicit. -- Snapshot-driven inventory is authoritative; event-driven signals are hints. -- Apply removals only after absence from a complete snapshot or an - authoritative resource-specific reread. Partial snapshots and raw signals - never remove targets. -- Use stable Target identity and monotonic generations. -- Revalidate ownership immediately before dispatch. -- Preserve periodic known-door reconciliation while priority work advances. -- Treat failed or incomplete work as UNKNOWN. -- Keep provider adapters independent of scanner and parser internals. -- Select typed named scan profiles; never pass source-controlled command fragments. -- Default dispatch off and account/CIDR allowlists empty. -- Minimize and redact provider metadata. - -Read [inventory/README.md](inventory/README.md) before adding an inventory or -signal adapter. - -## Fixtures and examples - -Use only: - -- `192.0.2.0/24`, `198.51.100.0/24`, or `203.0.113.0/24` for IPv4 documentation; -- `2001:db8::/32` for IPv6 documentation; -- the exact synthetic account fixture allowed by the sanitizer when an account-shaped - value is required; and -- invented identifiers such as `resource-synthetic-a`. - -Prefer placeholders such as `${ACCOUNT_ID}` over account-shaped values. Never copy a -cloud console export or real scanner result. - -## Pull requests - -Keep changes focused and explain why the change is needed. Complete the pull request -template, including: - -- behavior and contract changes; -- safety and data impact; -- migration/rollback plan; -- tests and generated outputs; -- documentation updates; and -- whether a separately authorized canary is needed. - -Generated files must be committed with their source changes and reproduce cleanly. -Database changes use expand/contract sequencing and must support a rolling upgrade. -Infrastructure changes default disabled and support staged activation. - -All required checks must pass. Do not weaken sanitizer, gitleaks, CodeQL, IaC, -Kubernetes, image, license, or test checks merely to make a pull request green. - -## Commit and review hygiene - -- Write clear commits with one purpose. -- Do not commit build output, Terraform state/variables, plans with live values, keys, - kubeconfigs, credentials, or large binary artifacts. -- Binary files are rejected by default. Do not add one without security review of an - exact path and SHA-256 publication-policy entry. -- Pin or lock dependencies using the component's existing mechanism. After changing a - Python Lambda dependency, update `uv.lock`, run - `uv run --frozen python tools/export_runtime_requirements.py`, and commit all six - reviewed hash exports together with the lock. -- Request security-focused review for authorization, IAM, parsing, scanner arguments, - source adapters, CI permissions, sanitizer exceptions, and evidence handling. -- Treat changes to allowlists or suppressions as security changes. - -## Documentation - -Describe current behavior and explicit exclusions. Do not publish unsupported latency, -scale, coverage, or cost claims. Examples must be runnable only with operator-supplied -authorized values and should default to no dispatch. diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index c353df3..b08226e 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,6 @@ containers: $(PYTHON) tools/build_tracked_image.py --dockerfile processor/Dockerfile --tag portscanner-processor:test $(PYTHON) tools/build_tracked_image.py --dockerfile db/migrator/Dockerfile --tag portscanner-migrator:test $(PYTHON) tools/build_tracked_image.py --dockerfile scanner/nmap/Dockerfile --tag portscanner-scanner:test - $(PYTHON) tools/build_tracked_image.py --dockerfile operator/Dockerfile --context operator --tag portscanner-operator:test + $(PYTHON) tools/build_tracked_image.py --dockerfile operator/Dockerfile --tag portscanner-operator:test ci: check test-postgresql test-go test-go-envtest vet-go sanitize terraform-script-tests terraform-validate kubernetes-validate diff --git a/README.md b/README.md index b3cf974..19adb81 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,48 @@ SPDX-License-Identifier: MIT Portscanner is a self-hosted system for continuously verifying the Internet-facing ports of cloud assets you are authorized to test. It discovers -current AWS ownership, dispatches bounded Nmap jobs from Kubernetes, and stores +current AWS ownership, dispatches bounded Nmap Jobs in Kubernetes, and stores generation-safe exposures and findings in PostgreSQL. > Only scan assets you own or are explicitly authorized to test. The AWS > evaluation creates EKS, Aurora, NAT, EC2, queues, buckets, and supporting -> services that incur charges. Dispatch stays off except for one managed -> canary until you explicitly activate an integration. +> services that incur charges. Only the managed-canary evaluation briefly +> enables dispatch; account inventory stays paused until you activate it. + +## Architecture at a glance + +The solid path is the default one-target evaluation; dashed paths require +explicit opt-in. Snapshots are authoritative and change signals only trigger a +current-state reread. For the design motivation, watch +[Minutes from Malice](https://tinyurl.com/minutes-from-malice). + +```mermaid +flowchart LR + canary["Default EC2 canary: TCP 18080"] + accounts["Opt-in AWS accounts and change hints"] + inventory["Inventory Lambdas and DynamoDB outbox"] + generator["Generator and ownership revalidation"] + operator["EKS operator"] + scanner["Bounded Nmap Job"] + projector["Target projector"] + results["Result in S3 and SQS"] + parser["Parser Lambda"] + database["Aurora PostgreSQL"] + export["Opt-in finding export"] + + canary -->|"Scoped EC2 snapshot"| inventory + accounts -.-> inventory + inventory -->|"TargetEvent in S3 and SQS"| generator + generator --> operator + operator --> scanner + scanner -->|"TCP connect through NAT"| canary + inventory --> projector + scanner --> results + projector --> database + results --> parser + parser --> database + database -.-> export +``` ## Try it locally @@ -26,7 +61,9 @@ make check ``` `make ci` adds PostgreSQL, Go/envtest, Terraform, Helm, and publication checks. -See [CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow. +Contributions must use synthetic fixtures, preserve authorization and +`UNKNOWN` semantics, commit generated outputs, and pass `make ci`. New source +adapters must follow [inventory/README.md](inventory/README.md). ## Evaluate the AWS stack @@ -83,16 +120,13 @@ Activation is never implied by installation. Signals only accelerate work; authoritative snapshots and immediate provider ownership rereads remain the scan gate. -## How it works +## Safety model -1. **Discover** complete provider inventory and record stable target - generations. -2. **Prioritize** new, changed, and periodic coverage work under explicit - account, CIDR, profile, rate, Pod, and Job limits. -3. **Verify** current ownership immediately before creating one digest-pinned - Scanner Job. -4. **Act** on complete evidence; failed or incomplete work remains `UNKNOWN` - and never silently closes an exposure. +Inventory records stable target generations, and the generator rereads current +provider ownership immediately before creating a digest-pinned Scanner Job. +Explicit account, CIDR, profile, rate, Pod, and Job limits bound dispatch; +failed or incomplete evidence remains `UNKNOWN` and never silently closes an +exposure. ## Supported scope @@ -118,6 +152,5 @@ Synthetic contracts are in `examples/`; generated JSON Schemas are in ## Project policy Portscanner is MIT licensed. Report vulnerabilities privately through -[SECURITY.md](SECURITY.md), follow [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md), -and review [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for bundled -dependencies. +[SECURITY.md](SECURITY.md), and review +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for bundled dependencies. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 72ebf48..b9137a5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -47,7 +47,7 @@ this notice. - Upstream licensing files: and - + ### Kubernetes Python client @@ -65,7 +65,22 @@ this notice. - Project: - Upstream licensing information: - + + +## Go operator + +The statically linked operator includes Kubernetes API machinery, client-go, +controller-runtime, and `golang.org/x` modules. Their exact resolved versions +are recorded in `operator/go.sum`. + +- Kubernetes projects: , + , and + +- Kubernetes and controller-runtime license: + +- Bundled license text: `LICENSES/Apache-2.0.txt` +- Go supplemental repositories and licenses: + ## Build, test, and repository tooling diff --git a/generator/src/portscanner_generator/revalidation.py b/generator/src/portscanner_generator/revalidation.py index 9ec9039..6d31b4b 100644 --- a/generator/src/portscanner_generator/revalidation.py +++ b/generator/src/portscanner_generator/revalidation.py @@ -2,12 +2,11 @@ from __future__ import annotations -import importlib import os from collections.abc import Mapping from dataclasses import dataclass from enum import StrEnum -from typing import Any, Protocol, TypeGuard, cast +from typing import Any, Protocol class OwnershipDecision(StrEnum): @@ -26,25 +25,11 @@ class OwnershipCheck: reason: str | None = None -class EventOwnershipService(Protocol): +class OwnershipService(Protocol): def validate_event(self, event: Any) -> Any: """Return a verdict for a complete shared event.""" -class TargetOwnershipService(Protocol): - def revalidate(self, target: Any, generation: int) -> Any: - """Return ACTIVE, STALE, MOVED, INACTIVE, or UNKNOWN.""" - - -type OwnershipService = EventOwnershipService | TargetOwnershipService - - -def _is_ownership_service(value: object) -> TypeGuard[OwnershipService]: - return callable(getattr(value, "validate_event", None)) or callable( - getattr(value, "revalidate", None) - ) - - def _field(verdict: Any, name: str, default: Any = None) -> Any: if isinstance(verdict, Mapping): return verdict.get(name, default) @@ -111,12 +96,7 @@ def revalidate_event( event: Any, ) -> OwnershipCheck: generation = event.target.generation - validate_event = getattr(service, "validate_event", None) - if callable(validate_event): - raw_verdict = validate_event(event) - else: - target_service = cast(TargetOwnershipService, service) - raw_verdict = target_service.revalidate(event.target, generation) + raw_verdict = service.validate_event(event) return classify_verdict(raw_verdict, generation) @@ -126,32 +106,9 @@ def ownership_service_from_environment( state_table: str | None = None, environment: Mapping[str, str] | None = None, ) -> OwnershipService: - """Load the sibling inventory package's environment-backed service.""" + """Build the inventory package's AWS ownership validator.""" env = environment if environment is not None else os.environ - module = importlib.import_module("portscanner_inventory") - candidate: object - for factory_name in ( - "ownership_service_from_environment", - "create_ownership_service", - "build_ownership_service", - ): - factory = getattr(module, factory_name, None) - if callable(factory): - candidate = factory() - if not _is_ownership_service(candidate): - raise RuntimeError("inventory ownership service has no validation method") - return candidate - - service_type = getattr(module, "OwnershipService", None) - from_environment = getattr(service_type, "from_environment", None) - if callable(from_environment): - candidate = from_environment() - if _is_ownership_service(candidate): - return candidate - - # The current inventory package exposes its AWS validator and safe client - # factory as composable services rather than a top-level factory. from portscanner_inventory.aws.ownership import OwnershipValidator from portscanner_inventory.aws.session import AwsClientFactory from portscanner_inventory.state import DynamoStateStore diff --git a/generator/tests/fakes.py b/generator/tests/fakes.py index 9f864ad..a96316e 100644 --- a/generator/tests/fakes.py +++ b/generator/tests/fakes.py @@ -185,14 +185,14 @@ def __init__( ) -> None: self.state = state self.current_generation = current_generation - self.calls: list[tuple[Any, int]] = [] + self.calls: list[Any] = [] - def revalidate(self, target: Any, generation: int) -> Any: - self.calls.append((target, generation)) + def validate_event(self, event: Any) -> Any: + self.calls.append(event) return SimpleNamespace( verdict=self.state, - target_id=target.target_id, - requested_generation=generation, + target_id=event.target.target_id, + requested_generation=event.target.generation, current_generation=self.current_generation, reason="fixture", ) diff --git a/generator/tests/test_dispatch.py b/generator/tests/test_dispatch.py index 4289ec3..8ed43f1 100644 --- a/generator/tests/test_dispatch.py +++ b/generator/tests/test_dispatch.py @@ -311,7 +311,8 @@ class SequencedOwnership: def __init__(self) -> None: self.states = iter(("UNKNOWN", "STALE")) - def revalidate(self, _target: object, generation: int) -> dict[str, object]: + def validate_event(self, event: object) -> dict[str, object]: + generation = event.target.generation # type: ignore[attr-defined] return { "state": next(self.states), "requested_generation": generation, @@ -353,7 +354,8 @@ class MovingOwnership: def __init__(self) -> None: self.states = iter(("ACTIVE", "MOVED")) - def revalidate(self, _target: object, generation: int) -> dict[str, object]: + def validate_event(self, event: object) -> dict[str, object]: + generation = event.target.generation # type: ignore[attr-defined] return { "state": next(self.states), "requested_generation": generation, diff --git a/operator/.dockerignore b/operator/.dockerignore deleted file mode 100644 index 90f9dc4..0000000 --- a/operator/.dockerignore +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Portscanner contributors -# SPDX-License-Identifier: MIT - -.git -.github -.claude -.cursor -.idea -.vscode -.DS_Store -Thumbs.db -**/.DS_Store -**/Thumbs.db -**/.claude -**/.cursor -**/.idea -**/.vscode -**/*.swp -**/*~ -bin -cover.out -*.log -**/.aws -**/.azure -**/.docker -**/.git-credentials -**/.kube -**/.ssh -**/.config/gcloud -**/.terraform -**/.terraformrc -**/terraform.rc -**/.env -**/.env.* -**/.netrc -**/.npmrc -**/.pypirc -**/application_default_credentials.json -**/credentials.json -**/service-account*.json -**/*.kubeconfig -**/*.pem -**/*.key -**/*.p12 -**/*.pfx -**/coverage.xml -**/htmlcov -**/crash.log -**/crash.*.log -**/override.tf -**/override.tf.json -**/*_override.tf -**/*_override.tf.json diff --git a/operator/Dockerfile b/operator/Dockerfile index f7d80e1..f532052 100644 --- a/operator/Dockerfile +++ b/operator/Dockerfile @@ -11,10 +11,11 @@ ARG TARGETOS ARG TARGETARCH WORKDIR /workspace -COPY go.mod go.sum ./ +COPY operator/go.mod operator/go.sum ./ RUN go mod download -COPY . . +COPY operator . +COPY LICENSES/Apache-2.0.txt /workspace/third-party-licenses/Apache-2.0.txt RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ go build -buildvcs=false -trimpath -tags="netgo,osusergo" \ -ldflags="-s -w" -o /out/manager ./cmd/manager @@ -25,5 +26,6 @@ LABEL org.opencontainers.image.source="https://github.com/Roblox/portscanner" \ COPY --from=builder --chmod=0555 /out/manager /manager COPY --from=builder /workspace/LICENSE /licenses/LICENSE COPY --from=builder /workspace/THIRD_PARTY_NOTICES.md /licenses/THIRD_PARTY_NOTICES.md +COPY --from=builder /workspace/third-party-licenses/Apache-2.0.txt /licenses/Apache-2.0.txt USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/operator/Makefile b/operator/Makefile index dc86f29..800ff4a 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -64,7 +64,7 @@ verify-manifests: .PHONY: docker-build docker-build: - docker build --tag $(IMG) . + docker build --file Dockerfile --tag $(IMG) .. $(CONTROLLER_GEN): mkdir -p $(LOCALBIN) diff --git a/operator/PROJECT b/operator/PROJECT deleted file mode 100644 index b2c2488..0000000 --- a/operator/PROJECT +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Portscanner contributors -# SPDX-License-Identifier: MIT - -domain: portscanner.io -layout: - - go.kubebuilder.io/v4 -projectName: portscanner -repo: github.com/Roblox/portscanner/operator -resources: - - api: - crdVersion: v1 - namespaced: true - controller: true - domain: portscanner.io - group: scanning - kind: Scanner - path: github.com/Roblox/portscanner/operator/api/v1alpha1 - version: v1alpha1 -version: "3" diff --git a/operator/THIRD_PARTY_NOTICES.md b/operator/THIRD_PARTY_NOTICES.md index 72ebf48..b30a347 100644 --- a/operator/THIRD_PARTY_NOTICES.md +++ b/operator/THIRD_PARTY_NOTICES.md @@ -1,6 +1,6 @@ # Third-party notices @@ -47,7 +47,7 @@ this notice. - Upstream licensing files: and - + ### Kubernetes Python client @@ -65,7 +65,22 @@ this notice. - Project: - Upstream licensing information: - + + +## Go operator + +The statically linked operator includes Kubernetes API machinery, client-go, +controller-runtime, and `golang.org/x` modules. Their exact resolved versions +are recorded in `operator/go.sum`. + +- Kubernetes projects: , + , and + +- Kubernetes and controller-runtime license: + +- Bundled image license text: `/licenses/Apache-2.0.txt` +- Go supplemental repositories and licenses: + ## Build, test, and repository tooling diff --git a/operator/chart/portscanner/templates/scanner-rbac.yaml b/operator/chart/portscanner/templates/scanner-rbac.yaml deleted file mode 100644 index e930547..0000000 --- a/operator/chart/portscanner/templates/scanner-rbac.yaml +++ /dev/null @@ -1,51 +0,0 @@ -{{/* -SPDX-FileCopyrightText: 2026 Portscanner contributors -SPDX-License-Identifier: MIT -*/}} -{{- $root := . -}} -{{- if $root.Values.scanner.serviceAccount.create }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "portscanner.scannerServiceAccountName" $root }} - namespace: {{ $root.Release.Namespace }} - labels: - {{- include "portscanner.labels" $root | nindent 4 }} - app.kubernetes.io/component: scanner - {{- with $root.Values.scanner.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ $root.Values.scanner.serviceAccount.automountServiceAccountToken }} -{{- end }} -{{- if $root.Values.scanner.rbac.create }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "portscanner.fullname" $root }}-scanner - namespace: {{ $root.Release.Namespace }} - labels: - {{- include "portscanner.labels" $root | nindent 4 }} - app.kubernetes.io/component: scanner -rules: - {{- toYaml $root.Values.scanner.rbac.rules | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "portscanner.fullname" $root }}-scanner - namespace: {{ $root.Release.Namespace }} - labels: - {{- include "portscanner.labels" $root | nindent 4 }} - app.kubernetes.io/component: scanner -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "portscanner.fullname" $root }}-scanner -subjects: - - kind: ServiceAccount - name: {{ include "portscanner.scannerServiceAccountName" $root }} - namespace: {{ $root.Release.Namespace }} -{{- end }} diff --git a/operator/chart/portscanner/templates/scanner-serviceaccount.yaml b/operator/chart/portscanner/templates/scanner-serviceaccount.yaml new file mode 100644 index 0000000..a4e8568 --- /dev/null +++ b/operator/chart/portscanner/templates/scanner-serviceaccount.yaml @@ -0,0 +1,19 @@ +{{/* +SPDX-FileCopyrightText: 2026 Portscanner contributors +SPDX-License-Identifier: MIT +*/}} +{{- if .Values.scanner.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "portscanner.scannerServiceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "portscanner.labels" . | nindent 4 }} + app.kubernetes.io/component: scanner + {{- with .Values.scanner.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/operator/chart/portscanner/values.yaml b/operator/chart/portscanner/values.yaml index 03b4835..d59ff54 100644 --- a/operator/chart/portscanner/values.yaml +++ b/operator/chart/portscanner/values.yaml @@ -67,11 +67,6 @@ scanner: create: true name: "" annotations: {} - automountServiceAccountToken: false - rbac: - create: true - # Scanner Jobs need no Kubernetes permissions by default. - rules: [] authorization: # Optional deployment policy. Empty allowedCidrs leaves account/ownership # authorization as the scope gate; scanner-enforced denies still apply first. diff --git a/parser/src/act_parser/contracts.py b/parser/src/act_parser/contracts.py index 3c34f77..a91a6b3 100644 --- a/parser/src/act_parser/contracts.py +++ b/parser/src/act_parser/contracts.py @@ -1,75 +1,33 @@ -"""Adapter to the authoritative shared ScanResult contract package.""" +"""Error translation around the required Portscanner contract package.""" from __future__ import annotations -import importlib from collections.abc import Mapping from typing import Any +import portscanner_contracts + class ContractValidationError(ValueError): """The shared contract rejected an event envelope.""" -def _mapping(value: Any, original: Mapping[str, Any]) -> Mapping[str, Any]: - if value is None: - return original - if isinstance(value, Mapping): - return value - model_dump = getattr(value, "model_dump", None) - if callable(model_dump): - mapped = model_dump(mode="json") - if isinstance(mapped, Mapping): - return mapped - to_dict = getattr(value, "to_dict", None) - if callable(to_dict): - mapped = to_dict() - if isinstance(mapped, Mapping): - return mapped - raise ContractValidationError("shared ScanResult validator returned an unsupported value") - - def validate_scan_result(payload: Mapping[str, Any]) -> Mapping[str, Any]: - """Validate with the public ScanResultEnvelope validator.""" - try: - contracts = importlib.import_module("portscanner_contracts") - validator = getattr(contracts, "validate_scan_result", None) - if not callable(validator): - raise RuntimeError("shared package does not export validate_scan_result") - return _mapping(validator(payload), payload) - except ContractValidationError: - raise + return portscanner_contracts.validate_scan_result(payload) except Exception as error: raise ContractValidationError("ScanResult contract validation failed") from error def validate_finding(payload: Mapping[str, Any]) -> Mapping[str, Any]: - """Validate with the public Finding payload validator.""" - try: - contracts = importlib.import_module("portscanner_contracts") - validator = getattr(contracts, "validate_finding", None) - if not callable(validator): - raise RuntimeError("shared package does not export validate_finding") - return _mapping(validator(payload), payload) - except ContractValidationError: - raise + return portscanner_contracts.validate_finding(payload) except Exception as error: raise ContractValidationError("Finding contract validation failed") from error def parse_target_event(payload: Mapping[str, Any]) -> Any: - """Parse a work event or removal through the authoritative shared parser.""" try: - contracts = importlib.import_module("portscanner_contracts") - parser = getattr(contracts, "parse_target_event", None) - if not callable(parser): - raise RuntimeError("portscanner_contracts does not expose parse_target_event") - return parser(payload) - except ContractValidationError: - raise - except RuntimeError: - raise + return portscanner_contracts.parse_target_event(payload) except Exception as error: raise ContractValidationError("TargetEvent contract validation failed") from error diff --git a/parser/src/act_parser/models.py b/parser/src/act_parser/models.py index 4cd75b1..c561f2f 100644 --- a/parser/src/act_parser/models.py +++ b/parser/src/act_parser/models.py @@ -4,7 +4,7 @@ import ipaddress import re -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -33,11 +33,9 @@ def parse_address(value: Any, field: str = "address") -> str: def port_spec_bounds(spec: str) -> tuple[int | None, int | None]: - """Return inclusive numeric bounds; symbolic declarations remain non-closing.""" + """Return inclusive numeric bounds for one canonical port declaration.""" match = _PORT_SPEC.fullmatch(spec) if match is None: - if spec == "nmap-default": - return None, None raise ValueError("coverage port spec is invalid") start = int(match.group("start")) end = int(match.group("end") or start) @@ -117,7 +115,7 @@ class ScanEnvelope: error_retryable: bool | None scan_started_at: datetime scan_completed_at: datetime - result_uploaded_at: datetime | None + result_uploaded_at: datetime coverage: tuple[CoverageDeclaration, ...] declared_open_tcp_ports: tuple[int, ...] raw_result: RawResultReference | None @@ -145,16 +143,13 @@ def from_mapping(cls, payload: Mapping[str, Any]) -> ScanEnvelope: ) raw_coverage = payload["coverage"] - coverage_items: Sequence[Mapping[str, Any]] - coverage_items = [raw_coverage] if isinstance(raw_coverage, Mapping) else raw_coverage + if not isinstance(raw_coverage, Mapping): + raise ValueError("coverage must be one declaration") coverage = tuple( - declaration - for item in coverage_items - for declaration in CoverageDeclaration.from_mapping(item) + declaration for declaration in CoverageDeclaration.from_mapping(raw_coverage) ) error = payload.get("error") - result_uploaded = timestamps.get("result_uploaded_at") return cls( attempt_id=str(payload["attempt_id"]), result_id=str(shared["result_id"]), @@ -178,10 +173,8 @@ def from_mapping(cls, payload: Mapping[str, Any]) -> ScanEnvelope: scan_completed_at=parse_timestamp( timestamps["scan_completed_at"], "timestamps.scan_completed_at" ), - result_uploaded_at=( - None - if result_uploaded is None - else parse_timestamp(result_uploaded, "timestamps.result_uploaded_at") + result_uploaded_at=parse_timestamp( + timestamps["result_uploaded_at"], "timestamps.result_uploaded_at" ), coverage=coverage, declared_open_tcp_ports=tuple(int(port) for port in shared["open_tcp_ports"]), diff --git a/parser/tests/test_contracts.py b/parser/tests/test_contracts.py index 1bca523..32125d7 100644 --- a/parser/tests/test_contracts.py +++ b/parser/tests/test_contracts.py @@ -1,103 +1,15 @@ from __future__ import annotations import json -import sys -import types from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any -import pytest -from act_parser.contracts import ( - ContractValidationError, - parse_target_event, - validate_finding, - validate_scan_result, -) +from act_parser.contracts import validate_scan_result from act_parser.database import Repository from act_parser.models import ScanEnvelope -def test_uses_shared_validator_without_local_fallback(monkeypatch: pytest.MonkeyPatch) -> None: - called: list[dict[str, object]] = [] - module = types.ModuleType("portscanner_contracts") - - def validator(payload: dict[str, object]) -> dict[str, object]: - called.append(payload) - return {**payload, "validated": True} - - module.validate_scan_result = validator # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "portscanner_contracts", module) - - payload = {"schema_version": "1.0"} - assert validate_scan_result(payload)["validated"] is True - assert called == [payload] - - -def test_rejects_non_mapping_model_dump(monkeypatch: pytest.MonkeyPatch) -> None: - module = types.ModuleType("portscanner_contracts") - - class InvalidModel: - def model_dump(self, *, mode: str) -> list[str]: - assert mode == "json" - return [] - - def validator(_payload: dict[str, object]) -> InvalidModel: - return InvalidModel() - - module.validate_scan_result = validator # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "portscanner_contracts", module) - - with pytest.raises(ContractValidationError, match="unsupported value"): - validate_scan_result({}) - - -def test_wraps_shared_contract_rejection(monkeypatch: pytest.MonkeyPatch) -> None: - module = types.ModuleType("portscanner_contracts") - - def validator(_payload: dict[str, object]) -> None: - raise ValueError("invalid") - - module.validate_scan_result = validator # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "portscanner_contracts", module) - - with pytest.raises(ContractValidationError): - validate_scan_result({}) - - -def test_finding_adapter_uses_public_validator(monkeypatch: pytest.MonkeyPatch) -> None: - called: list[dict[str, object]] = [] - module = types.ModuleType("portscanner_contracts") - - def validator(payload: dict[str, object]) -> dict[str, object]: - called.append(payload) - return {**payload, "validated": True} - - module.validate_finding = validator # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "portscanner_contracts", module) - - payload = {"schema_version": "1.0"} - assert validate_finding(payload)["validated"] is True - assert called == [payload] - - -def test_target_event_uses_shared_parser(monkeypatch: pytest.MonkeyPatch) -> None: - module = types.ModuleType("portscanner_contracts") - parsed = object() - calls: list[dict[str, object]] = [] - - def parser(payload: dict[str, object]) -> object: - calls.append(payload) - return parsed - - module.parse_target_event = parser # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "portscanner_contracts", module) - payload = {"event_type": "target.upsert"} - - assert parse_target_event(payload) is parsed - assert calls == [payload] - - def test_accepts_authoritative_typed_scanner_envelope() -> None: from portscanner_contracts import ( AddressFamily, diff --git a/parser/tests/test_models.py b/parser/tests/test_models.py index 4fc7b9b..d49ffe6 100644 --- a/parser/tests/test_models.py +++ b/parser/tests/test_models.py @@ -9,7 +9,6 @@ [ ("22", (22, 22)), ("1-65535", (1, 65535)), - ("nmap-default", (None, None)), ], ) def test_port_spec_bounds(spec: str, expected: tuple[int | None, int | None]) -> None: @@ -22,13 +21,6 @@ def test_invalid_port_specs_are_rejected(spec: str) -> None: port_spec_bounds(spec) -def test_symbolic_coverage_cannot_close_by_absence() -> None: - declaration = CoverageDeclaration.from_mapping( - {"protocol": "tcp", "ports": ["nmap-default"], "complete": True} - )[0] - assert not declaration.contains("tcp", 22) - - def test_incomplete_numeric_coverage_cannot_close_by_absence() -> None: declaration = CoverageDeclaration("tcp", "22", 22, None, True) diff --git a/scanner/nmap/README.md b/scanner/nmap/README.md index f856f04..d0b7791 100644 --- a/scanner/nmap/README.md +++ b/scanner/nmap/README.md @@ -183,9 +183,8 @@ the container runtime sends `SIGKILL`. Consumers must treat the envelope as the authoritative source for event, trace, run, attempt, target-generation, profile, and coverage fields. The -included parser validates the shared `ScanResultEnvelope` and nested -`ScanResult`, verifies the XML SHA-256, checks target/coverage consistency, and -ignores correlation-like XML attributes. +deployed parser Lambda validates the shared `ScanResultEnvelope`, nested +`ScanResult`, XML SHA-256, and target/coverage consistency. ## Container runtime diff --git a/scanner/nmap/src/portscanner_scanner/parser.py b/scanner/nmap/src/portscanner_scanner/parser.py deleted file mode 100644 index baec922..0000000 --- a/scanner/nmap/src/portscanner_scanner/parser.py +++ /dev/null @@ -1,176 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Portscanner contributors -# SPDX-License-Identifier: MIT - -"""Parse raw Nmap facts using ScanResult as authoritative correlation.""" - -from __future__ import annotations - -import hashlib -import hmac -from collections.abc import Mapping -from typing import Any - -from .coverage import PortCoverage -from .result import validate_scan_result -from .xml import ( - NmapXmlError, - nmap_state_to_observation, - parse_xml_bytes, - validate_complete_scan_bytes, -) - - -class ResultIntegrityError(ValueError): - """Raised when an envelope and its raw XML do not form one trusted result.""" - - -def _local_name(tag: str) -> str: - return tag.rsplit("}", 1)[-1] - - -def _children(element: Any, name: str) -> list[Any]: - return [child for child in element if _local_name(child.tag) == name] - - -def _partial_explicit_states( - xml_bytes: bytes, - *, - target: str, - coverage: PortCoverage, -) -> dict[int, str]: - root = parse_xml_bytes(xml_bytes) - hosts = _children(root, "host") - if len(hosts) != 1: - raise ResultIntegrityError("raw XML does not describe the envelope target") - addresses = [ - item.get("addr") - for item in _children(hosts[0], "address") - if item.get("addrtype") == "ipv4" - ] - if addresses != [target]: - raise ResultIntegrityError("raw XML target does not match ScanResult") - - states: dict[int, str] = {} - for ports_element in _children(hosts[0], "ports"): - for port_element in _children(ports_element, "port"): - if port_element.get("protocol") != "tcp": - continue - try: - port = int(port_element.get("portid", "")) - except ValueError as error: - raise ResultIntegrityError("raw XML contains an invalid TCP port") from error - if port in states or not coverage.contains(port): - raise ResultIntegrityError("raw XML port conflicts with declared coverage") - state_elements = _children(port_element, "state") - if len(state_elements) != 1 or not state_elements[0].get("state"): - raise ResultIntegrityError("raw XML port state is invalid") - states[port] = state_elements[0].get("state") - return states - - -def _observation_state( - nmap_state: str, - *, - complete: bool, - enrichment_set_changed: bool, -) -> str: - if complete: - return nmap_state_to_observation(nmap_state) - if enrichment_set_changed: - # Discovery and enrichment observed different open sets. Neither phase - # is authoritative for a state transition in this attempt. - return "UNKNOWN" - # A partial run can retain positive OPEN evidence. It can never prove - # closure for any port because Nmap did not finish the declared work. - return "OPEN" if nmap_state == "open" else "UNKNOWN" - - -def parse_authoritative_result( - envelope: Mapping[str, Any], - raw_xml: bytes, -) -> dict[str, Any]: - """Hash-check XML and correlate it only from the shared ScanResult envelope.""" - - canonical = validate_scan_result(envelope) - raw_result = canonical.get("raw_result") - if not isinstance(raw_result, Mapping) or raw_result.get("format") != "nmap-xml": - raise ResultIntegrityError("ScanResult has no Nmap XML artifact") - expected_hash = raw_result.get("sha256") - actual_hash = hashlib.sha256(raw_xml).hexdigest() - if not isinstance(expected_hash, str) or not hmac.compare_digest(expected_hash, actual_hash): - raise ResultIntegrityError("raw Nmap XML SHA-256 does not match ScanResult") - - target = canonical.get("target") - coverage_payload = canonical.get("coverage") - if not isinstance(target, Mapping) or not isinstance(coverage_payload, Mapping): - raise ResultIntegrityError("ScanResult target or coverage is invalid") - target_address = target.get("address") - declared_ports = coverage_payload.get("ports") - if not isinstance(target_address, str) or not isinstance(declared_ports, list): - raise ResultIntegrityError("ScanResult target or coverage is invalid") - try: - coverage = PortCoverage.parse(declared_ports) - except ValueError as error: - raise ResultIntegrityError("ScanResult coverage is invalid") from error - if list(coverage.as_strings()) != declared_ports: - raise ResultIntegrityError("ScanResult coverage is not canonical") - - complete = coverage_payload.get("complete") is True - error_payload = canonical.get("error") - enrichment_set_changed = ( - isinstance(error_payload, Mapping) - and error_payload.get("type") == "enrichment_port_set_changed" - ) - try: - if complete: - report = validate_complete_scan_bytes( - raw_xml, - target=target_address, - coverage=coverage, - ) - explicit_states = dict(report.explicit_states) - collapsed = [ - { - "state": nmap_state_to_observation(item.nmap_state), - "nmap_state": item.nmap_state, - "count": item.count, - } - for item in report.collapsed_states - ] - else: - explicit_states = _partial_explicit_states( - raw_xml, - target=target_address, - coverage=coverage, - ) - collapsed = [] - except NmapXmlError as error: - raise ResultIntegrityError("raw Nmap XML failed completeness checks") from error - - observations = [] - for port, nmap_state in sorted(explicit_states.items()): - observations.append( - { - "protocol": "tcp", - "port": port, - "state": _observation_state( - nmap_state, - complete=complete, - enrichment_set_changed=enrichment_set_changed, - ), - "nmap_state": nmap_state, - } - ) - - return { - "event_id": canonical["event_id"], - "directive_id": canonical["directive_id"], - "trace_id": canonical["trace_id"], - "run_id": canonical["run_id"], - "attempt_id": canonical["attempt_id"], - "target": dict(target), - "profile": canonical["profile"], - "coverage": dict(coverage_payload), - "observations": observations, - "collapsed_observations": collapsed, - } diff --git a/scanner/nmap/tests/test_worker_and_cli.py b/scanner/nmap/tests/test_worker_and_cli.py index 5787fdd..b64849b 100644 --- a/scanner/nmap/tests/test_worker_and_cli.py +++ b/scanner/nmap/tests/test_worker_and_cli.py @@ -25,7 +25,6 @@ ProcessResult, SubprocessRunner, ) -from portscanner_scanner.parser import parse_authoritative_result from portscanner_scanner.worker import ( ScanExecutionError, ScanRequest, @@ -362,21 +361,11 @@ def test_enrichment_port_change_is_partial_unknown_without_enrichment_reference( assert captured.value.error_type == "enrichment_port_set_changed" assert [operation[0] for operation in writer.operations] == ["file", "bytes"] - raw_discovery = writer.operations[0][2] failed = json.loads(writer.operations[-1][2]) assert failed["outcome"] == "partial" assert failed["coverage"]["complete"] is False assert failed["raw_result"]["enrichment"] is None assert failed["scan_result"]["open_tcp_ports"] == [] - parsed = parse_authoritative_result(failed, raw_discovery) - assert parsed["observations"] == [ - { - "protocol": "tcp", - "port": 80, - "state": "UNKNOWN", - "nmap_state": "open", - } - ] def test_cli_runs_with_fake_nmap_and_s3( diff --git a/scanner/nmap/tests/test_xml.py b/scanner/nmap/tests/test_xml.py new file mode 100644 index 0000000..07f5721 --- /dev/null +++ b/scanner/nmap/tests/test_xml.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: 2026 Portscanner contributors +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import pytest +from conftest import DOCUMENTATION_TARGET, nmap_xml +from portscanner_scanner.coverage import PortCoverage +from portscanner_scanner.xml import ( + NmapXmlError, + discovery_is_complete, + extract_open_tcp_ports, + parse_xml_bytes, + validate_complete_scan, +) + + +def test_discovery_completeness_and_open_port_extraction(tmp_path): + path = tmp_path / "raw.xml" + path.write_bytes( + nmap_xml( + explicit_states={80: "open"}, + collapsed_states=(("closed", 2),), + ) + ) + coverage = PortCoverage.parse("80-82") + + report = validate_complete_scan( + path, + target=DOCUMENTATION_TARGET, + coverage=coverage, + ) + assert report.open_ports == frozenset({80}) + assert report.represented_port_count == 3 + assert extract_open_tcp_ports(path) == frozenset({80}) + assert discovery_is_complete( + path, + target=DOCUMENTATION_TARGET, + coverage=coverage, + ) + + +@pytest.mark.parametrize( + "raw", + [ + nmap_xml( + explicit_states={80: "open"}, + collapsed_states=(("closed", 1),), + ), + nmap_xml( + explicit_states={80: "open"}, + collapsed_states=(("closed", 2),), + finished=False, + ), + nmap_xml( + explicit_states={80: "open"}, + collapsed_states=(("closed", 2),), + total_hosts=2, + ), + nmap_xml( + explicit_states={80: "open"}, + collapsed_states=(("open", 2),), + ), + ], +) +def test_incomplete_or_ambiguous_discovery_is_rejected(tmp_path, raw): + path = tmp_path / "raw.xml" + path.write_bytes(raw) + with pytest.raises(NmapXmlError): + validate_complete_scan( + path, + target=DOCUMENTATION_TARGET, + coverage=PortCoverage.parse("80-82"), + ) + + +def test_host_timeout_is_incomplete(tmp_path): + path = tmp_path / "raw.xml" + raw = nmap_xml( + explicit_states={80: "closed"}, + ).replace(b"", b'', 1) + path.write_bytes(raw) + + assert not discovery_is_complete( + path, + target=DOCUMENTATION_TARGET, + coverage=PortCoverage.parse("80"), + ) + + +def test_defused_parser_rejects_entity_expansion(): + raw = b']>&e;' + with pytest.raises(NmapXmlError): + parse_xml_bytes(raw) diff --git a/scanner/nmap/tests/test_xml_and_parser.py b/scanner/nmap/tests/test_xml_and_parser.py deleted file mode 100644 index a190e93..0000000 --- a/scanner/nmap/tests/test_xml_and_parser.py +++ /dev/null @@ -1,185 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Portscanner contributors -# SPDX-License-Identifier: MIT - -from __future__ import annotations - -import hashlib -from datetime import UTC, datetime - -import pytest -from conftest import DOCUMENTATION_TARGET, contract_target, nmap_xml -from portscanner_scanner.coverage import PortCoverage -from portscanner_scanner.parser import ResultIntegrityError, parse_authoritative_result -from portscanner_scanner.result import RawArtifact, build_scan_result_payload -from portscanner_scanner.xml import ( - NmapXmlError, - discovery_is_complete, - extract_open_tcp_ports, - parse_xml_bytes, - validate_complete_scan, -) - - -def test_discovery_completeness_and_open_port_extraction(tmp_path): - path = tmp_path / "raw.xml" - path.write_bytes( - nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("closed", 2),), - ) - ) - coverage = PortCoverage.parse("80-82") - - report = validate_complete_scan( - path, - target=DOCUMENTATION_TARGET, - coverage=coverage, - ) - assert report.open_ports == frozenset({80}) - assert report.represented_port_count == 3 - assert extract_open_tcp_ports(path) == frozenset({80}) - assert discovery_is_complete( - path, - target=DOCUMENTATION_TARGET, - coverage=coverage, - ) - - -@pytest.mark.parametrize( - "raw", - [ - nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("closed", 1),), - ), - nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("closed", 2),), - finished=False, - ), - nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("closed", 2),), - total_hosts=2, - ), - nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("open", 2),), - ), - ], -) -def test_incomplete_or_ambiguous_discovery_is_rejected(tmp_path, raw): - path = tmp_path / "raw.xml" - path.write_bytes(raw) - with pytest.raises(NmapXmlError): - validate_complete_scan( - path, - target=DOCUMENTATION_TARGET, - coverage=PortCoverage.parse("80-82"), - ) - - -def test_host_timeout_is_incomplete(tmp_path): - path = tmp_path / "raw.xml" - raw = nmap_xml( - explicit_states={80: "closed"}, - ).replace(b"", b'', 1) - path.write_bytes(raw) - - assert not discovery_is_complete( - path, - target=DOCUMENTATION_TARGET, - coverage=PortCoverage.parse("80"), - ) - - -def test_defused_parser_rejects_entity_expansion(): - raw = b']>&e;' - with pytest.raises(NmapXmlError): - parse_xml_bytes(raw) - - -def _envelope(raw: bytes, *, complete: bool) -> dict: - shared_target = contract_target() - timestamp = datetime(2026, 1, 1, tzinfo=UTC) - artifact = RawArtifact( - bucket="configured-results", - key="prefix/raw.xml", - sha256=hashlib.sha256(raw).hexdigest(), - ) - error = ( - None - if complete - else { - "type": "discovery_incomplete", - "message": "Nmap discovery XML did not prove complete coverage", - "retryable": True, - } - ) - return build_scan_result_payload( - event_id="1" * 64, - directive_id="2" * 64, - trace_id="trace-001", - run_id="run-001", - attempt_id="attempt-001", - contract_target=shared_target, - target=DOCUMENTATION_TARGET, - target_id=shared_target.target_id, - target_generation=3, - profile="targeted-tcp", - image_version=f"sha256:{'a' * 64}", - coverage=PortCoverage.parse("80-82"), - complete=complete, - scanned_coverage_complete=complete, - open_ports=(80,) if complete else (), - started_at=timestamp, - completed_at=timestamp, - result_uploaded_at=timestamp, - discovery_command={ - "argv": ["nmap", "-oX", "", ""], - "timeout_seconds": 60, - "exit_code": 0 if complete else None, - }, - enrichment_command=None, - outcome="complete" if complete else "partial", - exit_code=0 if complete else None, - raw_discovery=artifact, - raw_enrichment=None, - error=error, - ) - - -def test_parser_uses_envelope_correlation_and_hash_checks_xml(): - raw = nmap_xml( - explicit_states={80: "open"}, - collapsed_states=(("closed", 2),), - root_attributes='event_id="forged" trace_id="forged"', - ) - parsed = parse_authoritative_result(_envelope(raw, complete=True), raw) - - assert parsed["event_id"] == "1" * 64 - assert parsed["trace_id"] == "trace-001" - assert parsed["observations"] == [ - { - "protocol": "tcp", - "port": 80, - "state": "OPEN", - "nmap_state": "open", - } - ] - assert parsed["collapsed_observations"][0]["state"] == "CLOSED" - - with pytest.raises(ResultIntegrityError, match="SHA-256"): - parse_authoritative_result(_envelope(raw, complete=True), raw + b" ") - - -def test_partial_xml_never_proves_closure(): - raw = nmap_xml( - explicit_states={80: "closed", 81: "open"}, - finished=False, - ) - parsed = parse_authoritative_result(_envelope(raw, complete=False), raw) - states = {item["port"]: item["state"] for item in parsed["observations"]} - - assert states == {80: "UNKNOWN", 81: "OPEN"} - assert parsed["collapsed_observations"] == [] diff --git a/terraform/aws/scripts/build-images.sh b/terraform/aws/scripts/build-images.sh index fd8e8a8..e587b87 100755 --- a/terraform/aws/scripts/build-images.sh +++ b/terraform/aws/scripts/build-images.sh @@ -150,7 +150,7 @@ CONTEXTS=( . . . - operator + . . ) diff --git a/tools/sanitize-policy.toml b/tools/sanitize-policy.toml index f885e74..4238434 100644 --- a/tools/sanitize-policy.toml +++ b/tools/sanitize-policy.toml @@ -211,13 +211,9 @@ id = "public-owner-name" pattern = '''(?i)\broblox\b''' allow_paths = [ "LICENSE", - "NOTICE", - "NOTICE.md", "LICENSES/**", "THIRD_PARTY_NOTICES.md", "**/LICENSE", - "**/NOTICE", - "**/NOTICE.md", ] allow_patterns = [ '''(?:https://)?github\.com/Roblox/portscanner(?:/[A-Za-z0-9._~/%#?=:+-]*)?''', diff --git a/tools/tests/test_release_packaging.py b/tools/tests/test_release_packaging.py index 162b1a9..6542dda 100644 --- a/tools/tests/test_release_packaging.py +++ b/tools/tests/test_release_packaging.py @@ -72,10 +72,9 @@ def test_component_metadata_packages_the_repository_license(self) -> None: def test_operator_image_packages_repository_legal_files(self) -> None: operator_root = ROOT / "operator" self.assertEqual((operator_root / "LICENSE").read_bytes(), (ROOT / "LICENSE").read_bytes()) - self.assertEqual( - (operator_root / "THIRD_PARTY_NOTICES.md").read_bytes(), - (ROOT / "THIRD_PARTY_NOTICES.md").read_bytes(), - ) + operator_notices = (operator_root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") + self.assertIn("## Go operator", operator_notices) + self.assertIn("/licenses/Apache-2.0.txt", operator_notices) dockerfile = (operator_root / "Dockerfile").read_text(encoding="utf-8") self.assertIn('org.opencontainers.image.licenses="MIT"', dockerfile) self.assertIn("COPY --from=builder /workspace/LICENSE /licenses/LICENSE", dockerfile) @@ -84,6 +83,11 @@ def test_operator_image_packages_repository_legal_files(self) -> None: "/licenses/THIRD_PARTY_NOTICES.md", dockerfile, ) + self.assertIn( + "COPY --from=builder /workspace/third-party-licenses/Apache-2.0.txt " + "/licenses/Apache-2.0.txt", + dockerfile, + ) def test_runtime_exports_are_exact_hash_pins(self) -> None: for component in COMPONENTS: @@ -145,14 +149,12 @@ def test_docker_context_defenses_exclude_local_secrets_and_ide_files(self) -> No "**/credentials.json", "**/*.pem", } - for relative_path in (".dockerignore", "operator/.dockerignore"): - with self.subTest(path=relative_path): - patterns = { - line.strip() - for line in (ROOT / relative_path).read_text(encoding="utf-8").splitlines() - if line.strip() and not line.startswith("#") - } - self.assertTrue(required_patterns <= patterns) + patterns = { + line.strip() + for line in (ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines() + if line.strip() and not line.startswith("#") + } + self.assertTrue(required_patterns <= patterns) def test_container_workflow_watches_shared_build_inputs(self) -> None: workflow = (ROOT / ".github/workflows/containers.yml").read_text(encoding="utf-8") From 1bf59b0fb2c43a445ff6a380c6880667651ede73 Mon Sep 17 00:00:00 2001 From: Qiancheng Wu Date: Wed, 12 Aug 2026 21:40:32 -0700 Subject: [PATCH 3/4] Fix portable Terraform CI checks Use a platform-independent permission check in the bootstrap test and explicitly set the canonical operator namespace expected by the deployment contract. Co-authored-by: Cursor --- terraform/aws/deployment/main.tf | 1 + terraform/aws/scripts/tests/bootstrap-test.sh | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/terraform/aws/deployment/main.tf b/terraform/aws/deployment/main.tf index e660fc9..e4b7c62 100644 --- a/terraform/aws/deployment/main.tf +++ b/terraform/aws/deployment/main.tf @@ -155,6 +155,7 @@ module "portscanner" { image_digests = var.image_digests migration_checksum = var.migration_checksum + eks_namespace = "portscanner-system" eks_endpoint_public_access = true eks_public_access_cidrs = [var.environment.runner.restricted_public_cidr] eks_installer_principal_arns = [var.environment.runner.eks_installer_principal_arn] diff --git a/terraform/aws/scripts/tests/bootstrap-test.sh b/terraform/aws/scripts/tests/bootstrap-test.sh index 12b6604..22558b7 100755 --- a/terraform/aws/scripts/tests/bootstrap-test.sh +++ b/terraform/aws/scripts/tests/bootstrap-test.sh @@ -10,6 +10,17 @@ fail() { exit 1 } +file_mode() { + python3 - "$1" <<'PY' +import pathlib +import stat +import sys + +mode = stat.S_IMODE(pathlib.Path(sys.argv[1]).stat().st_mode) +print(f"{mode:03o}") +PY +} + WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/portscanner-bootstrap-test.XXXXXX")" cleanup() { rm -rf -- "${WORK_DIR}" @@ -304,7 +315,7 @@ run_bootstrap >"${WORK_DIR}/first-output" 2>"${WORK_DIR}/first-error" || for generated_file in "${BOOTSTRAP_STATE}" "${BACKEND_CONFIG}" "${IMAGE_INPUTS}"; do [[ -f "${generated_file}" ]] || fail "bootstrap did not generate ${generated_file}" - GENERATED_MODE="$(stat -f '%Lp' "${generated_file}" 2>/dev/null || stat -c '%a' "${generated_file}")" + GENERATED_MODE="$(file_mode "${generated_file}")" [[ "${GENERATED_MODE}" == "600" ]] || fail "generated file was not mode 0600: ${generated_file}" done From cf97d500f1127039e21c777847e3f0d01ad54a68 Mon Sep 17 00:00:00 2001 From: Qiancheng Wu Date: Wed, 12 Aug 2026 22:31:41 -0700 Subject: [PATCH 4/4] Fix GNU stat output handling Capture the BSD permission probe before falling back to GNU stat, and exercise the Linux behavior explicitly in the canary helper tests. Co-authored-by: Cursor --- terraform/aws/scripts/retire-canary.sh | 9 ++++++--- .../aws/scripts/tests/canary-helpers-test.sh | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/terraform/aws/scripts/retire-canary.sh b/terraform/aws/scripts/retire-canary.sh index 9f0dfc9..e12c328 100755 --- a/terraform/aws/scripts/retire-canary.sh +++ b/terraform/aws/scripts/retire-canary.sh @@ -18,10 +18,13 @@ require_command() { } file_mode() { - if stat -f '%Lp' "$1" 2>/dev/null; then - return + local path="$1" + local mode + if mode="$(stat -f '%Lp' "${path}" 2>/dev/null)"; then + printf '%s' "${mode}" + else + stat -c '%a' "${path}" fi - stat -c '%a' "$1" } ROOT="" diff --git a/terraform/aws/scripts/tests/canary-helpers-test.sh b/terraform/aws/scripts/tests/canary-helpers-test.sh index df0de9b..11d0689 100755 --- a/terraform/aws/scripts/tests/canary-helpers-test.sh +++ b/terraform/aws/scripts/tests/canary-helpers-test.sh @@ -88,6 +88,24 @@ mkdir -p "${WORK_DIR}/root" "${WORK_DIR}/bin" touch "${WORK_DIR}/root/main.tf" "${WORK_DIR}/backend.hcl" "${WORK_DIR}/images.tfvars" chmod 600 "${WORK_DIR}/backend.hcl" "${WORK_DIR}/images.tfvars" +cat >"${WORK_DIR}/bin/stat" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "-f" ]]; then + echo "simulated GNU stat filesystem output" + exit 1 +fi +[[ "${1:-}" == "-c" && "${2:-}" == "%a" && $# -eq 3 ]] +python3 - "$3" <<'PY' +import pathlib +import stat +import sys + +print(f"{stat.S_IMODE(pathlib.Path(sys.argv[1]).stat().st_mode):03o}") +PY +EOF +chmod 755 "${WORK_DIR}/bin/stat" + cat >"${WORK_DIR}/bin/terraform" <<'EOF' #!/usr/bin/env bash set -euo pipefail