From 8e6f31c50a3413ed8f878fb7eff68b1e979d7d50 Mon Sep 17 00:00:00 2001 From: incendiary Date: Fri, 14 Aug 2026 12:54:05 +0800 Subject: [PATCH] fix: preserve all cloud attributions --- README.md | 9 +- ROADMAP.md | 74 ++++++++++++++ classes/allocator_contract.py | 14 ++- classes/csp_ip_addresses.py | 17 ++-- docs/ALLOCATOR-CONTRACT.md | 8 +- docs/EXTERNAL-ACCEPTANCE.md | 77 +++++++++++++++ imports/cloud_ip_ranges.py | 59 +++++++++-- imports/cloud_service_provider_checks.py | 53 +++++----- resolver.py | 2 +- tests/conftest.py | 26 +++-- tests/test_allocator_contract.py | 14 +++ tests/test_allocator_publisher.py | 12 ++- tests/test_cloud_ip_ranges.py | 104 ++++++++++++++++++-- tests/test_cloud_service_provider_checks.py | 81 ++++++++++++--- tests/test_csp_ip_addresses.py | 21 ++++ 15 files changed, 484 insertions(+), 87 deletions(-) create mode 100644 docs/EXTERNAL-ACCEPTANCE.md diff --git a/README.md b/README.md index 8559788..52b41b3 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,8 @@ Each run creates a timestamped subdirectory under the output directory containin | `resolution_results_*.txt` | Successfully resolved domains and their IPv4/IPv6 addresses, pipe-delimited (`domain\|ip1\|ip2`). Prefixed `WILDCARD\|` (confirmed catch-all) or `WILDCARD_ZONE\|` (zone answers for anything, unverifiable) — see [Wildcard DNS detection](#wildcard-dns-detection) | | `unresolved_results_*.txt` | Domains that could not be resolved after all retries | | `takeover_candidates_*.txt` | `DANGLING\|origin\|target\|category\|recommendation\|evidence\|hops\|chain` — the chain records the full CNAME path (`a -> b -> c`), so the claimable hop is visible without re-resolving. Plus `NS_TAKEOVER\|` lines for unresolvable nameservers | -| `csp_matches_*.txt` | One handoff record per matched address: `domain\|ip\|provider\|region\|service\|prefix\|border_group`. Prefixed `WILDCARD\|` when the resolution was a catch-all. See [Cloud IP attribution](#cloud-ip-attribution) | -| `allocator-targets-v1.json` | Versioned provider-aware allocator handoff. Groups service, prefix, and border-group metadata per provider/hostname/address/region and excludes wildcard observations. | +| `csp_matches_*.txt` | One handoff record per provider-published attribution: `domain\|ip\|provider\|region\|service\|prefix\|border_group`. One address can produce several records. Prefixed `WILDCARD\|` when the resolution was a catch-all. See [Cloud IP attribution](#cloud-ip-attribution) | +| `allocator-targets-v1.json` | Versioned provider-aware allocator handoff. Groups service, prefix, and border-group metadata per provider/hostname/address/region and excludes wildcard observations; one address may produce multiple records when a provider publishes multiple regions. | | `environment_results_*.json` | Run metadata (command, external IP, Docker status) | | `provider_catalogues.json` | AWS, GCP, and Azure catalogue status, source, retrieval time, snapshot identifier, and any failure reason | | `{provider}_ip_ranges.json` | Validated provider ranges and provenance used by this run, from either a live source or a fresh cache | @@ -161,8 +161,9 @@ little: of roughly 10,500 published AWS prefixes, over half carry the generic `A and the ones that matter operationally — `EC2` in a named region — look identical unless the provider's own metadata is kept. -Each match is therefore written as a record carrying the provider's published region and -service: +Each attribution is therefore written as a record carrying the provider's published region and +service. Providers can publish the same prefix under multiple services or regions, and an address +can fall within overlapping prefixes, so one address may produce several records: ``` domain|ip|provider|region|service|prefix|border_group diff --git a/ROADMAP.md b/ROADMAP.md index c8d9e58..e03830c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,6 +2,80 @@ Single source of truth for planned work. +## Goal-aligned roadmap — agreed August 2026 + +This section supersedes the priority claims in the historical roadmap below. The +older material remains as delivery history, but it was largely derived from a +review performed against the wrong product goal. + +### Confirmed goal + +DNSResolver is the DNS-only front end for a separate cloud-address reclaim +allocator. Cloud IP candidates are the primary product; dangling CNAME and NS +findings are secondary observations. AWS, GCP, and Azure are required providers. +The tool retains complete DNS observations, excludes wildcard, uncertain, and +incomplete results from actionable output, and fails closed. It must work +correctly before it is optimised. Lambda expansion is shelved. + +### Delivery sequence + +Each item is one independently verifiable PR. Branch from current `main`, run the +exact local CI workflow, validate resolver changes against real DNS/provider data, +then confirm GitHub jobs `test`, `gitleaks`, and `trufflehog` before squash merge. + +| # | Work item | State | Done when | +|---|---|---|---| +| 1 | Scope secret scans to changed commits | Merged, PR 160 | New-branch push and PR scans use a valid merge-base range; all actions are SHA-pinned; both event paths pass. | +| 2 | Define the resolver-to-allocator contract | Merged, PR 159 | Schemas, examples, fail-closed invariants, current AWS consumer compatibility, and documentation drift tests pass. | +| 3 | Make DNS resolution reliable | Merged, PR 161 | Timeouts/nameservers reach both resolver libraries; A and AAAA fallback results survive; final retry state is correct; public dual-stack, IPv6-only, and negative controls behave correctly. | +| 4 | Fail closed on incomplete provider catalogues | Merged, PR 163 | AWS/GCP/Azure use bounded retries, validation, provenance, freshness-limited integrity-checked snapshots, and explicit states; no DNS processing occurs with an unusable provider. | +| 5 | Publish the versioned allocator target document | Merged, PR 164 | A successful run atomically publishes schema-valid provider-aware JSON, groups repeated metadata, excludes wildcard observations, and retains the legacy pipe output. | +| 6 | Pin Lambda input to the triggering object version | Merged, PR 165 | The reference handler reads the exact S3 object version from the event and rejects incomplete version information. This does not reopen Lambda expansion. | +| 7 | Preserve every provider attribution | Implemented on `fix/preserve-cloud-attribution`; PR pending | Duplicate services, overlapping prefixes, identical CIDRs across providers, and multiple provider-published regions survive catalogue parsing, matching, pipe output, and JSON publication without changing the schema. Legacy scalar cache snapshots remain readable. | +| 8 | Replace linear cloud-range matching with an indexed matcher | Deferred until measured need | A deterministic benchmark is defined before implementation; outputs are byte-for-byte equivalent to item 7; measured runtime and memory are reported at representative scale. | +| 9 | Make output publication atomic and observable | Deferred; not an initial-workability gate | Required writes cannot be swallowed; partial runs cannot leave a stale actionable document; injected open/write/replace failures produce a nonzero, explicit failure with regression tests. | +| 10 | Complete observation/run-manifest publication | Deferred; not an initial-workability gate | The checked observation and manifest contracts are emitted by real runs; manifest state reflects provider completeness and publication outcome; actionable output is null for incomplete/failed runs. | +| 11 | Validate the real allocator consumer end to end | Planned, cross-repository | The AWS consumer ingests a current DNSResolver document unchanged; GCP/Azure route only to provider-aware implementations or are explicitly rejected; a synthetic authorized fixture proves no provider is misrouted. | +| 12 | Measure and calibrate large-run behavior | Planned last | A repeatable representative benchmark replaces the unmeasured README scalability claim; resource limits and operational guidance reflect measured results. | + +### Explicitly not doing + +- No cloud allocation or reclaim API calls in DNSResolver. +- No HTTP probing, TLS inspection, screenshots, or active takeover confirmation. +- No Lambda expansion, deployment machinery, release, tag, or version bump in an + individual work-item PR. +- No matcher-performance refactor inside attribution item 7; correctness is frozen + first so optimisation has a trustworthy equivalence oracle. +- No separate security-hardening phase. Items 8-10 are optional reliability and + performance follow-ups, deferred until measured operational need. +- No claim that unit coverage proves resolver behavior; real DNS, real provider + catalogues, consumer validation, and actual merge CI remain separate gates. + +### Delivery state — 2026-08-14 + +Item 7 is implemented on `fix/preserve-cloud-attribution` and ready for its +focused pull request. + +- Focused suite: 91 tests passed. +- Full local CI: Ruff and format passed; baseline 5 passed; full suite 310 passed + at 95% coverage for `classes` and `imports`. +- Live catalogues: 4,826 AWS prefixes and 44,225 Azure prefixes have multiple + attributions; Azure has 44,101 prefixes with multiple published regions. +- Live end-to-end synthetic-address acceptance used current provider catalogues: + AWS preserved `AMAZON`, `EC2`, and `S3`; Azure preserved `global` and + `southeastasia` as separate schema-valid targets. +- A production run through the host system resolver processed `s3.amazonaws.com` + and `aws.amazon.com` using current AWS, GCP, and Azure catalogues. It emitted 43 + pipe attributions and 20 schema-valid allocator targets; one current S3 address + retained `AMAZON`, `EC2`, and `S3`. Direct public-resolver transport and the real + allocator consumer remain external acceptance gates. +- Before delivery: review the diff, run TruffleHog v3.96.0 on the intended commit + range, commit, push, open one focused PR, and verify push, PR, and post-merge + `main` checks. +- The live gate may be delegated using `docs/EXTERNAL-ACCEPTANCE.md`; its report + must keep automated, catalogue, system-resolver, public-resolver, and allocator + evidence separate. + > **On its derivation.** The plan below came from [`REVIEW.md`](REVIEW.md) (2026-07-15), which > assessed the tool against a misread goal — cloud attribution treated as a supporting attribute > rather than as one of two co-equal products. That review now carries a correction, and the work diff --git a/classes/allocator_contract.py b/classes/allocator_contract.py index ebada43..61a8576 100644 --- a/classes/allocator_contract.py +++ b/classes/allocator_contract.py @@ -23,14 +23,11 @@ def publish_allocator_targets(csp_path, output_dir): target["provider"], target["hostname"], target["ip"], + target["region"], ) if identity not in grouped: grouped[identity] = target continue - if grouped[identity]["region"] != target["region"]: - raise ValueError( - f"Conflicting regions for one allocator target at line {line_number}" - ) for field in ("services", "prefixes", "network_border_groups"): grouped[identity][field].update(target[field]) @@ -44,7 +41,14 @@ def publish_allocator_targets(csp_path, output_dir): "network_border_groups": sorted(target["network_border_groups"]), } ) - targets.sort(key=lambda item: (item["provider"], item["hostname"], item["ip"])) + targets.sort( + key=lambda item: ( + item["provider"], + item["hostname"], + item["ip"], + item["region"], + ) + ) temporary = destination.with_suffix(".json.tmp") try: diff --git a/classes/csp_ip_addresses.py b/classes/csp_ip_addresses.py index 5fca482..565d78d 100644 --- a/classes/csp_ip_addresses.py +++ b/classes/csp_ip_addresses.py @@ -7,7 +7,7 @@ def __init__( aws_ipv6, azure_ipv4, azure_ipv6, - metadata=None, + metadata_by_provider=None, ): self.gcp_ipv4 = gcp_ipv4 self.gcp_ipv6 = gcp_ipv6 @@ -15,19 +15,20 @@ def __init__( self.aws_ipv6 = aws_ipv6 self.azure_ipv4 = azure_ipv4 self.azure_ipv6 = azure_ipv6 - # CIDR -> (region, service). A match is only actionable downstream if the - # consumer knows where the address is allocated from and what it serves, - # so the publishers' own metadata is carried through rather than dropped. - self.metadata = metadata or {} + # Provider -> CIDR -> all published (region, service, border-group) + # attributions. Providers can publish the same CIDR, and one provider can + # publish a CIDR under multiple services, so neither dimension is scalar. + self.metadata_by_provider = metadata_by_provider or {} - def describe(self, cidr): + def describe(self, provider, cidr): """ - Region, service and network border group for a matched prefix. + Every region, service and network border group for a provider prefix. Unpublished fields read 'unknown' rather than being inferred — only AWS publishes a border group, and not every prefix carries a region. """ - return self.metadata.get(cidr, ("unknown", "unknown", "unknown")) + entries = self.metadata_by_provider.get(provider, {}).get(cidr) + return entries or [("unknown", "unknown", "unknown")] def get_gcp_ipv4(self): return self.gcp_ipv4 diff --git a/docs/ALLOCATOR-CONTRACT.md b/docs/ALLOCATOR-CONTRACT.md index b3cff6d..7b2f1cf 100644 --- a/docs/ALLOCATOR-CONTRACT.md +++ b/docs/ALLOCATOR-CONTRACT.md @@ -22,7 +22,8 @@ breaking schema change when those allocators arrive. The shared resolver pipeline publishes `allocator-targets-v1.json` after a successful run while retaining the pipe-delimited `csp_matches_*.txt` interface. -The JSON publisher groups provider metadata for identical targets and excludes +The JSON publisher groups provider metadata by provider, hostname, IP, and +provider-published region, and excludes `WILDCARD` and `WILDCARD_ZONE` observations. If any provider catalogue is unusable, or a CSP record is malformed, no allocator-target document is published. @@ -54,7 +55,10 @@ published. The array form and the `hostname`, `ip`, and `region` names are the compatibility surface used by the current AWS consumer. Consumers should deduplicate by the -combination of provider, hostname, and IP rather than assuming IP alone is unique. +combination of provider, hostname, IP, and region rather than assuming IP alone +is unique. One address may have multiple provider-published regions, particularly +when Azure publishes both a global service tag and a regional tag for the same +prefix; each region remains a separate target record. See the checked [AWS compatibility example](../contracts/examples/allocator-targets-v1.aws.json) and [multi-cloud example](../contracts/examples/allocator-targets-v1.multicloud.json). diff --git a/docs/EXTERNAL-ACCEPTANCE.md b/docs/EXTERNAL-ACCEPTANCE.md new file mode 100644 index 0000000..766995d --- /dev/null +++ b/docs/EXTERNAL-ACCEPTANCE.md @@ -0,0 +1,77 @@ +# External acceptance for cloud attribution + +This is the handoff for an agent running on a host that can reach provider +catalogue endpoints over HTTPS and public DNS resolvers over UDP/TCP port 53. +It complements the mocked automated suite; it is deliberately not a pytest +test because repository tests must never make real DNS or HTTP requests. + +## Copy/paste prompt + +```text +Take over the live acceptance of DNSResolver cloud attribution. Work from the +current pull-request branch; do not modify code unless a failed acceptance check +identifies a reproducible defect. Do not push to main, tag a release, bump +VERSION, or use real client/employer names. Use only public example/test domains +and provider-owned service names. + +Read HANDOVER.md and AGENTS.md before running anything. Then: + +1. Record `git status --short --branch` and the exact commit SHA. Stop if the + checkout contains unrelated changes. +2. Install both requirements files in Python 3.12 and run the repository's local + gates exactly: + `.venv/bin/ruff check .` + `.venv/bin/ruff format --check .` + `.venv/bin/pytest tests/baseline/ -v` + `.venv/bin/pytest --cov=classes --cov=imports --cov-report=term-missing` +3. Run the focused attribution tests: + `.venv/bin/pytest -q tests/test_allocator_contract.py tests/test_allocator_publisher.py tests/test_cloud_ip_ranges.py tests/test_cloud_service_provider_checks.py tests/test_csp_ip_addresses.py tests/test_resolver.py` +4. Run DNSResolver through its production CLI against a small file containing + public, provider-owned names. Include `s3.amazonaws.com`, which previously + resolved through the host's system resolver. Use a temporary config with no + `nameservers` key for the system-resolver run; the checked-in config supplies + explicit public resolvers and is not a system-resolver control. +5. Repeat the run with `--nameservers 1.1.1.1,8.8.8.8`. This explicit-resolver + check is mandatory on your host because the originating agent's execution + environment blocked direct public DNS. Capture whether UDP and TCP port 53 + work, rather than treating a timeout as a DNSResolver defect. +6. For every successful production run, verify: + - `provider_catalogues.json` marks aws, gcp, and azure usable and records a + source, retrieval time, and snapshot identifier; + - `csp_matches_*.txt` retains every provider-published attribution for each + matched address, including repeated services, overlapping prefixes, or + multiple regions when present; + - `allocator-targets-v1.json` validates against + `contracts/allocator-targets-v1.schema.json`; + - every JSON target has `actionability: actionable`, a provider in aws/gcp/ + azure, a non-empty provider-published region, and prefixes containing its IP; + - no `WILDCARD` or `WILDCARD_ZONE` observation enters the allocator document. +7. If the allocator checkout is available, feed the produced JSON to the real + consumer. Confirm AWS is accepted unchanged. Confirm GCP and Azure are routed + only to provider-aware implementations or are explicitly rejected; they must + never be sent through the AWS allocation path. +8. Report results as four separate evidence classes: automated tests, live + provider catalogue retrieval, system-resolver DNS, and explicit-public- + resolver DNS. Include commands, exit codes, output paths, redacted excerpts, + and the commit SHA. Do not describe a blocked or skipped check as passing. + +Success means all automated checks pass, all three catalogues are usable, at +least one real DNS answer traverses the production pipeline into a schema-valid +allocator document, and the explicit public-resolver result is evidenced. Real +allocator compatibility is a separate result: passed, failed, or unavailable. +``` + +## Evidence expected back + +- Commit SHA and clean/dirty status. +- Python and dependency versions. +- Test counts and coverage. +- Catalogue status for AWS, GCP, and Azure. +- System-resolver and explicit-resolver commands and exit codes. +- The relevant pipe records and corresponding allocator JSON records. +- JSON Schema validation result. +- Allocator-consumer result, or a precise statement that its checkout was not + available. +- Any failure separated into product defect, network/environment limitation, or + unavailable external dependency. + diff --git a/imports/cloud_ip_ranges.py b/imports/cloud_ip_ranges.py index a2001e0..89430f3 100644 --- a/imports/cloud_ip_ranges.py +++ b/imports/cloud_ip_ranges.py @@ -39,7 +39,7 @@ class ProviderCatalogue: provider: str ipv4_ranges: List[str] ipv6_ranges: List[str] - metadata: Dict[str, Tuple[str, str, str]] + metadata: Dict[str, List[Tuple[str, str, str]]] status: str usable: bool source_url: str @@ -112,6 +112,39 @@ def _validate_ranges(ipv4_ranges: List[str], ipv6_ranges: List[str]) -> None: raise CatalogueFetchError(f"invalid IPv6 prefix: {cidr}") +def _add_metadata(metadata, cidr, attribution): + entries = metadata.setdefault(cidr, []) + if attribution not in entries: + entries.append(attribution) + + +def _normalise_metadata(metadata, ranges): + if not isinstance(metadata, dict): + raise CatalogueFetchError("catalogue metadata is not an object") + normalised = {} + for cidr in ranges: + entries = metadata.get(cidr) + # Snapshots written before multi-valued metadata used one three-item + # list: [region, service, border_group]. + if ( + isinstance(entries, list) + and len(entries) == 3 + and all(isinstance(value, str) for value in entries) + ): + entries = [entries] + if not isinstance(entries, list) or not entries: + raise CatalogueFetchError(f"catalogue has no metadata for prefix: {cidr}") + for entry in entries: + if not ( + isinstance(entry, (list, tuple)) + and len(entry) == 3 + and all(isinstance(value, str) and value for value in entry) + ): + raise CatalogueFetchError(f"invalid metadata for prefix: {cidr}") + _add_metadata(normalised, cidr, tuple(entry)) + return normalised + + def _parse_standard(data: dict, extreme: bool) -> Tuple[List, List, Dict]: prefixes = data.get("prefixes") if not isinstance(prefixes, list): @@ -133,12 +166,12 @@ def _parse_standard(data: dict, extreme: bool) -> Tuple[List, List, Dict]: if keyword in prefix: cidr = prefix[keyword] ipv4_ranges.append(cidr) - metadata[cidr] = (region, service, border_group) + _add_metadata(metadata, cidr, (region, service, border_group)) for keyword in IPV6_KEYWORDS: if keyword in prefix: cidr = prefix[keyword] ipv6_ranges.append(cidr) - metadata[cidr] = (region, service, border_group) + _add_metadata(metadata, cidr, (region, service, border_group)) _validate_ranges(ipv4_ranges, ipv6_ranges) if extreme: print("IPv4 Ranges:", ipv4_ranges) @@ -165,7 +198,7 @@ def _parse_azure(data: dict, extreme: bool) -> Tuple[List, List, Dict]: service = props.get("systemService") or value.get("name") or "unknown" for cidr in prefixes: (ipv6_ranges if ":" in cidr else ipv4_ranges).append(cidr) - metadata[cidr] = (region, service, "unknown") + _add_metadata(metadata, cidr, (region, service, "unknown")) _validate_ranges(ipv4_ranges, ipv6_ranges) if extreme: print("IPv4 Ranges:", ipv4_ranges) @@ -217,13 +250,18 @@ def _load_cached_catalogue( raise CatalogueFetchError(f"cached catalogue has a future timestamp ({path})") if now - retrieved_at > MAX_CACHE_AGE[provider]: raise CatalogueFetchError(f"cached catalogue is stale ({path})") - ranges = (data["ipv4_ranges"], data["ipv6_ranges"], data["metadata"]) - _validate_ranges(ranges[0], ranges[1]) - calculated_snapshot = _snapshot_id(ranges) + raw_ranges = (data["ipv4_ranges"], data["ipv6_ranges"], data["metadata"]) + _validate_ranges(raw_ranges[0], raw_ranges[1]) + calculated_snapshot = _snapshot_id(raw_ranges) if data.get("snapshot_id") not in (None, calculated_snapshot): raise CatalogueFetchError( f"cached catalogue snapshot ID does not match ({path})" ) + ranges = ( + raw_ranges[0], + raw_ranges[1], + _normalise_metadata(raw_ranges[2], raw_ranges[0] + raw_ranges[1]), + ) catalogue = ProviderCatalogue( provider=provider, ipv4_ranges=ranges[0], @@ -233,7 +271,7 @@ def _load_cached_catalogue( usable=True, source_url=data.get("source_url") or source_url, retrieved_at=data["retrieved_at"], - snapshot_id=calculated_snapshot, + snapshot_id=_snapshot_id(ranges), ) _write_catalogue(Path(output_dir) / f"{provider}_ip_ranges.json", catalogue) return catalogue @@ -264,6 +302,11 @@ def _fetch_catalogue( if len(ranges) != 3: raise CatalogueFetchError("parser returned an incomplete catalogue") _validate_ranges(ranges[0], ranges[1]) + ranges = ( + ranges[0], + ranges[1], + _normalise_metadata(ranges[2], ranges[0] + ranges[1]), + ) catalogue = ProviderCatalogue( provider=provider, ipv4_ranges=ranges[0], diff --git a/imports/cloud_service_provider_checks.py b/imports/cloud_service_provider_checks.py index e116bed..5fd9c12 100644 --- a/imports/cloud_service_provider_checks.py +++ b/imports/cloud_service_provider_checks.py @@ -105,14 +105,16 @@ def match_ip_with_vendors(ip_obj, vendor_ips_context, domain_context, matches): ) # Keep the prefix that matched — it is the key to the region and # service the provider published for it. - matches[vendor][str(ip_obj)] = ip_range + matches[vendor].setdefault(str(ip_obj), set()).add(ip_range) def merge_matches(matches_ipv4, matches_ipv6, vendor_ips_context): - return { - vendor: {**matches_ipv4[vendor], **matches_ipv6[vendor]} - for vendor in vendor_ips_context - } + merged = {vendor: {} for vendor in vendor_ips_context} + for vendor in vendor_ips_context: + for family_matches in (matches_ipv4[vendor], matches_ipv6[vendor]): + for address, prefixes in family_matches.items(): + merged[vendor].setdefault(address, set()).update(prefixes) + return merged def log_and_write( @@ -125,13 +127,15 @@ def log_and_write( wildcard_verdict=None, ): """ - Write one line per matched address, as a handoff record for downstream + Write one line per published attribution, as a handoff record for downstream tooling: domain|ip|provider|region|service|prefix|border_group - One address per line, pipe-delimited, because this file is consumed by - another tool rather than read as prose. Region, service and border group + Each line is pipe-delimited because this file is consumed by another tool + rather than read as prose. One address can produce several lines when it + matches overlapping prefixes or a prefix has several published attributions. + Region, service and border group come from the provider's own published ranges and are what make a match actionable — an address is only worth pursuing if you know where it is allocated from and what it belongs to. @@ -148,22 +152,25 @@ def log_and_write( line_prefix = f"{wildcard_verdict}|" if wildcard_verdict else "" wrote_any = False - for ip, prefix in sorted(matched_ips.items()): - region, service, border_group = csp_ip_addresses.describe(prefix) - message = ( - f"{line_prefix}{domain}|{ip}|{vendor}|{region}|{service}" - f"|{prefix}|{border_group}" - ) + for ip, prefixes in sorted(matched_ips.items()): + for prefix in sorted(prefixes): + for region, service, border_group in sorted( + csp_ip_addresses.describe(vendor, prefix) + ): + message = ( + f"{line_prefix}{domain}|{ip}|{vendor}|{region}|{service}" + f"|{prefix}|{border_group}" + ) - # Deduplicate against an in-memory, run-scoped set of lines already - # written — avoids re-reading the whole output file on every call. - if message in written_lines: - continue + # Deduplicate against an in-memory, run-scoped set of lines already + # written — avoids re-reading the whole output file on every call. + if message in written_lines: + continue - with open(file_path, "a", encoding="utf-8") as file: - file.write(message + "\n") - written_lines.add(message) - domain_context.log_info(message) - wrote_any = True + with open(file_path, "a", encoding="utf-8") as file: + file.write(message + "\n") + written_lines.add(message) + domain_context.log_info(message) + wrote_any = True return wrote_any diff --git a/resolver.py b/resolver.py index 27d3389..6228e88 100644 --- a/resolver.py +++ b/resolver.py @@ -65,7 +65,7 @@ async def run(env_manager): aws_ipv6, azure_ipv4, azure_ipv6, - metadata={**gcp_meta, **aws_meta, **azure_meta}, + metadata_by_provider={"gcp": gcp_meta, "aws": aws_meta, "azure": azure_meta}, ) env_manager.set_domains() diff --git a/tests/conftest.py b/tests/conftest.py index 7950010..add75fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,15 +29,21 @@ # CIDR -> (region, service, network_border_group). Only AWS publishes a border # group; it is the boundary an Elastic IP is actually allocated from. CSP_METADATA = { - "34.0.0.0/8": ("europe-west2", "Google Cloud", "unknown"), - "35.0.0.0/8": ("us-central1", "Google Cloud", "unknown"), - "2600:1900::/35": ("europe-west2", "Google Cloud", "unknown"), - "3.0.0.0/8": ("eu-west-2", "EC2", "eu-west-2"), - "52.0.0.0/8": ("us-east-1", "AMAZON", "us-east-1"), - "2600:1f00::/25": ("eu-west-2", "EC2", "eu-west-2"), - "13.0.0.0/8": ("uksouth", "AzureCloud", "unknown"), - "20.0.0.0/8": ("global", "AzureCloud", "unknown"), - "2603:1000::/24": ("uksouth", "AzureCloud", "unknown"), + "gcp": { + "34.0.0.0/8": [("europe-west2", "Google Cloud", "unknown")], + "35.0.0.0/8": [("us-central1", "Google Cloud", "unknown")], + "2600:1900::/35": [("europe-west2", "Google Cloud", "unknown")], + }, + "aws": { + "3.0.0.0/8": [("eu-west-2", "EC2", "eu-west-2")], + "52.0.0.0/8": [("us-east-1", "AMAZON", "us-east-1")], + "2600:1f00::/25": [("eu-west-2", "EC2", "eu-west-2")], + }, + "azure": { + "13.0.0.0/8": [("uksouth", "AzureCloud", "unknown")], + "20.0.0.0/8": [("global", "AzureCloud", "unknown")], + "2603:1000::/24": [("uksouth", "AzureCloud", "unknown")], + }, } @@ -51,7 +57,7 @@ def csp_ips(): AWS_IPV6, AZURE_IPV4, AZURE_IPV6, - metadata=CSP_METADATA, + metadata_by_provider=CSP_METADATA, ) diff --git a/tests/test_allocator_contract.py b/tests/test_allocator_contract.py index 62e4730..dc34696 100644 --- a/tests/test_allocator_contract.py +++ b/tests/test_allocator_contract.py @@ -14,6 +14,7 @@ EXAMPLE_DIR = CONTRACT_DIR / "examples" DOC_PATH = REPO_ROOT / "docs" / "ALLOCATOR-CONTRACT.md" README_PATH = REPO_ROOT / "README.md" +EXTERNAL_ACCEPTANCE_PATH = REPO_ROOT / "docs" / "EXTERNAL-ACCEPTANCE.md" SCHEMA_EXAMPLES = { @@ -194,3 +195,16 @@ def test_observations_are_never_actionable(): observations = load_json(EXAMPLE_DIR / "dns-observations-v1.json") assert observations assert all(item["actionability"] == "excluded" for item in observations) + + +def test_external_acceptance_tracks_live_contract_and_provider_scope(): + document = EXTERNAL_ACCEPTANCE_PATH.read_text(encoding="utf-8") + + assert "contracts/allocator-targets-v1.schema.json" in document + assert "allocator-targets-v1.json" in document + assert "csp_matches_*.txt" in document + assert "provider_catalogues.json" in document + assert "--nameservers 1.1.1.1,8.8.8.8" in document + assert "system resolver" in document + assert all(provider in document for provider in ("aws", "gcp", "azure")) + assert "must never make real DNS or HTTP requests" in document diff --git a/tests/test_allocator_publisher.py b/tests/test_allocator_publisher.py index 917afc8..dcec040 100644 --- a/tests/test_allocator_publisher.py +++ b/tests/test_allocator_publisher.py @@ -78,7 +78,7 @@ def test_malformed_match_input_fails_without_publishing(line, tmp_path): assert not (tmp_path / "allocator-targets-v1.json").exists() -def test_conflicting_provider_regions_fail_closed(tmp_path): +def test_publisher_preserves_distinct_provider_regions(tmp_path): matches = write_matches( tmp_path, [ @@ -86,6 +86,10 @@ def test_conflicting_provider_regions_fail_closed(tmp_path): "api.example.com|192.0.2.10|aws|us-east-1|EC2|192.0.2.0/24|us-east-1", ], ) - with pytest.raises(ValueError, match="Conflicting regions"): - publish_allocator_targets(matches, tmp_path) - assert not (tmp_path / "allocator-targets-v1.json").exists() + targets = publish_allocator_targets(matches, tmp_path) + + assert len(targets) == 2 + assert {target["region"] for target in targets} == { + "ap-southeast-1", + "us-east-1", + } diff --git a/tests/test_cloud_ip_ranges.py b/tests/test_cloud_ip_ranges.py index c3e9c24..85d4567 100644 --- a/tests/test_cloud_ip_ranges.py +++ b/tests/test_cloud_ip_ranges.py @@ -68,8 +68,8 @@ def test_standard_parser_preserves_ranges_and_metadata(): assert ipv4 == ["3.5.140.0/22"] assert ipv6 == ["2600:1900::/28"] - assert metadata["3.5.140.0/22"] == ("eu-west-2", "EC2", "eu-west-2") - assert metadata["2600:1900::/28"] == ("global", "Google Cloud", "unknown") + assert metadata["3.5.140.0/22"] == [("eu-west-2", "EC2", "eu-west-2")] + assert metadata["2600:1900::/28"] == [("global", "Google Cloud", "unknown")] def test_aws_separate_ipv6_prefix_collection_is_loaded(): @@ -87,7 +87,33 @@ def test_aws_separate_ipv6_prefix_collection_is_loaded(): with patch("imports.cloud_ip_ranges.urlopen", return_value=_response(body=payload)): _ipv4, ipv6, metadata = fetch_ip_ranges("https://example.com/aws-ranges.json") assert ipv6 == ["2600:1f00::/40"] - assert metadata["2600:1f00::/40"] == ("us-east-1", "EC2", "us-east-1") + assert metadata["2600:1f00::/40"] == [("us-east-1", "EC2", "us-east-1")] + + +def test_duplicate_prefix_preserves_every_published_service(): + payload = { + "prefixes": [ + { + "ip_prefix": "192.0.2.0/24", + "region": "ap-southeast-1", + "service": "AMAZON", + "network_border_group": "ap-southeast-1", + }, + { + "ip_prefix": "192.0.2.0/24", + "region": "ap-southeast-1", + "service": "EC2", + "network_border_group": "ap-southeast-1", + }, + ] + } + with patch("imports.cloud_ip_ranges.urlopen", return_value=_response(body=payload)): + _ipv4, _ipv6, metadata = fetch_ip_ranges("https://example.com/aws-ranges.json") + + assert metadata["192.0.2.0/24"] == [ + ("ap-southeast-1", "AMAZON", "ap-southeast-1"), + ("ap-southeast-1", "EC2", "ap-southeast-1"), + ] def test_azure_parser_preserves_ranges_and_metadata(): @@ -101,7 +127,7 @@ def test_azure_parser_preserves_ranges_and_metadata(): assert ipv4 == ["20.26.0.0/16"] assert ipv6 == ["2603:1000::/24"] - assert metadata["20.26.0.0/16"] == ("uksouth", "AzureCloud", "unknown") + assert metadata["20.26.0.0/16"] == [("uksouth", "AzureCloud", "unknown")] def test_azure_global_region(): @@ -117,7 +143,7 @@ def test_azure_global_region(): _ipv4, _ipv6, metadata = fetch_ip_ranges_for_azure( "https://example.com/azure.json", False ) - assert metadata["13.64.0.0/16"] == ("global", "AzureCloud", "unknown") + assert metadata["13.64.0.0/16"] == [("global", "AzureCloud", "unknown")] @pytest.mark.parametrize( @@ -159,7 +185,14 @@ def test_transient_http_failure_is_retried_then_succeeds(): def test_successful_provider_fetch_records_provenance_and_snapshot(tmp_path): with patch( "imports.cloud_ip_ranges.fetch_ip_ranges", - return_value=(["34.0.0.0/8"], ["2600:1900::/28"], {}), + return_value=( + ["34.0.0.0/8"], + ["2600:1900::/28"], + { + "34.0.0.0/8": [("global", "Google Cloud", "unknown")], + "2600:1900::/28": [("global", "Google Cloud", "unknown")], + }, + ), ): catalogue = fetch_google_cloud_ip_ranges(str(tmp_path)) @@ -181,13 +214,21 @@ def _write_cache(output_dir, provider, retrieved_at=None): "provider": provider, "ipv4_ranges": ["192.0.2.0/24"], "ipv6_ranges": [], - "metadata": {}, + "metadata": { + "192.0.2.0/24": [["test-region", "test-service", "unknown"]] + }, "status": "complete", "usable": True, "source_url": f"https://example.com/{provider}.json", "retrieved_at": retrieved_at or datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "snapshot_id": _snapshot_id((["192.0.2.0/24"], [], {})), + "snapshot_id": _snapshot_id( + ( + ["192.0.2.0/24"], + [], + {"192.0.2.0/24": [["test-region", "test-service", "unknown"]]}, + ) + ), "error": None, } ) @@ -208,6 +249,41 @@ def test_live_failure_uses_fresh_cache_with_explicit_cached_state(tmp_path): assert (tmp_path / "aws_ip_ranges.json").exists() +def test_legacy_scalar_metadata_cache_is_upgraded(tmp_path): + cache = _cache_path(str(tmp_path), "aws") + cache.parent.mkdir(parents=True, exist_ok=True) + ranges = ( + ["192.0.2.0/24"], + [], + {"192.0.2.0/24": ["ap-southeast-1", "EC2", "ap-southeast-1"]}, + ) + cache.write_text( + json.dumps( + { + "provider": "aws", + "ipv4_ranges": ranges[0], + "ipv6_ranges": ranges[1], + "metadata": ranges[2], + "source_url": "https://example.com/aws.json", + "retrieved_at": datetime.now(timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "snapshot_id": _snapshot_id(ranges), + } + ) + ) + with patch( + "imports.cloud_ip_ranges.fetch_ip_ranges", + side_effect=CatalogueFetchError("offline"), + ): + catalogue = fetch_aws_ip_ranges(str(tmp_path)) + + assert catalogue.status == "cached" + assert catalogue.metadata["192.0.2.0/24"] == [ + ("ap-southeast-1", "EC2", "ap-southeast-1") + ] + + def test_live_failure_and_stale_cache_returns_unusable_state(tmp_path): stale = datetime.now(timezone.utc) - timedelta(days=2) _write_cache(tmp_path, "aws", stale.isoformat().replace("+00:00", "Z")) @@ -263,7 +339,11 @@ def test_azure_discovery_uses_current_download_url(tmp_path): patch("imports.cloud_ip_ranges.urlopen", return_value=_urlopen_html(html)), patch( "imports.cloud_ip_ranges.fetch_ip_ranges_for_azure", - return_value=(["20.0.0.0/8"], [], {}), + return_value=( + ["20.0.0.0/8"], + [], + {"20.0.0.0/8": [("global", "AzureCloud", "unknown")]}, + ), ) as fetch, ): catalogue = fetch_azure_ip_ranges(str(tmp_path)) @@ -278,7 +358,11 @@ def test_azure_discovery_failure_uses_pinned_source(tmp_path): patch("imports.cloud_ip_ranges.sleep"), patch( "imports.cloud_ip_ranges.fetch_ip_ranges_for_azure", - return_value=(["20.0.0.0/8"], [], {}), + return_value=( + ["20.0.0.0/8"], + [], + {"20.0.0.0/8": [("global", "AzureCloud", "unknown")]}, + ), ) as fetch, ): catalogue = fetch_azure_ip_ranges(str(tmp_path)) diff --git a/tests/test_cloud_service_provider_checks.py b/tests/test_cloud_service_provider_checks.py index b358c4a..bccbc55 100644 --- a/tests/test_cloud_service_provider_checks.py +++ b/tests/test_cloud_service_provider_checks.py @@ -10,6 +10,7 @@ import pytest +from classes.allocator_contract import publish_allocator_targets from classes.domain_processing_context import DomainProcessingContext from imports.cloud_service_provider_checks import ( get_ip_matches, @@ -92,10 +93,18 @@ def test_match_ip_with_vendors_records_match(ctx): assert "34.1.2.3" in matches["gcp"] # The matched prefix is retained — it is the key to region and service. - assert matches["gcp"]["34.1.2.3"] == "34.0.0.0/8" + assert matches["gcp"]["34.1.2.3"] == {"34.0.0.0/8"} assert matches["aws"] == {} +def test_match_keeps_every_overlapping_prefix(ctx): + ip_obj = ipaddress.IPv4Address("192.0.2.10") + vendor_ips = {"aws": ["192.0.2.0/24", "192.0.2.0/25"]} + matches = {"aws": {}} + match_ip_with_vendors(ip_obj, vendor_ips, ctx, matches) + assert matches["aws"]["192.0.2.10"] == {"192.0.2.0/24", "192.0.2.0/25"} + + def test_match_ip_with_vendors_no_match(ctx): ip_obj = ipaddress.IPv4Address("1.2.3.4") vendor_ips = {"gcp": ["34.0.0.0/8"]} @@ -124,13 +133,13 @@ def test_match_ip_with_vendors_ignores_invalid_range(ctx): def test_merge_matches_combines_ipv4_and_ipv6(): """Both families merge, each address keeping the prefix it matched.""" - v4 = {"gcp": {"1.2.3.4": "1.2.0.0/16"}, "aws": {}} - v6 = {"gcp": {"::1": "::/64"}, "aws": {}} + v4 = {"gcp": {"1.2.3.4": {"1.2.0.0/16"}}, "aws": {}} + v6 = {"gcp": {"::1": {"::/64"}}, "aws": {}} vendor_context = {"gcp": [], "aws": []} result = merge_matches(v4, v6, vendor_context) - assert result["gcp"] == {"1.2.3.4": "1.2.0.0/16", "::1": "::/64"} + assert result["gcp"] == {"1.2.3.4": {"1.2.0.0/16"}, "::1": {"::/64"}} assert result["aws"] == {} @@ -149,7 +158,7 @@ def test_merge_matches_empty_sets(): def test_get_ip_matches_finds_gcp_ipv4(ctx, csp_ips): # 34.1.2.3 falls in GCP's 34.0.0.0/8 range (from conftest) result = get_ip_matches(["34.1.2.3"], get_vendor_ips(ctx, 4), ctx, ip_version=4) - assert result["gcp"]["34.1.2.3"] == "34.0.0.0/8" + assert result["gcp"]["34.1.2.3"] == {"34.0.0.0/8"} def test_get_ip_matches_skips_wrong_ip_version(ctx): @@ -169,7 +178,7 @@ def test_log_and_write_creates_entry(tmp_path, ctx): output_files = {"standard": {"csp": str(out_file)}} result = log_and_write( - "gcp", {"34.1.2.3": "34.0.0.0/8"}, "example.com", output_files, ctx, set() + "gcp", {"34.1.2.3": {"34.0.0.0/8"}}, "example.com", output_files, ctx, set() ) assert result is True @@ -190,7 +199,7 @@ def test_log_and_write_no_duplicate_entries(tmp_path, ctx): first = log_and_write( "gcp", - {"34.1.2.3": "34.0.0.0/8"}, + {"34.1.2.3": {"34.0.0.0/8"}}, "example.com", output_files, ctx, @@ -198,7 +207,7 @@ def test_log_and_write_no_duplicate_entries(tmp_path, ctx): ) second = log_and_write( "gcp", - {"34.1.2.3": "34.0.0.0/8"}, + {"34.1.2.3": {"34.0.0.0/8"}}, "example.com", output_files, ctx, @@ -211,6 +220,50 @@ def test_log_and_write_no_duplicate_entries(tmp_path, ctx): assert len(lines) == 1 +def test_log_and_write_emits_every_prefix_and_service(tmp_path, mock_env_manager): + from classes.csp_ip_addresses import CSPIPAddresses + + csp = CSPIPAddresses( + [], + [], + ["192.0.2.0/24", "192.0.2.0/25"], + [], + [], + [], + metadata_by_provider={ + "aws": { + "192.0.2.0/24": [ + ("ap-southeast-1", "AMAZON", "ap-southeast-1"), + ("ap-southeast-1", "EC2", "ap-southeast-1"), + ], + "192.0.2.0/25": [("ap-southeast-1", "EC2", "ap-southeast-1")], + } + }, + ) + context = DomainProcessingContext(mock_env_manager, csp) + context.set_domain("api.example.com") + output = tmp_path / "csp.txt" + output.touch() + wrote = log_and_write( + "aws", + {"192.0.2.10": {"192.0.2.0/24", "192.0.2.0/25"}}, + "api.example.com", + {"standard": {"csp": str(output)}}, + context, + set(), + ) + assert wrote is True + assert output.read_text().splitlines() == [ + "api.example.com|192.0.2.10|aws|ap-southeast-1|AMAZON|192.0.2.0/24|ap-southeast-1", + "api.example.com|192.0.2.10|aws|ap-southeast-1|EC2|192.0.2.0/24|ap-southeast-1", + "api.example.com|192.0.2.10|aws|ap-southeast-1|EC2|192.0.2.0/25|ap-southeast-1", + ] + + targets = publish_allocator_targets(output, tmp_path) + assert targets[0]["services"] == ["AMAZON", "EC2"] + assert targets[0]["prefixes"] == ["192.0.2.0/24", "192.0.2.0/25"] + + def test_log_and_write_no_full_file_read(tmp_path, ctx, monkeypatch): """log_and_write must not open the output file for reading — only append.""" out_file = tmp_path / "csp.txt" @@ -229,7 +282,7 @@ def guarded_open(file, mode="r", *args, **kwargs): log_and_write( "gcp", - {"34.1.2.3": "34.0.0.0/8"}, + {"34.1.2.3": {"34.0.0.0/8"}}, "example.com", output_files, ctx, @@ -252,14 +305,14 @@ def test_log_and_write_substring_line_not_suppressed(tmp_path, ctx): # (example.com = exam + ple.com), but they are different domains/lines. first = log_and_write( "gcp", - {"34.1.2.3": "34.0.0.0/8"}, + {"34.1.2.3": {"34.0.0.0/8"}}, "example.com", output_files, ctx, written_lines, ) second = log_and_write( - "gcp", {"34.1.2.3": "34.0.0.0/8"}, "ple.com", output_files, ctx, written_lines + "gcp", {"34.1.2.3": {"34.0.0.0/8"}}, "ple.com", output_files, ctx, written_lines ) assert first is True @@ -351,7 +404,11 @@ def test_parse_network_equivalence_ip_inside_and_outside_range(ctx): match_ip_with_vendors(inside_ip, vendor_ips, ctx, matches) match_ip_with_vendors(outside_ip, vendor_ips, ctx, matches) - assert matches == {"gcp": {"34.1.2.3": "34.0.0.0/8"}, "aws": {}, "azure": {}} + assert matches == { + "gcp": {"34.1.2.3": {"34.0.0.0/8"}}, + "aws": {}, + "azure": {}, + } def test_parse_network_is_memoised_across_calls(): diff --git a/tests/test_csp_ip_addresses.py b/tests/test_csp_ip_addresses.py index dded6ab..1c0a28f 100644 --- a/tests/test_csp_ip_addresses.py +++ b/tests/test_csp_ip_addresses.py @@ -65,3 +65,24 @@ def test_ranges_are_returned_by_reference(): csp = CSPIPAddresses(original, [], [], [], [], []) csp.get_gcp_ipv4().append("192.168.0.0/16") assert len(original) == 2 + + +def test_identical_cidr_metadata_is_isolated_by_provider(): + shared = "192.0.2.0/24" + csp = CSPIPAddresses( + [shared], + [], + [shared], + [], + [], + [], + metadata_by_provider={ + "gcp": {shared: [("asia-southeast1", "Google Cloud", "unknown")]}, + "aws": {shared: [("ap-southeast-1", "EC2", "ap-southeast-1")]}, + }, + ) + + assert csp.describe("gcp", shared) == [ + ("asia-southeast1", "Google Cloud", "unknown") + ] + assert csp.describe("aws", shared) == [("ap-southeast-1", "EC2", "ap-southeast-1")]