diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5982e30..358ffbb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -11,12 +11,6 @@ on: branches: - main pull_request: - paths: - - "scripts/**" - - "charts/**" - - "platform-override-values.yaml" - - "traefik-values.yaml" - - ".github/workflows/e2e.yml" concurrency: group: e2e-${{ github.ref }} @@ -42,9 +36,15 @@ jobs: run: helm repo add traefik https://traefik.github.io/charts # The scripts reach the gateways via ve1.localhost / ve2.localhost; make resolution - # deterministic instead of relying on the resolver's *.localhost handling + # deterministic instead of relying on the resolver's *.localhost handling. The issuer./ + # identity. names are the VEs' did:web hostnames — only resolved from inside the clusters + # today (setup-did-dns.sh wires that up), but listed here so any host-side check against + # a DID URL resolves the same way regardless of the tool (curl short-circuits + # *.localhost internally; getent-based tools would otherwise NXDOMAIN) - name: Resolve VE hostnames - run: echo "127.0.0.1 ve1.localhost ve2.localhost" | sudo tee -a /etc/hosts + run: | + echo "127.0.0.1 ve1.localhost issuer.ve1.localhost identity.ve1.localhost" | sudo tee -a /etc/hosts + echo "127.0.0.1 ve2.localhost issuer.ve2.localhost identity.ve2.localhost" | sudo tee -a /etc/hosts - name: Run end-to-end test env: diff --git a/README.md b/README.md index 2c94a37..d88e09a 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,11 @@ DNS domain, and each VE trusting the peer's issuer): The script's defaults match the install commands above and it verifies itself by fetching the peer issuer's DID document from inside each cluster. The routes do not survive a restart of -the kind node containers — re-run the script after a docker restart. +the kind node containers — re-run the script after a docker restart. Every request that +crosses the VE boundary (DID resolution, DCP presentation exchange, DSP messages, data-plane +token renewal) is catalogued in [docs/cross-ve-communication.md](docs/cross-ve-communication.md); +[docs/sut-verification.md](docs/sut-verification.md) builds on it to define the state +obligations and request ping-pong for verifying a third-party solution in ve2's place. Finally, exercise the federation with a cross-VE DSP exchange: the provider VE's participant offers an asset whose access and contract policy require all three credentials issued at diff --git a/app/src/main/java/com/metaform/cxve/adapter/out/cfm/ParticipantOnboardingService.java b/app/src/main/java/com/metaform/cxve/adapter/out/cfm/ParticipantOnboardingService.java index 76ef9d7..19309fb 100644 --- a/app/src/main/java/com/metaform/cxve/adapter/out/cfm/ParticipantOnboardingService.java +++ b/app/src/main/java/com/metaform/cxve/adapter/out/cfm/ParticipantOnboardingService.java @@ -40,7 +40,7 @@ public class ParticipantOnboardingService implements WalletService { private final String dataplaneTokenSource; public ParticipantOnboardingService(@Autowired TenantManagerClient tenantManagerClient, - @Value("${participant.did.template:did:web:identityhub.edc-v.svc.cluster.local%3A7083:}") String didTemplate, + @Value("${participant.did.template:did:web:identity.cxve.localhost:}") String didTemplate, @Value("${participant.dataplane.transfer-type:HttpData-PULL}") String dataplaneTransferType, @Value("${participant.dataplane.endpoint-type:HTTP}") String dataplaneEndpointType, @Value("${participant.dataplane.endpoint:}") String dataplaneEndpoint, diff --git a/app/src/main/resources/application.yaml b/app/src/main/resources/application.yaml index bb391a6..5becf07 100644 --- a/app/src/main/resources/application.yaml +++ b/app/src/main/resources/application.yaml @@ -47,10 +47,13 @@ identityhub: url: http://cxve.localhost/api/identity # IDENTITYHUB_URL # Prefix for participant DIDs when the registration doesn't supply one; the participant's short -# name is appended. %3A is the percent-encoded port colon required by did:web. +# name is appended. Must match the platform's IdentityHub did:web hostname (identity.) — +# IdentityHub resolves a DID document by the request URL, so a DID minted under any other +# authority will not resolve through the gateway. A non-default gateway port is percent-encoded +# into the authority, e.g. "did:web:identity.example.com%3A8080:". participant: did: - template: "did:web:identityhub.edc-v.svc.cluster.local%3A7083:" # PARTICIPANT_DID_TEMPLATE + template: "did:web:identity.cxve.localhost:" # PARTICIPANT_DID_TEMPLATE # Transfer-type mapping configured on the participant's data plane (Siglet) at provisioning # time, via the cfm.dataplane VPA properties. The CFM siglet agent installs the mapping and # registers the data-plane instance with the control plane; the mapping key (transfer-type) diff --git a/charts/cx-ve/values.yaml b/charts/cx-ve/values.yaml index c640fdf..52d89ec 100644 --- a/charts/cx-ve/values.yaml +++ b/charts/cx-ve/values.yaml @@ -126,10 +126,14 @@ config: identityhub: url: http://identityhub.edc-v.svc.cluster.local:7081/api/identity/v1beta # Prefix for participant DIDs when the registration doesn't supply one; the participant's - # short name is appended. %3A is the percent-encoded port colon required by did:web. + # short name is appended. Must match the platform's IdentityHub did:web hostname + # (identity.): IdentityHub reconstructs the DID from the request URL and looks it up by + # exact match, so a DID minted under any other authority will not resolve through the gateway. + # A non-default gateway port belongs in the authority percent-encoded, e.g. + # "did:web:identity.example.com%3A8080:". participant: did: - template: "did:web:identityhub.edc-v.svc.cluster.local%3A7083:" + template: "did:web:identity.cxve.localhost:" # Transfer-type mapping installed on the participant's data plane (Siglet) at provisioning. # The mapping key (transfer-type) becomes the catalog distribution format of the # participant's assets; an empty endpoint disables data-plane registration. diff --git a/docs/cross-ve-communication.md b/docs/cross-ve-communication.md new file mode 100644 index 0000000..493c015 --- /dev/null +++ b/docs/cross-ve-communication.md @@ -0,0 +1,175 @@ +# Cross-VE communication + +Every network request that crosses the boundary between two connected VEs (as set up by +`scripts/install-ve.sh` + `scripts/connect-ves.sh`, exercised end to end by +`scripts/dsp-tests.sh`). Concrete hostnames/ports below use the dual-VE convention — ve1 +(`ve1.local`) in the provider role, ve2 (`ve2.local`) in the consumer role — but every flow is +symmetric: whichever VE initiates a catalog request is the consumer for that exchange. + +A key architectural fact up front: **cross-VE traffic never passes through the gateways** +(Traefik/clearglass). All of it is pod-to-pod HTTP between in-cluster FQDNs +(`.edc-v.svc.:`), riding on the static routes and DNS forwarding below. +DSP and DCP bring their own authentication (self-issued ID tokens + verifiable presentations), +so the platform's edge auth chain is not involved between VEs. + +The surface is deliberately exactly **DCP (layers 1–2) and DSP plus the agreed data-plane +profile (layers 3–4)** — that is the complete wire contract a third-party implementation must +speak to interoperate. Layer 0 is lab plumbing, not part of the contract. See +[sut-verification.md](sut-verification.md) for how this surface is used to verify a candidate +("system under test") that replaces ve2. + +## Layer 0 — infrastructure prerequisites (harness plumbing, set up by connect-ves.sh) + +| Flow | From → To | Purpose | +|---|---|---| +| IP routing | kind node ↔ kind node (docker `kind` network) | Static routes for the peer's pod + service CIDRs via the peer node's IP. Not a request flow itself, but every flow below depends on it. Does **not** survive a container restart. | +| DNS | CoreDNS (ve2) → kube-dns ClusterIP (ve1), port 53 UDP/TCP, and vice versa | Zone forwarding of the peer's cluster domain (`ve1.local:53 { forward . }`), so `.edc-v.svc.ve1.local` names resolve from inside ve2. Every cross-VE HTTP call is preceded by such a DNS query (subject to CoreDNS caching, `cache 30`). | + +## Layer 1 — identity resolution (DCP: did:web) + +Both control planes resolve the counterparty's DIDs by plain HTTP GET. The URLs are derived +from the DID (`did:web:%3A:` → `http://://did.json`); the +IdentityHub serves DID documents virtual-host style, so the `Host` header must match the DID. + +| # | From → To | Request | Purpose | +|---|---|---|---| +| 1 | controlplane (ve2) → identityhub (ve1) `:7083` | `GET /alphamanufacturing-/did.json` | Resolve the provider participant's DID document: verification key for its Self-Issued ID Tokens, plus its service endpoints (DSP `ProtocolEndpoint`, DCP `CredentialService`). | +| 1a | identityhub (ve2) → identityhub (ve1) `:7083` | `GET /alphamanufacturing-/did.json` | The consumer's Credential Service resolves the **verifier's** DID when validating the Self-Issued ID Token that ve1's controlplane presents with its presentation query (#5) — per DCP, the verifier's query is itself SI-token-authenticated. | +| 2 | controlplane (ve1) → identityhub (ve2) `:7083` | `GET /betalogistics-/did.json` | Same as #1, in the other direction — the provider verifies the consumer on every inbound DSP message. | +| 2a | identityhub (ve1) → identityhub (ve2) `:7083` | `GET /betalogistics-/did.json` | Mirror of #1a: ve1's Credential Service validates the SI token ve2's controlplane presents with #6. | +| 3 | controlplane (ve2) → issuerservice (ve1) `:10016` | `GET /issuer/did.json` | Resolve the peer VE's **issuer** DID to obtain the key material for verifying credential signatures inside received presentations. This is the fetch `connect-ves.sh` uses as its smoke test. Requires the issuer's DID in `edc.controlplane.trustedIssuers` *and* its `supportedtypes` ConfigMap entry (see connect-ves.sh). | +| 4 | controlplane (ve1) → issuerservice (ve2) `:10016` | `GET /issuer/did.json` | Same, other direction. | + +DID documents and presentations may be cached by implementations, so the wire-level frequency +can be lower than the logical model suggests — the *logical* model is per-message. + +The participant DID document advertises the endpoints used by the layers below (values from a +live deployment): + +```json +"service": [ + { "type": "ProtocolEndpoint", + "serviceEndpoint": "http://controlplane.edc-v.svc.ve1.local:8082/api/dsp//http-dsp-profile-2025-1" }, + { "type": "DataService", + "serviceEndpoint": "http://controlplane.edc-v.svc.ve1.local:8082/api/dsp//.well-known/dspace-version" }, + { "type": "CredentialService", + "serviceEndpoint": "http://identityhub.edc-v.svc.ve1.local:7082/api/credentials/v1/participants/" } +] +``` + +The DSP base path pattern is `/api/dsp/{participant-context-id}/{dataspace-profile}`; the +Catena-X deployment exchanges over the `cx-neptune` profile (the DID document additionally +advertises the default `http-dsp-profile-2025-1` binding). + +## Layer 2 — presentation exchange (DCP: identity & credentials) + +DCP defines **one invariant flow, identical for every DSP message** — there is nothing +catalog-, negotiation- or transfer-specific about it. DSP requires only that every request +carries an `Authorization` token (semantics delegated to the identity layer); with DCP that +token is a Self-Issued ID Token (`iss` = `sub` = sender DID, `aud` = receiver DID, plus a +`token` claim carrying an access token for the sender's Credential Service). On each inbound +DSP message the **receiver** acts as DCP Verifier: + +1. Validate the SI token — resolve the sender's DID document for the signing key (#1/#2). +2. Discover the sender's Credential Service from the DID document (`service` entry of type + `CredentialService`). +3. `POST /presentations/query` (`PresentationQueryMessage`, scope-based here) — bearing + the verifier's *own* SI token with the sender's access token embedded in the `token` + claim; validating that token is what triggers the reverse DID resolution (#1a/#2a). +4. Validate the returned Verifiable Presentations: VP signature (holder key), each VC's + issuer signature (issuer DID resolution, #3/#4, checked against the trust anchors) and, + where present, revocation status. + +The credentials checked in this deployment are Membership, BpnCredential and +DataExchangeGovernance — the demo policy requires all three: + +| # | From → To | Request | When | +|---|---|---|---| +| 5 | controlplane (ve1) → identityhub (ve2) `:7082` | `POST /api/credentials/v1/participants//presentations/query` | On every DSP request the consumer sends (catalog request, contract request, verification, transfer request). | +| 6 | controlplane (ve2) → identityhub (ve1) `:7082` | `POST /api/credentials/v1/participants//presentations/query` | On every DSP callback the provider sends (agreement, finalized event, transfer start). | + +Credential *signature* verification uses the issuer DID documents from layer 1. Credentials +carrying a `credentialStatus` reference would additionally cause the verifier to fetch the +issuer's status-list credential cross-VE; the Onboarding API's issuance flow does not +configure status lists, so no such fetches occur here. + +## Layer 3 — DSP (catalog → negotiation → transfer) + +The Dataspace Protocol is asynchronous: the consumer calls the provider's DSP endpoint, and +the provider *calls back* to the consumer's DSP endpoint (the `callbackAddress` sent in each +request — the consumer's own `/api/dsp//cx-neptune` base). Both directions target +`controlplane..svc.:8082`. + +| # | Direction | Request (relative to the peer's DSP base) | Purpose | +|---|---|---|---| +| 7 | ve2 → ve1 | `GET /.well-known/dspace-version` | Protocol version/metadata discovery (advertised as the `DataService` in the DID document). | +| 8 | ve2 → ve1 | `POST /catalog/request` | Catalog request (`dcat:Catalog` response filtered by the consumer's presented credentials — the access policy). | +| 9 | ve2 → ve1 | `POST /negotiations/request` | `ContractRequestMessage` — starts the negotiation, mirroring the offer policy verbatim. | +| 10 | ve1 → ve2 | `POST /negotiations/{consumerPid}/agreement` | `ContractAgreementMessage` callback. | +| 11 | ve2 → ve1 | `POST /negotiations/{providerPid}/agreement/verification` | `ContractAgreementVerificationMessage`. | +| 12 | ve1 → ve2 | `POST /negotiations/{consumerPid}/events` | `ContractNegotiationEventMessage` (FINALIZED). | +| 13 | ve2 → ve1 | `POST /transfers/request` | `TransferRequestMessage` (`HttpData-PULL`). | +| 14 | ve1 → ve2 | `POST /transfers/{consumerPid}/start` | `TransferStartMessage` — carries the **EDR** (data address: endpoint + provider-siglet-minted access token). The consumer control plane hands it to its siglet via data-plane signaling (intra-VE). | +| — | either | `POST /negotiations/{pid}/termination`, `POST /transfers/{pid}/termination` | Error/termination paths, when an exchange is aborted. | + +## Layer 4 — data plane (DSP HttpData-PULL profile, steady state after STARTED) + +| # | From → To | Request | Purpose | +|---|---|---|---| +| 15 | siglet (ve2) → siglet (ve1) `:8082` | `POST /token` (OAuth2-style refresh, endpoint from `SIGLET__TOKEN__REFRESH_ENDPOINT`, e.g. `http://siglet.edc-v.svc.ve1.local:8082/token`) | Automatic renewal of the EDR access token when a cached token nears expiry (`tx_renewal_support = true` for `HttpData-PULL`). The consumer siglet does this transparently for applications reading `GET /tokens/{ctx}/{transfer-id}`. | +| 16 | application → EDR endpoint | `GET ` with `Authorization: Bearer ` | The actual data pull. In the demo deployment the endpoint is the external sample source (`https://jsonplaceholder.typicode.com/todos/1`), so this request leaves both VEs; with a provider-hosted data source it would be another ve2 → ve1 flow. | + +## What does *not* cross the VE boundary + +NATS, Vault, PostgreSQL, jwtlet, clearglass/Traefik, the Tenant Manager/CFM agents and the +Onboarding API only ever talk within their own VE. Credential **issuance** is also intra-VE +(each participant's IdentityHub talks to its own VE's IssuerService); only issuer DID +*resolution* crosses over (layer 1). Management-API traffic (as driven by `dsp-tests.sh`) +enters each VE from the host through that VE's own gateway and never hops between VEs. + +## Sequence overview + +The DCP verification pattern is drawn once — it runs identically for **every** DSP arrow in +the second diagram, with Verifier = the receiver of that arrow (mirror the components for +callbacks, where ve2 verifies ve1). + +```mermaid +sequenceDiagram + participant CP2 as controlplane (ve2, sender) + participant IH2 as identityhub (ve2) + participant IS2 as issuerservice (ve2) + participant CP1 as controlplane (ve1, receiver = Verifier) + participant IH1 as identityhub (ve1) + + Note over CP2,IH1: DCP verification — identical for every DSP message + CP2->>CP1: any DSP request (Authorization: Self-Issued ID Token) + CP1->>IH2: GET /{sender-participant}/did.json :7083 (validate SI token) + CP1->>IH2: POST {cs-base}/presentations/query :7082 (verifier's SI token, embedded access token) + IH2->>IH1: GET /{verifier-participant}/did.json :7083 (CS validates the verifier's SI token) + IH2-->>CP1: PresentationResponseMessage (VPs) + CP1->>IS2: GET /issuer/did.json :10016 (verify VC issuer signatures against trust anchors) +``` + +```mermaid +sequenceDiagram + participant CP2 as controlplane (ve2, consumer) + participant SIG2 as siglet (ve2) + participant CP1 as controlplane (ve1, provider) + participant SIG1 as siglet (ve1) + + Note over CP2,CP1: catalog + CP2->>CP1: POST /catalog/request (DSP :8082) + CP1-->>CP2: dcat:Catalog (filtered by access policy) + + Note over CP2,CP1: negotiation + CP2->>CP1: POST /negotiations/request + CP1->>CP2: POST /negotiations/{pid}/agreement + CP2->>CP1: POST /negotiations/{pid}/agreement/verification + CP1->>CP2: POST /negotiations/{pid}/events (FINALIZED) + + Note over CP2,SIG1: transfer + data + CP2->>CP1: POST /transfers/request (HttpData-PULL) + CP1->>CP2: POST /transfers/{pid}/start (EDR) + CP2->>SIG2: DPS on_started (intra-VE) — EDR cached + SIG2->>SIG1: POST /token :8082 (renewal, near expiry) +``` diff --git a/docs/sut-verification.md b/docs/sut-verification.md new file mode 100644 index 0000000..7c92a4a --- /dev/null +++ b/docs/sut-verification.md @@ -0,0 +1,122 @@ +# Verifying a third-party solution against the reference VE + +The long-term purpose of the dual-VE setup: ve2 is a stand-in for a **system under test** +(SUT, also "candidate" or "contestant") — any third-party dataspace implementation. The +reference VE (ve1: Core Platform + Catena-X profile) verifies the SUT purely over the wire. + +Two principles follow: + +1. **The only interactions between the clusters are DSP and DCP.** The complete request + catalogue lives in [cross-ve-communication.md](cross-ve-communication.md): layers 1–2 are + DCP (did:web resolution + presentation exchange), layers 3–4 are DSP and the agreed + data-plane profile (HttpData-PULL incl. EDR token refresh). Anything else crossing the + boundary is a harness artifact, not part of the surface — a candidate that needs more than + DSP + DCP to interoperate fails by construction. +2. **How the SUT reaches a required state is immaterial.** Seeding assets, issuing + credentials, provisioning wallets, registering data planes — the SUT does this with + whatever management APIs, consoles or scripts it has. Verification only requires that at + defined checkpoints the SUT cluster is in a *valid state* so that ve1 can fire requests + against it (or observe the SUT's requests arriving). The reference harness drives ve2 + through the platform's own APIs today, but none of that tooling is part of the contract. + +## SUT registration (inputs the harness needs up front) + +Because the harness may not introspect the SUT, the SUT declares: + +| Input | Used for | +|---|---| +| Participant DID (resolvable from ve1) | Counterparty identity in DSP requests; DID document must advertise the DSP `ProtocolEndpoint` and DCP `CredentialService` | +| Issuer DID (if the SUT brings its own issuer) | ve1's trust anchor config: `edc.controlplane.trustedIssuers` **plus** the `supportedtypes` entry (see connect-ves.sh — without it presentations fail with "credential types not supported for issuer") | +| BPN of the SUT participant | Pinning the `BusinessPartnerNumber` policy constraint in offers made to the SUT | +| Supported DSP profile | Must include the dataspace profile in use (`cx-neptune`) | +| Network reachability + DNS | ve1 must resolve and reach the SUT's endpoints; the kind-specific routes/CoreDNS forwarding of connect-ves.sh are the lab instantiation of this | + +Conversely the harness publishes ve1's participant DID, issuer DID and gateway-independent +DSP/DCP endpoints to the SUT. + +## Checkpoints and obligations + +Each scenario states: what the SUT must have done beforehand (state obligations — *how* is +its business), what ve1 does, and which wire exchanges occur (numbers reference the tables in +[cross-ve-communication.md](cross-ve-communication.md)). + +### Checkpoint 0 — discovery & identity + +**SUT obligations (state):** +- Participant DID document served and resolvable from ve1 (#1), advertising `ProtocolEndpoint` + and `CredentialService`. +- Issuer DID document resolvable from ve1 (#3), if the SUT brings its own issuer. + +**Verified by:** ve1 resolving both DID documents. No DSP traffic yet. + +### Checkpoint 1 — credentials & trust + +**SUT obligations (state):** +- The SUT participant *holds* the three Catena-X credentials — `MembershipCredential` + (`memberOf == "Catena-X"`), `BpnCredential` (matching its declared BPN) and + `DataExchangeGovernanceCredential` (`contractVersion == "1.0.0"`) — issued by an issuer ve1 + trusts. Topology is the SUT's choice: its own (registered) issuer, or credentials issued by + ve1's issuer service via DCP issuance (the more complete conformance target, not yet part of + the harness). +- Its CredentialService answers DCP presentation queries (#5) for those credentials under the + Catena-X scope mapping. +- The SUT trusts ve1's issuer in return, so it can verify ve1's presentations (#6, #4). + +**Verified by:** implicitly in every subsequent scenario — presentation exchange happens on +each DSP message. A dedicated probe is possible (fire a catalog request and check the SUT +verifies ve1's VP and vice versa) before asserting anything content-related. + +### Checkpoint 2 — SUT as provider (ve1 fires the requests) + +**SUT obligations (state):** +- An agreed-upon asset exists with an `HttpData-PULL` distribution and a backing data source. +- Access + contract policy gate the asset on the three CX credential constraints + (`Membership`, `FrameworkAgreement == DataExchangeGovernance:1.0.0`, + `BusinessPartnerNumber == `). +- A contract definition exposes the asset in the catalog for authorized consumers. +- Its data plane can mint/serve EDR tokens and (if advertised) support token refresh. + +**ve1 actions & observed ping-pong** (ve1 = consumer, direction of #7–#14 reversed): +1. Catalog request → offer for the asset must be present, with the constrained policy (#8; + negative probe: a consumer without the credentials must NOT see the offer). +2. Contract negotiation mirroring the offer → SUT must callback agreement (#10), accept the + verification (#11) and finalize (#12). +3. Transfer request (`HttpData-PULL`) → SUT must send TransferStart with a working EDR (#14). +4. Data pull with the EDR token → payload bytes; renewal via the SUT's refresh endpoint where + advertised (#15/#16 mirrored). + +**Asserted on ve1's side only:** negotiation FINALIZED, transfer STARTED, payload received — +all observable through ve1's management API and the downloaded bytes. + +### Checkpoint 3 — SUT as consumer (the SUT fires the requests) + +**SUT obligations (actions, not just state):** against ve1's seeded, credential-constrained +offer (what `dsp-tests.sh` seeds today), the SUT must at some point initiate and complete: +1. Catalog request to ve1's DSP endpoint (#8) and locate the offer. +2. Negotiation mirroring the offer policy exactly (#9, #11) through to FINALIZED (#10, #12). +3. An `HttpData-PULL` transfer (#13) to STARTED (#14). +4. The data pull using the EDR from the TransferStartMessage (#16), with refresh against + ve1's siglet where needed (#15). + +**Asserted on ve1's side only:** ve1's management API shows the negotiation reaching agreement +/ FINALIZED and the transfer STARTED for the SUT's participant id; ve1's data source gets hit. +(With the current demo data source — public jsonplaceholder — the pull is not observable on +ve1; making the harness serve the payload itself is the planned improvement so that +consumer-role verification can assert the actual download.) + +## What is explicitly out of scope + +The SUT's management APIs, onboarding process, wallet/agent internals, auth stack and +deployment shape. Equally, the reference harness's own conveniences — the v5beta management +API, jwtlet/clearglass, the tenant manager, the siglet token-cache API used by +`dsp-tests.sh` to fetch the EDR on the consumer side — are driver tooling for ve2 and vanish +from the picture once ve2 is replaced by a real SUT. + +## Where the harness stands today + +`dsp-tests.sh` implements Checkpoint 2+3 with ve2 as a compliant pseudo-SUT: ve2's +obligations are fulfilled by the platform's own tooling (`onboard-participant.sh` for +credentials/wallet/data plane, the script's seeding steps for the offer), and the script +drives both sides. Evolving it toward this document means extracting the ve2-side operations +behind a "consumer/provider driver" interface and adding the reversed-role scenario — the +ve1-side halves stay as they are. diff --git a/scripts/README.md b/scripts/README.md index 33a97af..51f764f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,3 +1,39 @@ # Scripts -Utility and automation scripts for the cx-ve project. +Utility and automation scripts for the cx-ve project. Each script documents its own options in a header comment and +supports `--help`. + +## `setup-did-dns.sh` + +Makes the platform's gateway hostnames resolvable from **inside** the cluster, by adding `rewrite` +rules to CoreDNS that point every HTTPRoute hostname at the Traefik Service, then verifies both DNS and an end-to-end +`did:web` resolution. + +Needed because `core-platform-distribution` advertises itself under its gateway hostname: the DSP callback +address, the DCP credential-service and issuance endpoints and the `did:web` identifiers all point there +rather than at in-cluster Service FQDNs, and the runtimes dereference those URLs themselves — the control plane +resolves the issuer DID to verify credential signatures and calls the CredentialService on every DSP message. A +`*.localhost` name otherwise resolves to the pod's own loopback and every one of those lookups fails. + +```bash +./scripts/setup-did-dns.sh -c ve1 # apply + verify (own hostnames) +./scripts/setup-did-dns.sh -c ve1 --verify-only # dry-run +./scripts/setup-did-dns.sh -c ve1 --peer ve2 # make ve1 resolve ve2's hostnames +``` + +`install-ve.sh` runs the own-hostname mode automatically; recreating the KinD cluster discards the rules, so re-run it +after a reinstall. It is idempotent — each mode's rules live between marker comments in the Corefile and are replaced +wholesale. + +Everything but the cluster names is discovered from the clusters (DNS domain, Traefik Service, kube-dns, hostname +lists), so the rules cannot drift from what is deployed. That matters most for the DNS domain: each VE gets its own +(`ve1.local`, `ve2.local`), and a rewrite target under `cluster.local` silently NXDOMAINs because CoreDNS is not +authoritative for it. + +**Peer mode** (`--peer `, run automatically by `connect-ves.sh`) makes one VE resolve the *other's* gateway +hostnames, which every advertised URL and DID of the peer lives under. It cannot use `rewrite`: a rewritten name does +not re-enter CoreDNS server-block selection, so it would bypass the peer-zone forwarding and NXDOMAIN. Instead it adds +a dedicated server block forwarding the peer's hostnames themselves to the peer's kube-dns, whose own rewrite rules +answer with the peer's Traefik ClusterIP — automatically tracking it. Peer hostnames the local cluster also serves +(the telemetry hosts, identical in every VE) are excluded, since a dedicated server block would hijack the local name. +Peer mode needs the static routes and zone forwarding from `connect-ves.sh`, which sequences it correctly. diff --git a/scripts/connect-ves.sh b/scripts/connect-ves.sh index 4ab4f2b..4fe2f54 100755 --- a/scripts/connect-ves.sh +++ b/scripts/connect-ves.sh @@ -8,11 +8,17 @@ # 2. CoreDNS zone forwarding: each cluster forwards the peer's DNS domain (e.g. ve2.local) # to the peer's kube-dns service, making .edc-v.svc. names resolve # cluster-to-cluster. -# 3. Mutual trust: each VE's controlplane gets the peer's issuer DID added to its -# edc.controlplane.trustedIssuers via helm upgrade. Because a platform upgrade regenerates -# the NATS users.conf (dropping the Onboarding API's NKey entry), the obapi release is -# upgraded right after to re-merge it, and the obapi deployment is restarted. -# 4. Smoke test: each cluster resolves and fetches the peer issuer's DID document. +# 3. Peer gateway-hostname DNS (setup-did-dns.sh --peer): each cluster forwards the peer's +# HTTPRoute hostnames (ve2.localhost, issuer.ve2.localhost, identity.ve2.localhost) to the +# peer's kube-dns. Everything a VE advertises about itself — issuer and participant DIDs, +# the DSP callback, the CredentialService endpoint — lives under those hostnames, and the +# runtimes dereference them on every cross-VE interaction. Includes its own verification +# (DNS + an end-to-end fetch of the peer issuer's DID document). +# 4. Mutual trust: each VE's controlplane gets the peer's issuer DID +# (did:web:issuer.:issuer) added to its edc.controlplane.trustedIssuers via helm +# upgrade. Because a platform upgrade regenerates the NATS users.conf (dropping the +# Onboarding API's NKey entry), the obapi release is upgraded right after to re-merge it, +# and the obapi deployment is restarted. # # Assumes the two VEs were installed with the conventions of install-ve.sh, e.g.: # ./scripts/install-ve.sh -c ve1 -d ve1.local -H ve1.localhost @@ -34,7 +40,7 @@ NAMESPACE=edc-v # Charts (must match what the VEs were installed with) CORE_CHART="${CORE_CHART:-oci://ghcr.io/eclipse-cfm/charts/core-platform-distribution}" -CORE_CHART_VERSION=0.0.15 +CORE_CHART_VERSION=0.0.17 # The dual-VE setup requires the working-tree app chart (see install-ve.sh) OBAPI_CHART="${OBAPI_CHART:-charts/cx-ve}" @@ -112,13 +118,15 @@ forward_dns() { # kubectl --kubeconfig "$kc" rollout status deployment/coredns -n kube-system --timeout=120s } -# ---- 3. Mutual issuer trust ----------------------------------------------------------------- -trust_peer() { # +# ---- 4. Mutual issuer trust ----------------------------------------------------------------- +trust_peer() { # local kc; kc=$(kubeconfig "$1") export KUBECONFIG="$kc" # Full trustedIssuers list: this file wins over platform-override-values.yaml, so it must - # repeat the cxtck entry alongside the peer issuer + # repeat the cxtck entry alongside the peer issuer. The peer issuer DID is the one the peer + # platform minted from its gateway hostname (did:web:issuer.:issuer) — resolvable + # here thanks to the peer gateway-hostname forwarding in step 3. cat > "$GEN_DIR/trust.$1.yaml" <> $1: trusting issuer of $4 (did:web:issuerservice.${NAMESPACE}.svc.$5%3A10016:issuer)" + echo ">> $1: trusting issuer of $4 (did:web:issuer.$6:issuer)" helm upgrade --install core-platform "$CORE_CHART" \ --namespace "$NAMESPACE" \ -f platform-override-values.yaml \ @@ -166,7 +174,9 @@ config: url: http://identityhub.${NAMESPACE}.svc.$2:7081/api/identity/v1beta participant: did: - template: "did:web:identityhub.${NAMESPACE}.svc.$2%3A7083:" + # Keep in sync with install-ve.sh: must match the platform's IdentityHub did:web + # hostname (identity.), or participant DIDs will not resolve through the gateway. + template: "did:web:identity.$3:" dataplane: # Keep in sync with install-ve.sh: demo data source for HttpData-PULL transfers endpoint: https://jsonplaceholder.typicode.com/todos/1 @@ -193,25 +203,22 @@ EOF kubectl rollout status deployment/controlplane -n "$NAMESPACE" --timeout=420s } -# ---- 4. Smoke test: fetch the peer issuer's DID document across clusters -------------------- -smoke() { # - local url="http://issuerservice.${NAMESPACE}.svc.$2:10016/issuer/did.json" - echo ">> $1: fetching $url" - kubectl --kubeconfig "$(kubeconfig "$1")" run "peercheck-$RANDOM" --rm -i --restart=Never \ - --image=curlimages/curl:latest -- -sf --max-time 10 "$url" > /dev/null - echo ">> $1: peer issuer DID document of $2 resolved OK" -} - add_routes "$C1" "$C2" add_routes "$C2" "$C1" forward_dns "$C1" "$D2" "$(dns_ip_of "$C2")" forward_dns "$C2" "$D1" "$(dns_ip_of "$C1")" -trust_peer "$C1" "$D1" "$H1" "$C2" "$D2" -trust_peer "$C2" "$D2" "$H2" "$C1" "$D1" - -smoke "$C1" "$D2" -smoke "$C2" "$D1" +# ---- 3. Peer gateway-hostname DNS ----------------------------------------------------------- +# setup-did-dns.sh --peer forwards the peer's HTTPRoute hostnames to the peer's kube-dns and +# verifies the result, including an end-to-end fetch of the peer issuer's DID document from a +# pod — the exact resolution path the runtimes use. (This replaced the old smoke test, which +# fetched the in-cluster Service URL nothing advertises anymore.) Needs the routes and zone +# forwarding above. +"$(dirname "$0")/setup-did-dns.sh" -c "$C1" --peer "$C2" +"$(dirname "$0")/setup-did-dns.sh" -c "$C2" --peer "$C1" + +trust_peer "$C1" "$D1" "$H1" "$C2" "$D2" "$H2" +trust_peer "$C2" "$D2" "$H2" "$C1" "$D1" "$H1" echo "VEs connected: $C1 ($D1) <-> $C2 ($D2)" diff --git a/scripts/dsp-tests.sh b/scripts/dsp-tests.sh index c8410ef..77ce0e3 100755 --- a/scripts/dsp-tests.sh +++ b/scripts/dsp-tests.sh @@ -26,22 +26,23 @@ # /api/tm → tenant-manager /api/v1alpha1 # /api/siglet → siglet / (data-plane token cache: EDR retrieval) # -# The DSP counterparty address stays the in-cluster URL (controlplane.edc-v.svc.:8082) -# — it is dereferenced by the consumer VE's controlplane over the connect-ves.sh routing, not -# from the host (the DSP port has no gateway route). +# The DSP counterparty address is the provider's gateway URL (http:///api/dsp/…) — +# the address the platform actually advertises to counterparties. It is dereferenced by the +# consumer VE's controlplane, which resolves the provider's gateway hostname through the peer +# DNS forwarding connect-ves.sh sets up (setup-did-dns.sh --peer). # # Prerequisites: both VEs installed (install-ve.sh), a participant onboarded in each # (onboard-participant.sh) and the VEs connected (connect-ves.sh). Requires kubectl, curl, jq. # # Usage: -# ./scripts/dsp-tests.sh [--pc ] [--pd ] [--pu ] -# [--cc ] [--cd ] [--cu ] +# ./scripts/dsp-tests.sh [--pc ] [--pu ] +# [--cc ] [--cu ] # [--asset ] [-h|--help] set -euo pipefail -PROV_CLUSTER=ve1; PROV_DOMAIN=ve1.local; PROV_URL=http://ve1.localhost -CONS_CLUSTER=ve2; CONS_DOMAIN=ve2.local; CONS_URL=http://ve2.localhost:8081 +PROV_CLUSTER=ve1; PROV_URL=http://ve1.localhost +CONS_CLUSTER=ve2; CONS_URL=http://ve2.localhost:8081 # Default derived from the provider participant context after discovery: ids are unique across # participant contexts in the store, so a fixed id would 409 against an older participant's # asset without ever appearing in this participant's catalog @@ -51,27 +52,25 @@ AUDIENCE=edcv usage() { cat <] [--pd ] [--pu ] - [--cc ] [--cd ] [--cu ] +Usage: $(basename "$0") [--pc ] [--pu ] + [--cc ] [--cu ] [--asset ] [-h|--help] Options (defaults match the dual-VE install convention): - --pc/--pd/--pu provider cluster / DNS domain / gateway base URL - (default: ve1 / ve1.local / http://ve1.localhost) - --cc/--cd/--cu consumer cluster / DNS domain / gateway base URL - (default: ve2 / ve2.local / http://ve2.localhost:8081) - --asset asset id to offer and negotiate (default: demo-asset-) - -h, --help show this help + --pc/--pu provider cluster / gateway base URL (default: ve1 / http://ve1.localhost) + --cc/--cu consumer cluster / gateway base URL (default: ve2 / http://ve2.localhost:8081) + --asset asset id to offer and negotiate (default: demo-asset-) + -h, --help show this help EOF } while [[ $# -gt 0 ]]; do case "$1" in - --pc|--pd|--pu|--cc|--cd|--cu|--asset) + --pc|--pu|--cc|--cu|--asset) [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage >&2; exit 1; } case "$1" in - --pc) PROV_CLUSTER="$2" ;; --pd) PROV_DOMAIN="$2" ;; --pu) PROV_URL="$2" ;; - --cc) CONS_CLUSTER="$2" ;; --cd) CONS_DOMAIN="$2" ;; --cu) CONS_URL="$2" ;; + --pc) PROV_CLUSTER="$2" ;; --pu) PROV_URL="$2" ;; + --cc) CONS_CLUSTER="$2" ;; --cu) CONS_URL="$2" ;; --asset) ASSET_ID="$2" ;; esac shift 2 @@ -119,20 +118,33 @@ xtoken() { # -> access token on stdout return 1 } -# mgmt [json-file] -> body on stdout, fails on HTTP >= 400 (409 ok) +# mgmt [json-file] -> body on stdout, fails on HTTP >= 400 (409 ok). +# 502/503/504 are retried: they mean Traefik could not reach (or complete against) the backend, +# so the request was not processed. The window is real — connect-ves.sh restarts the controlplanes +# for the trusted-issuer config, and the gateway's endpoint view can lag the rollout by a few +# seconds, so the first management call afterwards may hit a stale endpoint. mgmt() { - local base tok out status body + local base tok out status body attempt base=$(base_of "$1") tok=$(xtoken "$1" admin) - if [[ $# -ge 4 ]]; then - out=$(curl -s -m 30 -w '\n%{http_code}' -X "$2" "$base/api/management/v5beta$3" \ - -H "Authorization: Bearer $tok" -H 'Content-Type: application/json' --data @"$4") - else - out=$(curl -s -m 30 -w '\n%{http_code}' -X "$2" "$base/api/management/v5beta$3" \ - -H "Authorization: Bearer $tok") - fi - status=$(printf '%s' "$out" | tail -1) - body=$(printf '%s' "$out" | sed '$d') + for attempt in 1 2 3 4 5; do + if [[ $# -ge 4 ]]; then + out=$(curl -s -m 30 -w '\n%{http_code}' -X "$2" "$base/api/management/v5beta$3" \ + -H "Authorization: Bearer $tok" -H 'Content-Type: application/json' --data @"$4") + else + out=$(curl -s -m 30 -w '\n%{http_code}' -X "$2" "$base/api/management/v5beta$3" \ + -H "Authorization: Bearer $tok") + fi + status=$(printf '%s' "$out" | tail -1) + body=$(printf '%s' "$out" | sed '$d') + case "$status" in + 502|503|504) + echo ">> $1 $2 $3: HTTP $status (gateway/backend not settled), retry $attempt" >&2 + sleep 3 + ;; + *) break ;; + esac + done if [[ "$status" == 409 ]]; then echo ">> $1 $2 $3: already present (409)" >&2 elif [[ "$status" -ge 400 ]]; then @@ -152,7 +164,11 @@ CONS_DID=$(mgmt "$CONS_CLUSTER" GET /participants | jq -r '.[-1].identity') for v in PROV_CTX PROV_DID CONS_CTX CONS_DID; do [[ -n "${!v}" && "${!v}" != null ]] || { echo "ERROR: participant discovery failed ($v empty) — are participants onboarded in both VEs?" >&2; exit 1; } done -PROV_DSP="http://controlplane.${NAMESPACE}.svc.${PROV_DOMAIN}:8082/api/dsp/${PROV_CTX}/cx-neptune" +# The provider's advertised DSP endpoint (see header). Note the consumer's controlplane — not +# the host — dereferences this, so the URL must work from the consumer's pods: true for the +# default topology, where the provider's gateway listens on port 80 both in-cluster and on the +# host. A provider on a non-default host port would need the in-cluster port here instead. +PROV_DSP="${PROV_URL}/api/dsp/${PROV_CTX}/cx-neptune" # Scope the demo object ids by participant context (ids are store-unique across contexts) ASSET_ID="${ASSET_ID:-demo-asset-${PROV_CTX:0:8}}" POLICY_ID="policy-credentials-${PROV_CTX:0:8}" diff --git a/scripts/install-ve.sh b/scripts/install-ve.sh index ba4eba4..32fe50c 100755 --- a/scripts/install-ve.sh +++ b/scripts/install-ve.sh @@ -4,10 +4,11 @@ # Onboarding API — on a kind cluster. Run from the repository root. # # Multiple VEs can coexist on one host: give each its own cluster name, DNS domain, hostname, -# host ports and (for later cross-cluster routing) pod/service subnets. The cluster's DNS -# domain doubles as the VE's identity domain — participant and issuer DIDs embed the in-cluster -# FQDNs (….edc-v.svc.…), so two VEs MUST use distinct DNS domains to mint -# distinct DIDs, e.g.: +# host ports and (for later cross-cluster routing) pod/service subnets. The HOSTNAME (-H) is the +# VE's identity domain — participant and issuer DIDs embed the gateway hostnames derived from it +# (did:web:identity.:, did:web:issuer.:issuer), so two VEs MUST use +# distinct hostnames to mint distinct DIDs. Distinct DNS DOMAINS (-d) are still required, but +# for cross-cluster DNS (connect-ves.sh forwards the peer's zone to the peer's kube-dns), e.g.: # # ./scripts/install-ve.sh -c ve1 -d ve1.local -H ve1.localhost # ./scripts/install-ve.sh -c ve2 -d ve2.local -H ve2.localhost --http-port 8081 --https-port 8444 \ @@ -25,10 +26,12 @@ # -c, --cluster name of the kind cluster to (re)create (default: cxve). CAUTION: # an existing cluster of that name is deleted first. The kubeconfig # is written to ~/.kube/.config -# -d, --dns-domain cluster DNS domain (default: cluster.local). Distinct per VE: it -# is embedded in participant/issuer DIDs -# -H, --host HTTPRoute hostname the VE's APIs are exposed under (default: -# cxve.localhost; any *.localhost name resolves to loopback) +# -d, --dns-domain cluster DNS domain (default: cluster.local). Distinct per VE for +# cross-cluster DNS (connect-ves.sh zone forwarding) +# -H, --host HTTPRoute hostname the VE's APIs are exposed under, and the VE's +# identity domain — DIDs embed identity. / issuer. +# (default: cxve.localhost; *.localhost resolves to loopback on the +# host, and setup-did-dns.sh makes it resolve in-cluster) # --http-port host port mapped to the gateway's HTTP port 80 (default: 80) # --https-port host port mapped to the gateway's HTTPS port 443 (default: 443) # --pod-subnet pod CIDR (default: 10.244.0.0/16); distinct per VE to allow @@ -58,9 +61,10 @@ Options: -c, --cluster name of the kind cluster to (re)create (default: cxve). CAUTION: an existing cluster of that name is deleted first. The kubeconfig is written to ~/.kube/.config - -d, --dns-domain cluster DNS domain (default: cluster.local). Distinct per VE: - it is embedded in participant/issuer DIDs. - -H, --host HTTPRoute hostname of the VE's APIs (default: cxve.localhost) + -d, --dns-domain cluster DNS domain (default: cluster.local). Distinct per VE for + cross-cluster DNS (connect-ves.sh zone forwarding). + -H, --host HTTPRoute hostname of the VE's APIs and its identity domain — DIDs + embed identity. / issuer. (default: cxve.localhost) --http-port host port mapped to the gateway's HTTP port (default: 80) --https-port host port mapped to the gateway's HTTPS port (default: 443) --pod-subnet pod CIDR (default: 10.244.0.0/16) @@ -100,14 +104,17 @@ KUBECONFIG_FILE="$HOME/.kube/$CLUSTER_NAME.config" # Helm chart of the core-platform-distribution CORE_CHART="${CORE_CHART:-oci://ghcr.io/eclipse-cfm/charts/core-platform-distribution}" -CORE_CHART_VERSION=0.0.15 +CORE_CHART_VERSION=0.0.17 # Helm chart of the Catena-X Profile CXPROF_CHART="${CXPROF_CHART:-oci://ghcr.io/metaform/charts/catenax-profile}" -CXPROF_CHART_VERSION=0.0.4 +CXPROF_CHART_VERSION=0.0.5 -# Helm chart of the Onboarding API -OBAPI_CHART="${OBAPI_CHART:-oci://ghcr.io/metaform/charts/cx-ve}" +# Helm chart of the Onboarding API. Defaults to the working-tree chart: the script always +# builds and loads the working-tree IMAGE anyway, so deploying the published chart with it +# would mix versions. Set OBAPI_CHART=oci://ghcr.io/metaform/charts/cx-ve for the published +# one (--version below only applies to such remote refs; helm ignores it for a directory). +OBAPI_CHART="${OBAPI_CHART:-charts/cx-ve}" OBAPI_CHART_VERSION=0.0.1 # Always tear down the cluster on exit, whether the run succeeds, fails on any @@ -183,7 +190,10 @@ config: url: http://identityhub.${NAMESPACE}.svc.${DNS_DOMAIN}:7081/api/identity/v1beta participant: did: - template: "did:web:identityhub.${NAMESPACE}.svc.${DNS_DOMAIN}%3A7083:" + # Must match the platform's IdentityHub did:web hostname (identity.), since + # IdentityHub resolves a DID document by the request URL — a participant DID minted + # under any other authority will not resolve through the gateway. + template: "did:web:identity.${HOST}:" dataplane: # Demo data source served for HttpData-PULL transfers: a public sample-JSON API, so the # demo payload is unambiguous sample data rather than platform infrastructure @@ -214,13 +224,27 @@ helm upgrade --install core-platform "$CORE_CHART" \ --version $CORE_CHART_VERSION \ --wait --timeout 15m +# Make the platform's gateway hostnames resolvable from inside the cluster. The platform +# advertises itself under them (DSP callback, credential service, issuance, did:web), and the +# runtimes dereference those URLs themselves — without this, a *.localhost name resolves to the +# pod's own loopback and every DID lookup fails. Must run after the platform install, since the +# hostname list is read off the deployed HTTPRoutes; also verifies the result, so a failure here +# stops the install rather than surfacing later as a credential-verification error. +"$(dirname "$0")/setup-did-dns.sh" -c "$CLUSTER_NAME" + # Deploy the CX Profile chart (global.namespace/clusterDomain tell its seed jobs where the -# platform lives) +# platform lives). issuer.did must equal the DID the platform minted for the issuer participant +# context — it is pinned into the dataspace profile's credentialSpecs, and a mismatch only +# surfaces at credential verification during onboarding, far from the cause. The platform derives +# that DID from its gateway hostname as did:web:issuer.:issuer unless it was pinned there +# via edc.issuerservice.did.id; read the live value back with +# kubectl -n "$NAMESPACE" get httproute issuerservice-did -o jsonpath='{.spec.hostnames[0]}' helm upgrade --install cx-profile "$CXPROF_CHART" \ --namespace "$NAMESPACE" \ --set global.namespace="$NAMESPACE" \ --set global.clusterDomain="svc.${DNS_DOMAIN}" \ + --set issuer.did="did:web:issuer.${HOST}:issuer" \ --version "$CXPROF_CHART_VERSION" \ --wait diff --git a/scripts/setup-did-dns.sh b/scripts/setup-did-dns.sh new file mode 100755 index 0000000..855c62a --- /dev/null +++ b/scripts/setup-did-dns.sh @@ -0,0 +1,418 @@ +#!/bin/bash + +# Makes the platform's gateway hostnames resolvable from INSIDE a cluster, then verifies it. +# +# Why this is needed: the URLs core-platform-distribution advertises about itself — the DSP +# callback address, the DCP credential-service and issuance endpoints, and the did:web +# identifiers — all point at the gateway hostname +# (ve1.localhost, issuer.ve1.localhost, identity.ve1.localhost) instead of in-cluster Service +# FQDNs. Those URLs are dereferenced by the runtimes themselves: the control plane resolves the +# issuer DID to verify credential signatures and calls the CredentialService on every DSP +# message. A `*.localhost` name resolves to the pod's own loopback, so without CoreDNS help +# every one of those lookups fails. +# +# Two modes, each owning its own marker-delimited block in the Corefile (idempotent — the block +# is replaced wholesale on every run): +# +# OWN mode (default): adds `rewrite` rules to the cluster's CoreDNS pointing each HTTPRoute +# hostname at the cluster's own Traefik Service, so the same name resolves to the gateway from +# inside the cluster and to 127.0.0.1 (via the KinD host port mapping) from the host. +# +# PEER mode (--peer ): makes the PEER VE's gateway hostnames resolvable, for the +# dual-VE setup — each side dereferences the other's advertised URLs and DIDs. A `rewrite` +# cannot do this: it runs inside the `.:53` server block, and the rewritten name +# (traefik.traefik.svc.) does NOT re-enter server-block selection, so it bypasses +# the `:53` zone-forwarding block connect-ves.sh installs and falls through to +# `forward . /etc/resolv.conf` -> NXDOMAIN (and `.:53` cannot take a second forward). Instead a +# dedicated multi-zone server block forwards the peer's hostnames THEMSELVES to the peer's +# kube-dns — whose own rewrite rules (installed by this script in OWN mode during the peer's +# install-ve.sh) answer with the peer's Traefik ClusterIP, automatically tracking it. +# Reachability of that ClusterIP comes from the static routes connect-ves.sh installs, so run +# peer mode after connect-ves.sh's routing steps (connect-ves.sh does this itself). +# Peer hostnames that the local cluster serves too (the telemetry hosts — grafana.localhost +# etc. are identical in every VE) are excluded: a dedicated server block wins zone selection +# over `.:53`, so forwarding them would hijack the LOCAL name. +# +# Everything except the cluster names is discovered from the clusters themselves — DNS domains, +# Traefik Services, kube-dns and the hostname lists — so the rules cannot drift from what is +# deployed. This matters most for the DNS domain: install-ve.sh gives each VE its own (ve1.local, +# ve2.local), and a rewrite target of `…svc.cluster.local` on a VE whose domain is `ve1.local` +# silently NXDOMAINs, because CoreDNS's kubernetes plugin is not authoritative for it. +# +# NOTE the Corefile changes are lost when a KinD cluster is recreated — re-run after +# install-ve.sh (own mode; install-ve.sh does it) and connect-ves.sh (peer mode; ditto). +# +# Usage: +# ./scripts/setup-did-dns.sh [-c|--cluster ] [--peer ] [--verify-only] [-h|--help] +# +# -c, --cluster KinD cluster whose CoreDNS is configured (default: ve1). The +# kubeconfig is read from ~/.kube/.config, matching install-ve.sh +# --peer peer mode: make THIS cluster resolve 's gateway hostnames +# (peer kubeconfig from ~/.kube/.config) +# --verify-only run the checks without touching CoreDNS +# -h, --help show usage and exit + +set -euo pipefail + +CLUSTER_NAME=ve1 +PEER_NAME="" +VERIFY_ONLY=false + +# Fixed, as in install-ve.sh: the CFM agents hardcode system:serviceaccount:edc-v:… client ids +NAMESPACE=edc-v + +# Probe image for the verification pods. MUST be glibc-based and MUST NOT use curl: there are two +# independent RFC 6761 short-circuits that resolve `*.localhost` to loopback without ever asking +# the resolver, and either one turns a working setup into a confusing failure — +# * musl (Alpine, so curlimages/curl too) synthesizes *.localhost in the libc; +# * curl >= 7.77 does it internally, on ANY libc, before the resolver is consulted. +# glibc's getaddrinfo does neither, so `getent` and bash's /dev/tcp both see the real DNS — as +# does the JVM, which is what the EDC runtimes actually use. (If you do want curl here, it needs +# `--connect-to :80:traefik..svc.:80` to bypass its own short-circuit.) +PROBE_IMAGE=debian:stable-slim + +usage() { + cat <] [--peer ] [--verify-only] [-h|--help] + +Options: + -c, --cluster KinD cluster whose CoreDNS is configured (default: ve1). Kubeconfig is + read from ~/.kube/.config + --peer peer mode: make THIS cluster resolve 's gateway hostnames instead + of its own (run after connect-ves.sh's routing/zone-forwarding steps) + --verify-only run the checks without modifying CoreDNS + -h, --help show this help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -c|--cluster|--peer) + [[ $# -ge 2 ]] || { echo "Error: $1 requires a value" >&2; usage >&2; exit 1; } + case "$1" in + -c|--cluster) CLUSTER_NAME="$2" ;; + --peer) PEER_NAME="$2" ;; + esac + shift 2 ;; + --verify-only) VERIFY_ONLY=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Error: unknown argument '$1'" >&2; usage >&2; exit 1 ;; + esac +done + +kubeconfig_of() { # -> kubeconfig path, verified readable + local f="$HOME/.kube/$1.config" + [[ -r "$f" ]] || { + echo "Error: no kubeconfig at $f — was cluster '$1' created by install-ve.sh?" >&2 + exit 1 + } + echo "$f" +} + +KUBECONFIG_FILE=$(kubeconfig_of "$CLUSTER_NAME") +# Exported for the probe pods (kubectl run); discovery calls pass --kubeconfig explicitly so +# they can target either cluster. +export KUBECONFIG="$KUBECONFIG_FILE" + +GEN_DIR=$(mktemp -d) +trap 'rm -rf "$GEN_DIR"' EXIT + +# ---- discovery (all parameterized by kubeconfig, so peer mode can query both clusters) ------ + +corefile() { # + kubectl --kubeconfig "$1" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' +} + +# Cluster DNS domain, from the zone the kubernetes plugin is authoritative for. A rewrite/forward +# target must live in a resolvable zone or CoreDNS will not answer for it. +detect_dns_domain() { # + corefile "$1" | awk '/^[[:space:]]*kubernetes[[:space:]]/{print $2; exit}' +} + +# " " of the Traefik Service backing the Gateway +detect_traefik() { # + kubectl --kubeconfig "$1" get svc -A -l app.kubernetes.io/name=traefik \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.namespace}{"\n"}{end}' | head -1 +} + +# Every hostname any HTTPRoute in the platform namespace is served under. Taking the full set +# (rather than a hardcoded list) keeps the rules correct as routes come and go. +detect_hostnames() { # + kubectl --kubeconfig "$1" get httproute -n "$NAMESPACE" \ + -o jsonpath='{range .items[*]}{range .spec.hostnames[*]}{@}{"\n"}{end}{end}' | sort -u +} + +# Hostname the issuer's DID document is served on (empty when the route is not exposed) +detect_issuer_did_host() { # + kubectl --kubeconfig "$1" get httproute -n "$NAMESPACE" issuerservice-did \ + -o jsonpath='{.spec.hostnames[0]}' 2>/dev/null || true +} + +read_hostnames_into() { # (plain loop: macOS bash 3.2 has no mapfile) + local line + while IFS= read -r line; do + [[ -n "$line" ]] && eval "$1+=(\"\$line\")" + done < <(detect_hostnames "$2") +} + +# ---- Corefile block management -------------------------------------------------------------- +# Each mode owns one block between exact-match marker lines, replaced wholesale per run. Markers +# are compared as whole (whitespace-trimmed) lines, so the own block and per-peer blocks can +# never swallow each other, including peers whose names are prefixes of one another. + +strip_block() { # — stdin -> stdout without the block + awk -v b="$1" -v e="$2" ' + { t = $0; sub(/^[ \t]+/, "", t) } + t == b { skip = 1 } + !skip { print } + t == e { skip = 0 } + ' +} + +apply_corefile() { # + if diff -q <(corefile "$KUBECONFIG_FILE") "$1" >/dev/null 2>&1; then + echo ">> CoreDNS already up to date ($2)" + return + fi + echo ">> applying CoreDNS $2" + kubectl -n kube-system create configmap coredns \ + --from-file=Corefile="$1" --dry-run=client -o yaml \ + | kubectl apply -f - >/dev/null + # The Corefile has the `reload` plugin; the restart only makes the change take effect + # immediately rather than within its poll interval. + kubectl -n kube-system rollout restart deployment/coredns >/dev/null + kubectl -n kube-system rollout status deployment/coredns --timeout=120s >/dev/null + echo ">> CoreDNS reloaded" +} + +# ---- verification --------------------------------------------------------------------------- + +# Every hostname must resolve, from a pod in THIS cluster, to the given IP. Values reach the +# probe as environment variables so its script can stay single-quoted. +# +# The probe retries before judging: apply_corefile's rollout wait does not make the new config +# authoritative — terminating pods keep answering with the OLD Corefile through their lameduck +# window, and the kubelet can hand a freshly started pod a stale ConfigMap volume (healed by its +# periodic sync plus CoreDNS's `reload` poll, worst case ~2 minutes). Inside that window a +# *.localhost query falls through `.:53` to the host resolver, which per RFC 6761 answers ::1 — +# a transient WRONG answer, not a timeout, so only comparing against the expected IP catches it. +verify_dns() { # + local expect="$1"; shift + echo ">> verifying in-cluster DNS resolution" + kubectl run "dnscheck-$RANDOM" --rm -i --restart=Never --image="$PROBE_IMAGE" \ + --env="NAMES=$*" --env="EXPECT_IP=$expect" \ + --timeout=300s --command -- sh -c ' + attempt=1 + while :; do + fail=0 + for n in $NAMES; do + ip=$(getent ahosts "$n" 2>/dev/null | awk "{print \$1; exit}") + [ "$ip" = "$EXPECT_IP" ] || fail=1 + done + [ $fail -eq 0 ] || [ $attempt -ge 36 ] && break + echo " ... not propagated yet, retrying ($attempt/36)" + attempt=$((attempt + 1)) + sleep 5 + done + fail=0 + for n in $NAMES; do + ip=$(getent ahosts "$n" 2>/dev/null | awk "{print \$1; exit}") + if [ "$ip" = "$EXPECT_IP" ]; then + printf " OK %-32s -> %s\n" "$n" "$ip" + else + printf " FAIL %-32s -> %s [expected %s]\n" "$n" "${ip:-NXDOMAIN}" "$EXPECT_IP" + fail=1 + fi + done + exit $fail' +} + +# End-to-end: fetch the issuer DID document through the gateway (from a pod in THIS cluster) and +# confirm the document's own id matches the DID. That equality is the point — the IssuerService +# reconstructs the DID from the request URL (authority + path) and looks it up by exact match, so +# a mismatch means the route, the Host header or the DID itself is wrong, even when DNS is fine. +verify_did_document() { # + local did_host="$1" + # "issuer" is the issuer's participantContextId, created by the platform's issuerservice-seed + # job and hardcoded across cx-ve (see issuer.id in the reg/onboarding agent configs). + local path="/issuer/did.json" + local expect="\"id\":\"did:web:${did_host}:issuer\"" + + echo ">> verifying DID resolution: did:web:${did_host}:issuer" + # bash rather than sh: the probe image ships no HTTP client, and bash's /dev/tcp needs no extra + # image. The explicit Host header is the whole point — it is what the DID is reconstructed from. + # Retries for the same reason as verify_dns — and a dnscheck pass does not cover this probe: + # CoreDNS runs two replicas, so this lookup may hit a replica the dnscheck never exercised. + kubectl run "didcheck-$RANDOM" --rm -i --restart=Never --image="$PROBE_IMAGE" \ + --env="DID_HOST=$did_host" --env="DID_PATH=$path" --env="EXPECT=$expect" \ + --timeout=300s --command -- bash -c ' + for attempt in $(seq 1 36); do + response="" + if { exec 3<>"/dev/tcp/$DID_HOST/80"; } 2>/dev/null; then + printf "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n" "$DID_PATH" "$DID_HOST" >&3 + response=$(cat <&3) + exec 3<&- + fi + printf "%s" "$response" | grep -qF "$EXPECT" && break + [ "$attempt" -lt 36 ] && { echo " ... document not served yet, retrying ($attempt/36)"; sleep 5; } + done + if [ -z "$response" ]; then + echo " FAIL could not connect to $DID_HOST:80"; exit 1 + fi + printf " %s\n" "$(printf "%s" "$response" | head -1 | tr -d "\r")" + if printf "%s" "$response" | grep -qF "$EXPECT"; then + echo " OK document id matches the DID" + else + echo " FAIL expected $EXPECT in the response body" + printf "%s" "$response" | tail -3 | sed "s/^/ /" + exit 1 + fi' +} + +# ---- OWN mode: rewrite this cluster's hostnames to its own Traefik -------------------------- + +run_own_mode() { + local begin_marker="# BEGIN cx-ve did-dns:own (managed by setup-did-dns.sh)" + local end_marker="# END cx-ve did-dns:own" + + local dns_domain traefik_name traefik_ns traefik_fqdn traefik_ip + dns_domain=$(detect_dns_domain "$KUBECONFIG_FILE") + [[ -n "$dns_domain" ]] || { echo "Error: could not read the cluster DNS domain from the Corefile" >&2; exit 1; } + read -r traefik_name traefik_ns <<< "$(detect_traefik "$KUBECONFIG_FILE")" + [[ -n "${traefik_name:-}" ]] || { echo "Error: no Service labelled app.kubernetes.io/name=traefik found" >&2; exit 1; } + traefik_fqdn="${traefik_name}.${traefik_ns}.svc.${dns_domain}" + traefik_ip=$(kubectl --kubeconfig "$KUBECONFIG_FILE" -n "$traefik_ns" get svc "$traefik_name" -o jsonpath='{.spec.clusterIP}') + + local hostnames=() + read_hostnames_into hostnames "$KUBECONFIG_FILE" + [[ ${#hostnames[@]} -gt 0 ]] || { echo "Error: no HTTPRoutes found in namespace $NAMESPACE" >&2; exit 1; } + + echo "Cluster: $CLUSTER_NAME (kubeconfig $KUBECONFIG_FILE)" + echo "DNS domain: $dns_domain" + echo "Gateway: $traefik_fqdn ($traefik_ip)" + echo "Hostnames: ${hostnames[*]}" + echo + + if [[ "$VERIFY_ONLY" == "false" ]]; then + # Block content in a file: BSD awk (macOS) rejects newlines in -v assignments, and sed's `r` + # reads a file verbatim on both BSD and GNU. + local h + printf ' %s\n' "$begin_marker" > "$GEN_DIR/block" + for h in "${hostnames[@]}"; do + printf ' rewrite name %s %s\n' "$h" "$traefik_fqdn" >> "$GEN_DIR/block" + done + printf ' %s\n' "$end_marker" >> "$GEN_DIR/block" + + # Replace the managed block, inserted at the top of the .:53 server block. Placement is + # cosmetic — CoreDNS plugin ordering is fixed at build time by plugin.cfg, and `rewrite` + # runs before `kubernetes` wherever it appears. + corefile "$KUBECONFIG_FILE" \ + | strip_block "$begin_marker" "$end_marker" \ + | sed "/^\\.:53 {/r ${GEN_DIR}/block" \ + > "$GEN_DIR/Corefile" + apply_corefile "$GEN_DIR/Corefile" "rewrites for ${CLUSTER_NAME}'s own hostnames" + echo + fi + + verify_dns "$traefik_ip" "${hostnames[@]}" + echo + + local did_host + did_host=$(detect_issuer_did_host "$KUBECONFIG_FILE") + if [[ -n "$did_host" ]]; then + verify_did_document "$did_host" + else + echo ">> skipping DID document check: no issuerservice-did HTTPRoute" + echo " (expected when edc.issuerservice.did.exposed=false)" + fi + echo + echo "DID DNS setup verified for cluster '$CLUSTER_NAME'." +} + +# ---- PEER mode: forward the peer's hostnames to the peer's kube-dns ------------------------- + +run_peer_mode() { + local peer_kubeconfig + peer_kubeconfig=$(kubeconfig_of "$PEER_NAME") + + local begin_marker="# BEGIN cx-ve did-dns:peer:${PEER_NAME} (managed by setup-did-dns.sh)" + local end_marker="# END cx-ve did-dns:peer:${PEER_NAME}" + + # Peer-side facts: its hostnames, its kube-dns (the forward target) and its Traefik ClusterIP + # (what those names must resolve to once forwarded). + local peer_dns_ip peer_traefik_name peer_traefik_ns peer_traefik_ip + peer_dns_ip=$(kubectl --kubeconfig "$peer_kubeconfig" -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}') + [[ -n "$peer_dns_ip" ]] || { echo "Error: could not read kube-dns ClusterIP of peer '$PEER_NAME'" >&2; exit 1; } + read -r peer_traefik_name peer_traefik_ns <<< "$(detect_traefik "$peer_kubeconfig")" + [[ -n "${peer_traefik_name:-}" ]] || { echo "Error: no Traefik Service found in peer '$PEER_NAME'" >&2; exit 1; } + peer_traefik_ip=$(kubectl --kubeconfig "$peer_kubeconfig" -n "$peer_traefik_ns" get svc "$peer_traefik_name" -o jsonpath='{.spec.clusterIP}') + + local own_hostnames=() peer_hostnames=() + read_hostnames_into own_hostnames "$KUBECONFIG_FILE" + read_hostnames_into peer_hostnames "$peer_kubeconfig" + # Both must be non-empty — also keeps bash 3.2 (macOS) happy, where expanding an empty array + # under `set -u` is an unbound-variable error. + [[ ${#own_hostnames[@]} -gt 0 ]] || { echo "Error: no HTTPRoutes found in namespace $NAMESPACE of '$CLUSTER_NAME' — is the platform installed?" >&2; exit 1; } + [[ ${#peer_hostnames[@]} -gt 0 ]] || { echo "Error: no HTTPRoutes found in peer '$PEER_NAME'" >&2; exit 1; } + + # Collision filter: never forward a name this cluster serves itself (telemetry hosts are + # identical in every VE) — a dedicated server block wins zone selection over `.:53` and would + # hijack the local name. + local forwarded=() h o skip + for h in "${peer_hostnames[@]}"; do + skip=false + for o in "${own_hostnames[@]}"; do + [[ "$h" == "$o" ]] && { skip=true; break; } + done + [[ "$skip" == "false" ]] && forwarded+=("$h") + done + [[ ${#forwarded[@]} -gt 0 ]] || { echo "Error: every peer hostname collides with a local one — nothing to forward" >&2; exit 1; } + + echo "Cluster: $CLUSTER_NAME (kubeconfig $KUBECONFIG_FILE)" + echo "Peer: $PEER_NAME (kube-dns $peer_dns_ip, gateway $peer_traefik_ip)" + echo "Forwarded: ${forwarded[*]}" + echo "Skipped: $(comm -12 <(printf '%s\n' "${peer_hostnames[@]}") <(printf '%s\n' "${own_hostnames[@]}") | tr '\n' ' ')(served locally)" + echo + + if [[ "$VERIFY_ONLY" == "false" ]]; then + # One multi-zone server block, appended at file end (server blocks are top-level; zone + # selection picks the longest matching zone regardless of order). + local zones="" + for h in "${forwarded[@]}"; do zones+="${h}:53 "; done + { + printf '%s\n' "$begin_marker" + printf '%s{\n' "$zones" + printf ' errors\n cache 30\n forward . %s\n' "$peer_dns_ip" + printf '}\n' + printf '%s\n' "$end_marker" + } > "$GEN_DIR/block" + + corefile "$KUBECONFIG_FILE" | strip_block "$begin_marker" "$end_marker" > "$GEN_DIR/Corefile" + cat "$GEN_DIR/block" >> "$GEN_DIR/Corefile" + apply_corefile "$GEN_DIR/Corefile" "forwarding for peer '${PEER_NAME}' hostnames" + echo + fi + + # Both checks need connect-ves.sh's static routes: the DNS query goes to the peer's kube-dns + # ClusterIP, the document fetch to the peer's Traefik ClusterIP. + verify_dns "$peer_traefik_ip" "${forwarded[@]}" + echo + + local peer_did_host + peer_did_host=$(detect_issuer_did_host "$peer_kubeconfig") + if [[ -n "$peer_did_host" ]]; then + verify_did_document "$peer_did_host" + else + echo ">> skipping DID document check: peer has no issuerservice-did HTTPRoute" + fi + echo + echo "Peer DID DNS verified: '$CLUSTER_NAME' resolves '$PEER_NAME'." +} + +# ---- main ----------------------------------------------------------------------------------- + +if [[ -n "$PEER_NAME" ]]; then + run_peer_mode +else + run_own_mode +fi