diff --git a/control_plane/contracts/generic_web_deploy_recovery.py b/control_plane/contracts/generic_web_deploy_recovery.py new file mode 100644 index 000000000..1f40dbb01 --- /dev/null +++ b/control_plane/contracts/generic_web_deploy_recovery.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import hashlib +import json +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from control_plane.drivers.generic_web_dispatch import GenericWebDeployEnvelope + + +GenericWebDeployRecoveryAction = Literal[ + "replay_completed", + "wait_for_active_lease", + "adopt_observed", + "retry_original_operation", + "hold_unknown", +] +GenericWebDeployRecoveryProviderOutcome = Literal[ + "present", + "absent", + "unknown", + "not_inspected", +] + + +class GenericWebDeployRecoveryDryRunRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + product: str + instance: str + original_deploy: GenericWebDeployEnvelope + reason: str = Field(max_length=1000) + + @model_validator(mode="after") + def _validate_request(self) -> GenericWebDeployRecoveryDryRunRequest: + self.product = self.product.strip() + self.instance = self.instance.strip() + self.reason = self.reason.strip() + if not self.product: + raise ValueError("Generic web deploy recovery requires product.") + if not self.instance: + raise ValueError("Generic web deploy recovery requires instance.") + if not self.reason: + raise ValueError("Generic web deploy recovery requires reason.") + if self.original_deploy.product.strip() != self.product: + raise ValueError("Generic web deploy recovery requires matching product values.") + if self.original_deploy.deploy.instance.strip() != self.instance: + raise ValueError("Generic web deploy recovery requires matching instance values.") + return self + + +class GenericWebDeployRecoveryDryRunResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + status: Literal["ok"] = "ok" + mode: Literal["dry-run"] = "dry-run" + product: str + context: str + instance: str + reservation_state: Literal["running", "completed", "reconcile_required"] + reservation_attempt: int = Field(ge=1) + reservation_created_at: str + reservation_updated_at: str + reservation_lease_expires_at: str = "" + observed_at: str + reconciliation_key_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + provider_target_key_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + provider_effect_phase: str = Field(max_length=128) + provider_outcome: GenericWebDeployRecoveryProviderOutcome + provider_status: str = Field(default="", max_length=128) + retry_safe: bool + proposed_action: GenericWebDeployRecoveryAction + recovery_digest: str = Field(pattern=r"^[0-9a-f]{64}$") + + +def generic_web_deploy_recovery_identifier_sha256(value: str) -> str: + return hashlib.sha256(value.strip().encode("utf-8")).hexdigest() + + +def build_generic_web_deploy_recovery_digest(payload: dict[str, object]) -> str: + canonical = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/control_plane/generic_web_deploy_recovery_http.py b/control_plane/generic_web_deploy_recovery_http.py new file mode 100644 index 000000000..2e2e716a6 --- /dev/null +++ b/control_plane/generic_web_deploy_recovery_http.py @@ -0,0 +1,433 @@ +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Any, cast + +import click +from fastapi import Depends, Header, Request +from pydantic import ValidationError + +from control_plane.contracts.generic_web_deploy_recovery import ( + GenericWebDeployRecoveryAction, + GenericWebDeployRecoveryDryRunRequest, + GenericWebDeployRecoveryDryRunResponse, + GenericWebDeployRecoveryProviderOutcome, + build_generic_web_deploy_recovery_digest, + generic_web_deploy_recovery_identifier_sha256, +) +from control_plane.contracts.idempotency_record import parse_launchplane_mutation_timestamp +from control_plane.generic_web_deploy_http import ( + GENERIC_WEB_DEPLOY_ROUTE, + GenericWebDeployProductMismatchError, + GenericWebDeployRouteDependencyError, + resolve_generic_web_deploy_lane, +) +from control_plane.generic_web_deploy_provider_adapter import ( + GenericWebDeployProviderMutationAdapter, +) +from control_plane.http_routes.support import AuthorizationAllows, HttpErrorFactory +from control_plane.provider_operations import build_provider_operation_key +from control_plane.service_auth import AuthorizationTarget, LaunchplaneIdentity +from control_plane.storage.postgres import PostgresRecordStore +from control_plane.workflows.generic_web_deploy import ( + GenericWebDeployResult, + normalize_generic_web_artifact_id, +) +from control_plane.workflows.generic_web_deploy_provider import ( + build_generic_web_provider_target_key, + decode_generic_web_provider_reconciliation_target, + resolve_generic_web_provider_reconciliation_target, +) + + +GENERIC_WEB_DEPLOY_RECOVERY_DRY_RUN_ROUTE = "/v1/admin/generic-web/deploy-recovery/dry-run" + +__all__ = [ + "GENERIC_WEB_DEPLOY_RECOVERY_DRY_RUN_ROUTE", + "GenericWebDeployRecoveryDependencies", + "build_generic_web_deploy_recovery_dry_run_handler", +] + + +@dataclass(frozen=True, slots=True) +class GenericWebDeployRecoveryDependencies: + read_write_identity: Callable[..., LaunchplaneIdentity] + get_record_store: Callable[[], object] + next_trace_id: Callable[[], str] + authorization_allows: AuthorizationAllows + http_error: HttpErrorFactory + control_plane_root: Path + idempotency_request_fingerprint: Callable[..., str] + + +def _bounded_recovery_value(value: object, *, limit: int = 128) -> str: + return str(value or "").strip()[:limit] + + +def build_generic_web_deploy_recovery_dry_run_handler( + *, dependencies: GenericWebDeployRecoveryDependencies +) -> Callable[..., Any]: + async def dry_run_generic_web_deploy_recovery( + request: Request, + recovery_request: GenericWebDeployRecoveryDryRunRequest, + identity: Annotated[LaunchplaneIdentity, Depends(dependencies.read_write_identity)], + record_store: Annotated[object, Depends(dependencies.get_record_store)], + idempotency_key: Annotated[str, Header(alias="Idempotency-Key")], + ) -> GenericWebDeployRecoveryDryRunResponse: + trace_id = dependencies.next_trace_id() + try: + profile, lane = resolve_generic_web_deploy_lane( + record_store=record_store, + product=recovery_request.product, + instance=recovery_request.instance, + ) + except GenericWebDeployRouteDependencyError as error: + raise dependencies.http_error( + status_code=503, + trace_id=trace_id, + code="storage_unavailable", + message="Generic web deploy recovery requires database-backed profile storage.", + ) from error + except GenericWebDeployProductMismatchError as error: + raise dependencies.http_error( + status_code=403, + trace_id=trace_id, + code="product_driver_mismatch", + message="Product is not configured for generic web deploy recovery.", + ) from error + except (ValueError, click.ClickException) as error: + raise dependencies.http_error( + status_code=400, + trace_id=trace_id, + code="invalid_request", + message="Request could not be completed.", + ) from error + if not dependencies.authorization_allows( + identity=identity, + action="generic_web_deploy.execute", + product=profile.product, + context=lane.context, + target=AuthorizationTarget(scope="instance", instances=(lane.instance,)), + ): + raise dependencies.http_error( + status_code=403, + trace_id=trace_id, + code="authorization_denied", + message=( + "Identity cannot inspect generic web deploy recovery for the requested " + "product/context." + ), + ) + normalized_key = idempotency_key.strip() + if not normalized_key: + raise dependencies.http_error( + status_code=400, + trace_id=trace_id, + code="idempotency_key_required", + message="Generic web deploy recovery requires an Idempotency-Key header.", + ) + if not isinstance(record_store, PostgresRecordStore): + raise dependencies.http_error( + status_code=503, + trace_id=trace_id, + code="database_storage_required", + message="Generic web deploy recovery requires database storage.", + ) + raw_payload = await request.json() + original_payload = ( + raw_payload.get("original_deploy") if isinstance(raw_payload, dict) else None + ) + if not isinstance(original_payload, dict): + raise dependencies.http_error( + status_code=400, + trace_id=trace_id, + code="invalid_request", + message="Generic web deploy recovery requires the exact original deploy payload.", + ) + original_fingerprint = dependencies.idempotency_request_fingerprint( + route_path=GENERIC_WEB_DEPLOY_ROUTE, + payload=cast(dict[str, object], original_payload), + ) + lookup = record_store.lookup_existing_mutation_reservation( + route_path=GENERIC_WEB_DEPLOY_ROUTE, + idempotency_key=normalized_key, + request_fingerprint=original_fingerprint, + ) + if lookup.status == "missing": + raise dependencies.http_error( + status_code=404, + trace_id=trace_id, + code="reservation_not_found", + message="No matching generic web deploy reservation exists.", + ) + if lookup.status in {"conflict", "ambiguous"}: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code=f"reservation_{lookup.status}", + message="Generic web deploy recovery reservation identity is not unique and exact.", + ) + if lookup.status == "hold_unknown": + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="hold_unknown", + message="Generic web deploy recovery lease timing is not authoritative.", + ) + reservation = lookup.record + if reservation is None: + raise dependencies.http_error( + status_code=503, + trace_id=trace_id, + code="reservation_lookup_failed", + message="Generic web deploy recovery could not inspect the reservation.", + ) + if ( + reservation.route_path != GENERIC_WEB_DEPLOY_ROUTE + or reservation.idempotency_key != normalized_key + or reservation.request_fingerprint != original_fingerprint + ): + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_conflict", + message="Generic web deploy recovery reservation identity is not exact.", + ) + + provider_outcome: GenericWebDeployRecoveryProviderOutcome = "not_inspected" + provider_status = "" + retry_safe = False + proposed_action: GenericWebDeployRecoveryAction = "hold_unknown" + provider_operation_key = "" + provider_observation_payload: dict[str, object] = {"outcome": provider_outcome} + operation_product = recovery_request.original_deploy.product.strip() + operation_context = "" + operation_instance = recovery_request.original_deploy.deploy.instance.strip() + authoritative_lane = lane + if reservation.state == "completed": + stored_result = reservation.response_payload.get("result") + try: + completed_result = GenericWebDeployResult.model_validate(stored_result) + except ValidationError as error: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Completed generic web deploy recovery context is not authoritative.", + ) from error + completed_product = completed_result.product.strip() + completed_context = completed_result.context.strip() + completed_instance = completed_result.instance.strip() + if ( + not completed_product + or not completed_context + or not completed_instance + or completed_product != operation_product + or completed_instance != operation_instance + ): + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Completed generic web deploy recovery context is not authoritative.", + ) + operation_context = completed_context + proposed_action = "replay_completed" + provider_status = _bounded_recovery_value(completed_result.deploy_status) + else: + if not reservation.reconciliation_key or not reservation.provider_target_key: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identity is incomplete.", + ) + try: + stored_target = decode_generic_web_provider_reconciliation_target( + reservation.reconciliation_key + ) + except ValueError as error: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identity is invalid.", + ) from error + operation_context = stored_target.context.strip() + stored_instance = stored_target.instance.strip() + stored_product = stored_target.product.strip() + legacy_snapshot_without_product = "product" not in stored_target.model_fields_set + if ( + not operation_context + or (not legacy_snapshot_without_product and stored_product != operation_product) + or stored_instance != operation_instance + ): + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identity conflicts.", + ) + authoritative_lane = lane.model_copy( + update={"context": operation_context, "instance": stored_instance} + ) + try: + resolved_stored_target = resolve_generic_web_provider_reconciliation_target( + reconciliation_key=reservation.reconciliation_key, + request_artifact_id=recovery_request.original_deploy.deploy.artifact_id, + request_source_git_ref=recovery_request.original_deploy.deploy.source_git_ref, + request_timeout_seconds=recovery_request.original_deploy.deploy.timeout_seconds, + request_no_cache=recovery_request.original_deploy.deploy.no_cache, + normalized_artifact_id=normalize_generic_web_artifact_id( + profile=profile, + artifact_id=recovery_request.original_deploy.deploy.artifact_id, + ), + request_deploy_reference=( + recovery_request.original_deploy.deploy.deploy_reference + ), + lane=authoritative_lane, + ) + except (ValueError, click.ClickException) as error: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identity is invalid.", + ) from error + if ( + resolved_stored_target.ship_request.context != operation_context + or resolved_stored_target.ship_request.instance != stored_instance + or build_generic_web_provider_target_key(resolved_stored_target) + != reservation.provider_target_key + ): + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identity conflicts.", + ) + + if not dependencies.authorization_allows( + identity=identity, + action="generic_web_deploy.execute", + product=operation_product, + context=operation_context, + target=AuthorizationTarget(scope="instance", instances=(operation_instance,)), + ): + raise dependencies.http_error( + status_code=403, + trace_id=trace_id, + code="authorization_denied", + message=( + "Identity cannot inspect generic web deploy recovery for the stored " + "product/context." + ), + ) + + if reservation.state != "completed": + lease_is_active = False + if reservation.state == "running": + try: + lease_is_active = parse_launchplane_mutation_timestamp( + reservation.lease_expires_at, + field_name="lease_expires_at", + ) > parse_launchplane_mutation_timestamp( + lookup.observed_at, + field_name="observed_at", + ) + except ValueError as error: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="hold_unknown", + message="Generic web deploy recovery lease timing is not authoritative.", + ) from error + if lease_is_active: + proposed_action = "wait_for_active_lease" + if reservation.state != "running" or not lease_is_active: + try: + provider_operation_key = build_provider_operation_key( + scope=reservation.scope, + route_path=reservation.route_path, + idempotency_key=reservation.idempotency_key, + request_fingerprint=reservation.request_fingerprint, + reconciliation_key=reservation.reconciliation_key, + ) + except ValueError as error: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Generic web deploy recovery reservation identity is incomplete.", + ) from error + adapter = GenericWebDeployProviderMutationAdapter( + control_plane_root=dependencies.control_plane_root, + record_store=record_store, + deploy_request=recovery_request.original_deploy, + profile=profile, + lane=authoritative_lane, + trace_id=trace_id, + ) + inspection = adapter.inspect( + provider_operation_key=provider_operation_key, + provider_effect_phase=reservation.provider_effect_phase, + reconciliation_key=reservation.reconciliation_key, + expected_provider_target_key=reservation.provider_target_key, + ) + if not inspection.identity_matches: + raise dependencies.http_error( + status_code=409, + trace_id=trace_id, + code="reservation_target_conflict", + message="Stored generic web deploy recovery target identities conflict.", + ) + provider_outcome = inspection.observation.outcome + provider_status = _bounded_recovery_value(inspection.observation.deployment_status) + retry_safe = inspection.retry_safe + provider_observation_payload = inspection.observation.model_dump(mode="json") + if provider_outcome == "present": + proposed_action = "adopt_observed" + elif provider_outcome == "absent" and retry_safe: + proposed_action = "retry_original_operation" + else: + proposed_action = "hold_unknown" + + digest_payload: dict[str, object] = { + "schema_version": 1, + "mode": "dry-run", + "request": recovery_request.model_dump(mode="json"), + "original_route": GENERIC_WEB_DEPLOY_ROUTE, + "idempotency_key": normalized_key, + "request_fingerprint": original_fingerprint, + "reservation": reservation.model_dump(mode="json"), + "observed_at": lookup.observed_at, + "provider_operation_key": provider_operation_key, + "provider_observation": provider_observation_payload, + "retry_safe": retry_safe, + "proposed_action": proposed_action, + } + return GenericWebDeployRecoveryDryRunResponse( + product=operation_product, + context=operation_context, + instance=operation_instance, + reservation_state=reservation.state, + reservation_attempt=reservation.attempt, + reservation_created_at=reservation.created_at, + reservation_updated_at=reservation.updated_at, + reservation_lease_expires_at=reservation.lease_expires_at, + observed_at=lookup.observed_at, + reconciliation_key_sha256=generic_web_deploy_recovery_identifier_sha256( + reservation.reconciliation_key + ), + provider_target_key_sha256=generic_web_deploy_recovery_identifier_sha256( + reservation.provider_target_key + ), + provider_effect_phase=_bounded_recovery_value(reservation.provider_effect_phase), + provider_outcome=provider_outcome, + provider_status=provider_status, + retry_safe=retry_safe, + proposed_action=proposed_action, + recovery_digest=build_generic_web_deploy_recovery_digest(digest_payload), + ) + + return dry_run_generic_web_deploy_recovery diff --git a/control_plane/http_routes/generic_web.py b/control_plane/http_routes/generic_web.py index 3cdb840bc..45bbe71d9 100644 --- a/control_plane/http_routes/generic_web.py +++ b/control_plane/http_routes/generic_web.py @@ -11,6 +11,9 @@ from pydantic import BaseModel, ValidationError from starlette.exceptions import HTTPException as StarletteHTTPException +from control_plane.contracts.generic_web_deploy_recovery import ( + GenericWebDeployRecoveryDryRunResponse, +) from control_plane.contracts.idempotency_record import LaunchplaneIdempotencyRecord from control_plane.contracts.preview_desired_state_record import PreviewDesiredStateRecord from control_plane.contracts.product_profile_record import ( @@ -28,6 +31,11 @@ from control_plane.generic_web_deploy_provider_adapter import ( GenericWebDeployProviderMutationAdapter, ) +from control_plane.generic_web_deploy_recovery_http import ( + GENERIC_WEB_DEPLOY_RECOVERY_DRY_RUN_ROUTE as _GENERIC_WEB_DEPLOY_RECOVERY_DRY_RUN_ROUTE, + GenericWebDeployRecoveryDependencies, + build_generic_web_deploy_recovery_dry_run_handler, +) from control_plane.generic_web_preview_http import ( GENERIC_WEB_PREVIEW_DESIRED_STATE_ROUTE as _GENERIC_WEB_PREVIEW_DESIRED_STATE_ROUTE, GENERIC_WEB_PREVIEW_DESTROY_ROUTE as _GENERIC_WEB_PREVIEW_DESTROY_ROUTE, @@ -301,6 +309,7 @@ class GenericWebWriteRouteHandlers: apply_generic_web_preview_refresh: Callable[..., Any] apply_generic_web_preview_destroy: Callable[..., Any] apply_generic_web_deploy: Callable[..., Any] + dry_run_generic_web_deploy_recovery: Callable[..., Any] apply_generic_web_prod_promotion: Callable[..., Any] dispatch_generic_web_prod_promotion_workflow: Callable[..., Any] apply_generic_web_stable_verification: Callable[..., Any] @@ -333,6 +342,18 @@ def build_generic_web_write_route_handlers( *, dependencies: GenericWebWriteRouteDependencies, ) -> GenericWebWriteRouteHandlers: + dry_run_generic_web_deploy_recovery = build_generic_web_deploy_recovery_dry_run_handler( + dependencies=GenericWebDeployRecoveryDependencies( + read_write_identity=dependencies.read_write_identity, + get_record_store=dependencies.get_record_store, + next_trace_id=dependencies.next_trace_id, + authorization_allows=dependencies.authorization_allows, + http_error=dependencies.http_error, + control_plane_root=dependencies.control_plane_root, + idempotency_request_fingerprint=dependencies.idempotency_request_fingerprint, + ) + ) + def manager_preview_pr_number( *, profile: LaunchplaneProductProfileRecord, @@ -1927,6 +1948,7 @@ async def apply_generic_web_preview_verification( apply_generic_web_preview_refresh=apply_generic_web_preview_refresh, apply_generic_web_preview_destroy=apply_generic_web_preview_destroy, apply_generic_web_deploy=apply_generic_web_deploy, + dry_run_generic_web_deploy_recovery=dry_run_generic_web_deploy_recovery, apply_generic_web_prod_promotion=apply_generic_web_prod_promotion, dispatch_generic_web_prod_promotion_workflow=dispatch_generic_web_prod_promotion_workflow, apply_generic_web_stable_verification=apply_generic_web_stable_verification, @@ -2107,6 +2129,24 @@ def register_generic_web_write_routes( }, ) + app.add_api_route( + _GENERIC_WEB_DEPLOY_RECOVERY_DRY_RUN_ROUTE, + handlers.dry_run_generic_web_deploy_recovery, + methods=["POST"], + response_model=GenericWebDeployRecoveryDryRunResponse, + response_model_exclude_none=True, + operation_id="dry_run_generic_web_deploy_recovery", + summary="Inspect an existing generic web deploy reservation without writing", + responses={ + 400: {"model": dependencies.error_response_model}, + 401: {"model": dependencies.error_response_model}, + 403: {"model": dependencies.error_response_model}, + 404: {"model": dependencies.error_response_model}, + 409: {"model": dependencies.error_response_model}, + 503: {"model": dependencies.error_response_model}, + }, + ) + app.add_api_route( _GENERIC_WEB_PROD_PROMOTION_ROUTE, handlers.apply_generic_web_prod_promotion, diff --git a/docs/operations.md b/docs/operations.md index a1e608acc..85c30ee2f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -366,6 +366,42 @@ former replaces the previous process-local apply lock. Both require an explaining that cleanup won. Provider read failures remain fail-closed. Do not reproduce this transition with direct SQL or provider-side deletion. +### Generic-web deploy recovery dry-run + +Stage 1 recovery for legacy generic-web deploys is read-only. Operators call +`POST /v1/admin/generic-web/deploy-recovery/dry-run` with the exact original +`GenericWebDeployEnvelope` under `original_deploy`, the original +`Idempotency-Key`, the product and instance, and a non-empty reason. The service +authorizes `generic_web_deploy.execute` for the current product/context/instance +before looking up any reservation. After lookup, it derives the original +operation target from the stored reconciliation snapshot, or from an exact +typed completed response, and authorizes the stored product/context/instance +again before returning evidence. + +The lookup is existing-only and database-backed. It fingerprints the exact +original payload using `/v1/drivers/generic-web/deploy`, searches across the +original GitHub Actions scope, and fails closed for a missing, conflicting, or +multi-scope match. It never probes through the normal deploy route: calling +`/v1/drivers/generic-web/deploy` can reserve and start a new production deploy +when the assumed legacy reservation does not exist. + +Dry-run uses the reservation's stored reconciliation and provider-target +identities to inspect the original provider operation. Reconciliation identity, +product, instance, provider-target, or completed-response drift returns a +conflict before provider inspection; provider read uncertainty alone produces +`hold_unknown`. It does not reserve, +release, supersede, retry, adopt, or write deployment, inventory, idempotency, +or provider state. The bounded response reports only reservation state and +timestamps, hashed identifiers, provider outcome/status, retry safety, one of +`replay_completed`, `wait_for_active_lease`, `adopt_observed`, +`retry_original_operation`, or `hold_unknown`, and a canonical recovery digest. +Raw scopes, idempotency keys, reconciliation keys, provider-target keys, +original payloads, target URLs, and provider payloads are never returned. + +There is no recovery apply endpoint in Stage 1. A later stage must bind any +mutation to reviewed dry-run evidence and revalidate the reservation atomically; +until then, the recovery digest is evidence only and authorizes no mutation. + ## Target Launchplane Ingress The target communication model is: diff --git a/frontend/generated/openapi-canonical.json b/frontend/generated/openapi-canonical.json index 1d77bd4fe..0407d6032 100644 --- a/frontend/generated/openapi-canonical.json +++ b/frontend/generated/openapi-canonical.json @@ -6499,6 +6499,177 @@ "title": "GenericWebDeployEnvelope", "type": "object" }, + "GenericWebDeployRecoveryDryRunRequest": { + "additionalProperties": false, + "properties": { + "instance": { + "title": "Instance", + "type": "string" + }, + "original_deploy": { + "$ref": "#/components/schemas/GenericWebDeployEnvelope" + }, + "product": { + "title": "Product", + "type": "string" + }, + "reason": { + "maxLength": 1000, + "title": "Reason", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "product", + "instance", + "original_deploy", + "reason" + ], + "title": "GenericWebDeployRecoveryDryRunRequest", + "type": "object" + }, + "GenericWebDeployRecoveryDryRunResponse": { + "additionalProperties": false, + "properties": { + "context": { + "title": "Context", + "type": "string" + }, + "instance": { + "title": "Instance", + "type": "string" + }, + "mode": { + "const": "dry-run", + "default": "dry-run", + "title": "Mode", + "type": "string" + }, + "observed_at": { + "title": "Observed At", + "type": "string" + }, + "product": { + "title": "Product", + "type": "string" + }, + "proposed_action": { + "enum": [ + "replay_completed", + "wait_for_active_lease", + "adopt_observed", + "retry_original_operation", + "hold_unknown" + ], + "title": "Proposed Action", + "type": "string" + }, + "provider_effect_phase": { + "maxLength": 128, + "title": "Provider Effect Phase", + "type": "string" + }, + "provider_outcome": { + "enum": [ + "present", + "absent", + "unknown", + "not_inspected" + ], + "title": "Provider Outcome", + "type": "string" + }, + "provider_status": { + "default": "", + "maxLength": 128, + "title": "Provider Status", + "type": "string" + }, + "provider_target_key_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Provider Target Key Sha256", + "type": "string" + }, + "reconciliation_key_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Reconciliation Key Sha256", + "type": "string" + }, + "recovery_digest": { + "pattern": "^[0-9a-f]{64}$", + "title": "Recovery Digest", + "type": "string" + }, + "reservation_attempt": { + "minimum": 1.0, + "title": "Reservation Attempt", + "type": "integer" + }, + "reservation_created_at": { + "title": "Reservation Created At", + "type": "string" + }, + "reservation_lease_expires_at": { + "default": "", + "title": "Reservation Lease Expires At", + "type": "string" + }, + "reservation_state": { + "enum": [ + "running", + "completed", + "reconcile_required" + ], + "title": "Reservation State", + "type": "string" + }, + "reservation_updated_at": { + "title": "Reservation Updated At", + "type": "string" + }, + "retry_safe": { + "title": "Retry Safe", + "type": "boolean" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "status": { + "const": "ok", + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "required": [ + "product", + "context", + "instance", + "reservation_state", + "reservation_attempt", + "reservation_created_at", + "reservation_updated_at", + "observed_at", + "reconciliation_key_sha256", + "provider_target_key_sha256", + "provider_effect_phase", + "provider_outcome", + "retry_safe", + "proposed_action", + "recovery_digest" + ], + "title": "GenericWebDeployRecoveryDryRunResponse", + "type": "object" + }, "GenericWebDeployRequest": { "additionalProperties": false, "properties": { @@ -33738,6 +33909,115 @@ "summary": "Logout human auth session" } }, + "/v1/admin/generic-web/deploy-recovery/dry-run": { + "post": { + "operationId": "dry_run_generic_web_deploy_recovery", + "parameters": [ + { + "in": "header", + "name": "Idempotency-Key", + "required": true, + "schema": { + "title": "Idempotency-Key", + "type": "string" + } + }, + { + "in": "header", + "name": "Authorization", + "required": false, + "schema": { + "default": "", + "title": "Authorization", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenericWebDeployRecoveryDryRunRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenericWebDeployRecoveryDryRunResponse" + } + } + }, + "description": "Successful Response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Conflict" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LaunchplaneErrorResponse" + } + } + }, + "description": "Service Unavailable" + } + }, + "summary": "Inspect an existing generic web deploy reservation without writing" + } + }, "/v1/agent/context": { "get": { "operationId": "read_agent_context", diff --git a/tests/test_generic_web_deploy_recovery.py b/tests/test_generic_web_deploy_recovery.py new file mode 100644 index 000000000..bb8672d81 --- /dev/null +++ b/tests/test_generic_web_deploy_recovery.py @@ -0,0 +1,1169 @@ +import base64 +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Literal, cast +from unittest.mock import patch + +from click import ClickException + +from control_plane.contracts.deployment_record import ResolvedTargetEvidence +from control_plane.contracts.idempotency_record import LaunchplaneIdempotencyRecord +from control_plane.contracts.product_profile_record import LaunchplaneProductProfileRecord +from control_plane.contracts.promotion_record import HealthcheckEvidence +from control_plane.contracts.ship_request import ShipRequest +from control_plane.http_app import idempotency_request_fingerprint +from control_plane.service_auth import BearerIdentityConfig +from control_plane.storage.postgres import ( + ExistingMutationReservationLookupResult, + PostgresRecordStore, +) +from control_plane.workflows.generic_web_deploy import GenericWebDeployResult +from control_plane.workflows.generic_web_deploy_provider import ( + GenericWebProviderDeploymentObservation, + GenericWebResolvedDeployTarget, + build_generic_web_provider_reconciliation_key, + build_generic_web_provider_target_key, +) +from tests.support.auth import _StubVerifier, _identity, _local_operator_policy +from tests.support.profiles import _product_profile_payload +from tests.support.stores import _sqlite_database_url +from tests.test_service import _invoke_app, create_launchplane_fastapi_test_app + + +_RECOVERY_ROUTE = "/v1/admin/generic-web/deploy-recovery/dry-run" +_OPERATOR_TOKEN = "local-operator-token" + + +def _create_recovery_app( + *, + root: Path, + store: PostgresRecordStore, + actions: tuple[str, ...] = ("generic_web_deploy.execute",), + contexts: tuple[str, ...] = ("sellyouroutboard-testing",), +) -> Any: + return create_launchplane_fastapi_test_app( + local_record_store_for_tests=store, + state_dir=root / "state", + verifier=_StubVerifier(_identity()), + authz_policy=_local_operator_policy( + actions=actions, + products=("sellyouroutboard",), + contexts=contexts, + ), + control_plane_root_path=root, + bearer_identity_config=BearerIdentityConfig( + local_operator_token=_OPERATOR_TOKEN, + local_operator_subject="local-owner-agent", + local_operator_token_label="local-owner-write", + ), + ) + + +def _invoke_recovery( + app: Any, + *, + original_deploy: dict[str, object], + idempotency_key: str, + reason: str, +) -> tuple[int, dict[str, Any]]: + return _invoke_app( + app, + method="POST", + path=_RECOVERY_ROUTE, + authorization=f"Bearer {_OPERATOR_TOKEN}", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": reason, + }, + headers={"Idempotency-Key": idempotency_key}, + ) + + +def _generic_web_deploy_result( + *, + deployment_record_id: str = "deployment-syo-testing", + deploy_status: Literal["pass", "fail"] = "pass", + post_deploy_status: Literal["pass", "fail", "skipped"] = "skipped", + error_message: str = "", +) -> GenericWebDeployResult: + return GenericWebDeployResult( + deployment_record_id=deployment_record_id, + deploy_status=deploy_status, + deploy_started_at="2026-05-26T02:00:00Z", + deploy_finished_at="2026-05-26T02:05:00Z", + product="sellyouroutboard", + context="sellyouroutboard-testing", + instance="testing", + target_name="syo-testing", + target_id="app-syo-testing", + target_category="application", + provider_id="dokploy", + provider_target_type="application", + post_deploy_status=post_deploy_status, + error_message=error_message, + ) + + +def _generic_web_recovery_original_deploy() -> dict[str, object]: + return { + "schema_version": 1, + "product": "sellyouroutboard", + "deploy": { + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "artifact_id": "ghcr.io/cbusillo/sellyouroutboard@sha256:abc123", + "source_git_ref": "abc123", + }, + } + + +def _generic_web_recovery_target( + *, context: str = "sellyouroutboard-testing" +) -> GenericWebResolvedDeployTarget: + return GenericWebResolvedDeployTarget( + ship_request=ShipRequest( + artifact_id="ghcr.io/cbusillo/sellyouroutboard@sha256:abc123", + context=context, + instance="testing", + source_git_ref="abc123", + target_name="syo-testing", + target_type="application", + provider_id="dokploy", + target_category="application", + provider_target_type="application", + deploy_mode="application", + provider_deploy_mode="application", + destination_health=HealthcheckEvidence(status="skipped"), + ), + resolved_target=ResolvedTargetEvidence( + target_type="application", + target_id="app-syo-testing", + target_name="syo-testing", + ), + deploy_timeout_seconds=900, + ) + + +def _legacy_generic_web_reconciliation_key( + target: GenericWebResolvedDeployTarget, +) -> str: + ship_request = target.ship_request + legacy_snapshot = { + "schema_version": 1, + "context": ship_request.context, + "instance": ship_request.instance, + "provider_id": ship_request.provider_id, + "target_category": ship_request.target_category, + "provider_target_type": ship_request.provider_target_type, + "target_type": target.resolved_target.target_type, + "target_id": target.resolved_target.target_id, + "target_name": target.resolved_target.target_name, + "deploy_mode": ship_request.deploy_mode, + "provider_deploy_mode": ship_request.provider_deploy_mode, + "deploy_timeout_seconds": target.deploy_timeout_seconds, + } + encoded = base64.urlsafe_b64encode( + json.dumps(legacy_snapshot, separators=(",", ":")).encode("utf-8") + ).decode("ascii") + return f"generic-web-provider-target:{encoded.rstrip('=')}" + + +def _generic_web_recovery_reservation( + *, + original_deploy: dict[str, object], + idempotency_key: str, + state: Literal["running", "reconcile_required"] = "reconcile_required", + context: str = "sellyouroutboard-testing", + lease_expires_at: str = "2099-08-16T18:00:00Z", + provider_effect_phase: str = "target_update", + provider_target_key: str | None = None, +) -> LaunchplaneIdempotencyRecord: + target = _generic_web_recovery_target(context=context) + reconciliation_key = build_generic_web_provider_reconciliation_key( + target, + product="sellyouroutboard", + ) + return LaunchplaneIdempotencyRecord( + record_id=f"idempotency-{idempotency_key}", + scope="legacy/github-actions/scope", + route_path="/v1/drivers/generic-web/deploy", + idempotency_key=idempotency_key, + request_fingerprint=idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ), + state=state, + lease_owner="legacy-worker", + lease_expires_at=lease_expires_at if state == "running" else "", + reconciliation_key=reconciliation_key, + provider_target_key=( + build_generic_web_provider_target_key(target) + if provider_target_key is None + else provider_target_key + ), + provider_effect_phase=provider_effect_phase, + provider_effect_started_at=("2026-08-15T12:00:00Z" if provider_effect_phase else ""), + created_at="2026-08-15T11:55:00Z", + updated_at="2026-08-15T12:00:00Z", + ) + + +def _write_generic_web_recovery_reservation( + store: PostgresRecordStore, + reservation: LaunchplaneIdempotencyRecord, +) -> LaunchplaneIdempotencyRecord: + reserved = store.reserve_mutation( + scope=reservation.scope, + route_path=reservation.route_path, + idempotency_key=reservation.idempotency_key, + request_fingerprint=reservation.request_fingerprint, + lease_owner=reservation.lease_owner, + lease_seconds=300, + reconciliation_key=reservation.reconciliation_key, + provider_target_key=reservation.provider_target_key, + ).record + if reservation.provider_effect_phase: + checkpointed = store.checkpoint_mutation_provider_effect( + reservation=reserved, + effect_phase=reservation.provider_effect_phase, + lease_seconds=300, + ) + assert checkpointed.record is not None + reserved = checkpointed.record + if reservation.state == "reconcile_required": + reconciled = store.mark_mutation_reconcile_required( + reservation=reserved, + reconciliation_key=reservation.reconciliation_key, + ) + assert reconciled.record is not None + reserved = reconciled.record + return reserved + + +class _RecoveryObservationProvider: + provider_id = "dokploy" + delegated_executor = "test" + + def __init__(self, observation: GenericWebProviderDeploymentObservation) -> None: + self.observation = observation + self.observation_calls = 0 + + def observe_artifact_deploy(self, **_kwargs: object) -> GenericWebProviderDeploymentObservation: + self.observation_calls += 1 + return self.observation + + +class _FailingRecoveryObservationProvider(_RecoveryObservationProvider): + def observe_artifact_deploy(self, **_kwargs: object) -> GenericWebProviderDeploymentObservation: + self.observation_calls += 1 + raise ClickException("provider read failed") + + +class GenericWebDeployRecoveryHttpTests(unittest.TestCase): + def test_generic_web_deploy_recovery_dry_run_replays_without_writes(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = { + "schema_version": 1, + "product": "sellyouroutboard", + "deploy": { + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "artifact_id": "ghcr.io/cbusillo/sellyouroutboard@sha256:abc123", + "source_git_ref": "abc123", + }, + } + idempotency_key = "legacy-generic-web-deploy" + reservation = LaunchplaneIdempotencyRecord( + record_id="idempotency-legacy-generic-web-deploy", + scope="legacy/github-actions/scope", + route_path="/v1/drivers/generic-web/deploy", + idempotency_key=idempotency_key, + request_fingerprint=idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ), + response_status_code=202, + response_trace_id="trace-original-deploy", + recorded_at="2026-08-15T12:00:00Z", + response_payload={ + "status": "accepted", + "trace_id": "trace-original-deploy", + "records": {"deployment_record_id": "deployment-original"}, + "result": _generic_web_deploy_result( + deployment_record_id="deployment-original" + ).model_dump(mode="json"), + }, + ) + store.write_idempotency_record(reservation) + app = _create_recovery_app(root=root, store=store) + before = store.read_idempotency_record( + scope=reservation.scope, + route_path=reservation.route_path, + idempotency_key=reservation.idempotency_key, + ) + + with ( + patch.object(store, "reserve_mutation", side_effect=AssertionError("mutation")), + patch.object( + store, + "write_deployment_record", + side_effect=AssertionError("deployment write"), + ), + patch.object( + store, + "write_environment_inventory", + side_effect=AssertionError("inventory write"), + ), + patch.object( + store, + "write_idempotency_record", + side_effect=AssertionError("idempotency write"), + ), + ): + status_code, payload = _invoke_recovery( + app, + original_deploy=original_deploy, + idempotency_key=idempotency_key, + reason="Inspect the legacy reservation before recovery.", + ) + + after = store.read_idempotency_record( + scope=reservation.scope, + route_path=reservation.route_path, + idempotency_key=reservation.idempotency_key, + ) + deployments = store.list_deployment_records() + inventory = store.list_environment_inventory() + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["proposed_action"], "replay_completed") + self.assertEqual(payload["provider_outcome"], "not_inspected") + self.assertEqual(payload["provider_status"], "pass") + self.assertRegex(payload["recovery_digest"], r"^[0-9a-f]{64}$") + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn(idempotency_key, serialized) + self.assertNotIn(reservation.scope, serialized) + self.assertNotIn("trace-original-deploy", serialized) + self.assertEqual(after, before) + self.assertEqual(deployments, ()) + self.assertEqual(inventory, ()) + + def test_generic_web_deploy_recovery_missing_reservation_fails_closed(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + app = _create_recovery_app(root=root, store=store) + status_code, payload = _invoke_recovery( + app, + original_deploy=_generic_web_recovery_original_deploy(), + idempotency_key="missing-legacy-reservation", + reason="Inspect a suspected legacy reservation.", + ) + store.close() + + self.assertEqual(status_code, 404) + self.assertEqual(payload["error"]["code"], "reservation_not_found") + + def test_generic_web_deploy_recovery_denies_before_reservation_lookup(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + app = _create_recovery_app( + root=root, + store=store, + actions=("generic_web_preview.execute",), + ) + with patch.object( + store, + "lookup_existing_mutation_reservation", + side_effect=AssertionError("lookup disclosure"), + ): + status_code, payload = _invoke_recovery( + app, + original_deploy=_generic_web_recovery_original_deploy(), + idempotency_key="legacy-deploy-key", + reason="Verify authorization before lookup.", + ) + store.close() + + self.assertEqual(status_code, 403) + self.assertEqual(payload["error"]["code"], "authorization_denied") + + def test_generic_web_deploy_recovery_rejects_conflict_and_ambiguity(self) -> None: + for expected_code, scopes, stored_fingerprint in ( + ("reservation_conflict", ("scope-a",), "different-fingerprint"), + ("reservation_ambiguous", ("scope-a", "scope-b"), "exact"), + ): + with ( + self.subTest(expected_code=expected_code), + TemporaryDirectory() as temporary_directory_name, + ): + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + exact_fingerprint = idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ) + for index, scope in enumerate(scopes): + store.reserve_mutation( + scope=scope, + route_path="/v1/drivers/generic-web/deploy", + idempotency_key="legacy-deploy-key", + request_fingerprint=( + exact_fingerprint + if stored_fingerprint == "exact" + else stored_fingerprint + ), + lease_owner=f"worker-{index}", + lease_seconds=300, + ) + app = _create_recovery_app(root=root, store=store) + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Verify exact cross-scope lookup.", + }, + headers={"Idempotency-Key": "legacy-deploy-key"}, + ) + store.close() + + self.assertEqual(status_code, 409) + self.assertEqual(payload["error"]["code"], expected_code) + + def test_generic_web_deploy_recovery_waits_at_active_lease_boundary_without_writes( + self, + ) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key="active-lease", + state="running", + ) + reservation = _write_generic_web_recovery_reservation(store, reservation) + provider = _RecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="unknown") + ) + app = _create_recovery_app(root=root, store=store) + with ( + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + patch.object(store, "reserve_mutation", side_effect=AssertionError("reserve")), + patch.object( + store, "write_deployment_record", side_effect=AssertionError("deployment write") + ), + patch.object( + store, + "write_environment_inventory", + side_effect=AssertionError("inventory write"), + ), + patch.object( + store, + "write_idempotency_record", + side_effect=AssertionError("idempotency write"), + ), + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Verify active lease fencing.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + provider.observation = GenericWebProviderDeploymentObservation(outcome="absent") + boundary_lookup = ExistingMutationReservationLookupResult( + status="found", + record=reservation, + observed_at=reservation.lease_expires_at, + ) + with ( + patch.object( + store, + "lookup_existing_mutation_reservation", + return_value=boundary_lookup, + ), + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + ): + boundary_status_code, boundary_payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Verify the exact lease expiry boundary.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["proposed_action"], "wait_for_active_lease") + self.assertEqual(boundary_status_code, 200) + self.assertEqual(boundary_payload["proposed_action"], "retry_original_operation") + self.assertEqual(provider.observation_calls, 1) + + def test_generic_web_deploy_recovery_classifies_provider_observations_without_writes( + self, + ) -> None: + present = GenericWebProviderDeploymentObservation( + outcome="present", + deployment_status="success", + deployment_id="deployment-123", + started_at="2026-08-15T12:00:00Z", + finished_at="2026-08-15T12:05:00Z", + ) + for observation, expected_action, expected_retry_safe in ( + (present, "adopt_observed", False), + ( + GenericWebProviderDeploymentObservation(outcome="absent"), + "retry_original_operation", + True, + ), + ( + GenericWebProviderDeploymentObservation(outcome="unknown"), + "hold_unknown", + False, + ), + ): + with ( + self.subTest(outcome=observation.outcome), + TemporaryDirectory() as temporary_directory_name, + ): + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key=f"provider-{observation.outcome}", + ) + reservation = _write_generic_web_recovery_reservation(store, reservation) + provider = _RecoveryObservationProvider(observation) + app = _create_recovery_app(root=root, store=store) + with ( + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + patch.object(store, "reserve_mutation", side_effect=AssertionError("reserve")), + patch.object( + store, + "write_deployment_record", + side_effect=AssertionError("deployment write"), + ), + patch.object( + store, + "write_environment_inventory", + side_effect=AssertionError("inventory write"), + ), + patch.object( + store, + "write_idempotency_record", + side_effect=AssertionError("idempotency write"), + ), + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Classify provider observation without mutation.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["proposed_action"], expected_action) + self.assertEqual(payload["retry_safe"], expected_retry_safe) + self.assertEqual(provider.observation_calls, 1) + + def test_generic_web_deploy_recovery_holds_on_provider_read_uncertainty(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = _write_generic_web_recovery_reservation( + store, + _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key="provider-read-failure", + ), + ) + provider = _FailingRecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="unknown") + ) + app = _create_recovery_app(root=root, store=store) + with ( + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + patch.object(store, "reserve_mutation", side_effect=AssertionError("reserve")), + patch.object( + store, "write_deployment_record", side_effect=AssertionError("deployment write") + ), + patch.object( + store, + "write_environment_inventory", + side_effect=AssertionError("inventory write"), + ), + patch.object( + store, + "write_idempotency_record", + side_effect=AssertionError("idempotency write"), + ), + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Hold when provider observation is uncertain.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["provider_outcome"], "unknown") + self.assertEqual(payload["proposed_action"], "hold_unknown") + self.assertEqual(provider.observation_calls, 1) + + def test_generic_web_deploy_recovery_rejects_stored_target_identity_before_provider_read( + self, + ) -> None: + for identity_kind in ( + "provider_target", + "malformed_reconciliation", + "instance_mismatch", + "product_mismatch", + "empty_product", + ): + with ( + self.subTest(identity_kind=identity_kind), + TemporaryDirectory() as temporary_directory_name, + ): + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key=f"identity-{identity_kind}", + provider_target_key=( + "generic-web-provider-target:" + "0" * 64 + if identity_kind == "provider_target" + else None + ), + ) + if identity_kind == "malformed_reconciliation": + reservation = reservation.model_copy( + update={"reconciliation_key": "malformed-reconciliation-key"} + ) + elif identity_kind == "instance_mismatch": + mismatched_target = _generic_web_recovery_target() + mismatched_target.ship_request.instance = "prod" + reservation = reservation.model_copy( + update={ + "reconciliation_key": build_generic_web_provider_reconciliation_key( + mismatched_target, + product="sellyouroutboard", + ), + "provider_target_key": build_generic_web_provider_target_key( + mismatched_target + ), + } + ) + elif identity_kind == "product_mismatch": + reservation = reservation.model_copy( + update={ + "reconciliation_key": build_generic_web_provider_reconciliation_key( + _generic_web_recovery_target(), + product="different-product", + ) + } + ) + elif identity_kind == "empty_product": + reservation = reservation.model_copy( + update={ + "reconciliation_key": build_generic_web_provider_reconciliation_key( + _generic_web_recovery_target() + ) + } + ) + reservation = _write_generic_web_recovery_reservation(store, reservation) + provider = _RecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="unknown") + ) + app = _create_recovery_app(root=root, store=store) + with patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Reject invalid stored reconciliation identity.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 409) + self.assertEqual(payload["error"]["code"], "reservation_target_conflict") + self.assertEqual(provider.observation_calls, 0) + + def test_generic_web_deploy_recovery_accepts_origin_main_legacy_snapshot(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + target = _generic_web_recovery_target() + reservation = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key="origin-main-legacy-snapshot", + ).model_copy( + update={ + "reconciliation_key": _legacy_generic_web_reconciliation_key(target), + "provider_target_key": build_generic_web_provider_target_key(target), + } + ) + reservation = _write_generic_web_recovery_reservation(store, reservation) + provider = _RecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="absent") + ) + app = _create_recovery_app(root=root, store=store) + with ( + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + patch.object(store, "reserve_mutation", side_effect=AssertionError("mutation")), + patch.object( + store, + "write_deployment_record", + side_effect=AssertionError("deployment write"), + ), + patch.object( + store, + "write_environment_inventory", + side_effect=AssertionError("inventory write"), + ), + patch.object( + store, + "write_idempotency_record", + side_effect=AssertionError("idempotency write"), + ), + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Classify the origin/main legacy reservation safely.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["context"], "sellyouroutboard-testing") + self.assertEqual(payload["provider_outcome"], "absent") + self.assertTrue(payload["retry_safe"]) + self.assertEqual(payload["proposed_action"], "retry_original_operation") + self.assertEqual(provider.observation_calls, 1) + + def test_generic_web_deploy_recovery_reauthorizes_and_reports_stored_context(self) -> None: + for allowed_contexts, expected_status in ( + (("sellyouroutboard-current",), 403), + (("sellyouroutboard-current", "sellyouroutboard-testing"), 200), + ): + with ( + self.subTest(expected_status=expected_status), + TemporaryDirectory() as temporary_directory_name, + ): + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + profile_payload = _product_profile_payload() + lanes = cast(tuple[dict[str, object], ...], profile_payload["lanes"]) + lanes[0]["context"] = "sellyouroutboard-current" + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(profile_payload) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key=f"context-drift-{expected_status}", + ) + reservation = _write_generic_web_recovery_reservation(store, reservation) + provider = _RecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="absent") + ) + app = _create_recovery_app( + root=root, + store=store, + contexts=allowed_contexts, + ) + with patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Verify stored context authority after profile drift.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, expected_status) + if expected_status == 403: + self.assertEqual(payload["error"]["code"], "authorization_denied") + self.assertEqual(provider.observation_calls, 0) + else: + self.assertEqual(payload["context"], "sellyouroutboard-testing") + self.assertEqual(payload["proposed_action"], "retry_original_operation") + self.assertEqual(provider.observation_calls, 1) + + def test_generic_web_deploy_recovery_rejects_malformed_running_lease_as_hold_unknown( + self, + ) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + valid = _generic_web_recovery_reservation( + original_deploy=original_deploy, + idempotency_key="malformed-lease", + state="running", + ) + malformed = valid.model_copy(update={"lease_expires_at": "not-a-timestamp"}) + provider = _RecoveryObservationProvider( + GenericWebProviderDeploymentObservation(outcome="unknown") + ) + app = _create_recovery_app(root=root, store=store) + lookup = ExistingMutationReservationLookupResult( + status="found", + record=malformed, + observed_at="2026-08-16T17:00:00Z", + ) + with ( + patch.object(store, "lookup_existing_mutation_reservation", return_value=lookup), + patch( + "control_plane.generic_web_deploy_provider_adapter." + "default_generic_web_deploy_provider", + return_value=provider, + ), + ): + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Reject malformed lease timing.", + }, + headers={"Idempotency-Key": valid.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 409) + self.assertEqual(payload["error"]["code"], "hold_unknown") + self.assertEqual(provider.observation_calls, 0) + + def test_generic_web_deploy_recovery_completed_context_must_be_exact(self) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = LaunchplaneIdempotencyRecord( + record_id="idempotency-completed-inexact", + scope="legacy/github-actions/scope", + route_path="/v1/drivers/generic-web/deploy", + idempotency_key="completed-inexact", + request_fingerprint=idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ), + response_status_code=202, + response_trace_id="trace-completed-inexact", + recorded_at="2026-08-15T12:00:00Z", + response_payload={"result": {"deploy_status": "pass"}}, + ) + store.write_idempotency_record(reservation) + app = _create_recovery_app(root=root, store=store) + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Require exact completed operation context.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 409) + self.assertEqual(payload["error"]["code"], "reservation_target_conflict") + + def test_generic_web_deploy_recovery_completed_identity_must_be_nonblank(self) -> None: + for field_name in ("product", "context", "instance"): + with ( + self.subTest(field_name=field_name), + TemporaryDirectory() as temporary_directory_name, + ): + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(_product_profile_payload()) + ) + original_deploy = _generic_web_recovery_original_deploy() + completed_result = _generic_web_deploy_result().model_dump(mode="json") + completed_result[field_name] = " " + reservation = LaunchplaneIdempotencyRecord( + record_id=f"idempotency-completed-blank-{field_name}", + scope="legacy/github-actions/scope", + route_path="/v1/drivers/generic-web/deploy", + idempotency_key=f"completed-blank-{field_name}", + request_fingerprint=idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ), + response_status_code=202, + response_trace_id=f"trace-completed-blank-{field_name}", + recorded_at="2026-08-15T12:00:00Z", + response_payload={"result": completed_result}, + ) + store.write_idempotency_record(reservation) + app = _create_recovery_app(root=root, store=store) + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Reject a blank completed operation identity.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 409) + self.assertEqual(payload["error"]["code"], "reservation_target_conflict") + + def test_generic_web_deploy_recovery_completed_response_reports_stored_context_after_drift( + self, + ) -> None: + with TemporaryDirectory() as temporary_directory_name: + root = Path(temporary_directory_name) + store = PostgresRecordStore( + database_url=_sqlite_database_url(root / "launchplane.sqlite3") + ) + store.ensure_schema() + profile_payload = _product_profile_payload() + lanes = cast(tuple[dict[str, object], ...], profile_payload["lanes"]) + lanes[0]["context"] = "sellyouroutboard-current" + store.write_product_profile_record( + LaunchplaneProductProfileRecord.model_validate(profile_payload) + ) + original_deploy = _generic_web_recovery_original_deploy() + reservation = LaunchplaneIdempotencyRecord( + record_id="idempotency-completed-context-drift", + scope="legacy/github-actions/scope", + route_path="/v1/drivers/generic-web/deploy", + idempotency_key="completed-context-drift", + request_fingerprint=idempotency_request_fingerprint( + route_path="/v1/drivers/generic-web/deploy", + payload=original_deploy, + ), + response_status_code=202, + response_trace_id="trace-completed-context-drift", + recorded_at="2026-08-15T12:00:00Z", + response_payload={ + "result": _generic_web_deploy_result( + deployment_record_id="deployment-completed-context-drift" + ).model_dump(mode="json") + }, + ) + store.write_idempotency_record(reservation) + app = _create_recovery_app( + root=root, + store=store, + contexts=("sellyouroutboard-current", "sellyouroutboard-testing"), + ) + status_code, payload = _invoke_app( + app, + method="POST", + path="/v1/admin/generic-web/deploy-recovery/dry-run", + authorization="Bearer local-operator-token", + payload={ + "schema_version": 1, + "product": "sellyouroutboard", + "instance": "testing", + "original_deploy": original_deploy, + "reason": "Report authoritative completed operation context.", + }, + headers={"Idempotency-Key": reservation.idempotency_key}, + ) + store.close() + + self.assertEqual(status_code, 200) + self.assertEqual(payload["context"], "sellyouroutboard-testing") + self.assertEqual(payload["proposed_action"], "replay_completed") diff --git a/tests/test_http_write_route_registrars.py b/tests/test_http_write_route_registrars.py index 979784ee0..1c5da0ae6 100644 --- a/tests/test_http_write_route_registrars.py +++ b/tests/test_http_write_route_registrars.py @@ -14,8 +14,30 @@ def setUp(self) -> None: authz_policy=LaunchplaneAuthzPolicy(), record_store_factory=object, ) + self.app = app self.api_routes = [route for route in app.routes if isinstance(route, APIRoute)] + def test_generic_web_deploy_recovery_openapi_contract(self) -> None: + route = self.app.openapi()["paths"]["/v1/admin/generic-web/deploy-recovery/dry-run"]["post"] + + self.assertEqual(route["operationId"], "dry_run_generic_web_deploy_recovery") + self.assertEqual( + route["requestBody"]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/GenericWebDeployRecoveryDryRunRequest", + ) + self.assertEqual( + route["responses"]["200"]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/GenericWebDeployRecoveryDryRunResponse", + ) + idempotency_header = next( + parameter + for parameter in route["parameters"] + if parameter["in"] == "header" and parameter["name"] == "Idempotency-Key" + ) + self.assertTrue(idempotency_header["required"]) + for status_code in ("400", "401", "403", "404", "409", "503"): + self.assertIn(status_code, route["responses"]) + def test_evidence_write_routes_preserve_contracts_and_ownership(self) -> None: expected_routes = ( ("/v1/evidence/backup-gates", "write_backup_gate_evidence"), @@ -96,6 +118,11 @@ def test_generic_web_write_routes_preserve_contracts_and_ownership(self) -> None "apply_generic_web_deploy", "AcceptedEvidenceResponse", ), + ( + "/v1/admin/generic-web/deploy-recovery/dry-run", + "dry_run_generic_web_deploy_recovery", + "GenericWebDeployRecoveryDryRunResponse", + ), ( "/v1/drivers/generic-web/prod-promotion", "apply_generic_web_prod_promotion", @@ -143,7 +170,12 @@ def test_generic_web_write_routes_preserve_contracts_and_ownership(self) -> None ): self.assertEqual(route.operation_id, operation_id) self.assertEqual(route.response_model.__name__, response_model) - self.assertEqual(route.endpoint.__module__, "control_plane.http_routes.generic_web") + expected_module = ( + "control_plane.generic_web_deploy_recovery_http" + if route.path == "/v1/admin/generic-web/deploy-recovery/dry-run" + else "control_plane.http_routes.generic_web" + ) + self.assertEqual(route.endpoint.__module__, expected_module) def test_generic_web_write_routes_preserve_interleaved_route_order(self) -> None: route_keys = [(next(iter(route.methods or set())), route.path) for route in self.api_routes] @@ -155,6 +187,7 @@ def test_generic_web_write_routes_preserve_interleaved_route_order(self) -> None ("POST", "/v1/drivers/generic-web/preview-refresh"), ("POST", "/v1/drivers/generic-web/preview-destroy"), ("POST", "/v1/drivers/generic-web/deploy"), + ("POST", "/v1/admin/generic-web/deploy-recovery/dry-run"), ("POST", "/v1/drivers/generic-web/prod-promotion"), ("POST", "/v1/drivers/generic-web/prod-promotion-workflow"), ("POST", "/v1/drivers/generic-web/stable-verification"),