diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5850930..2f8ab79 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -159,6 +159,16 @@ jobs: - name: Exercise prepared backend with fake dependencies only run: python3 scripts/incident_transformers_controller.py fixture --out "$RUNNER_TEMP/incident-transformers-fixture" + - name: Exercise hosted contract and accounting without HTTP or credentials + run: | + python3 scripts/incident_hosted_controller.py fixture --out "$RUNNER_TEMP/incident-hosted-fixture" + python3 scripts/incident_hosted_controller.py audit "$RUNNER_TEMP/incident-hosted-fixture" + + - name: Exercise bounded hosted process runner without network access + run: | + python3 scripts/incident_hosted_runner.py fixture --out "$RUNNER_TEMP/incident-hosted-runner" + python3 scripts/incident_hosted_runner.py audit "$RUNNER_TEMP/incident-hosted-runner" + checkout-previous: name: checkout previous stable (v6.1.0) runs-on: ubuntu-latest diff --git a/README.md b/README.md index d8ed0c0..f44d364 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,12 @@ by default. Its automated checks use fake dependencies only; real calibration still requires provisioned assets and separately approved execution. The [proposed GitHub/NixOS comparison](docs/calibration-host-matrix.md) lists matching requirements, readiness gates and unapproved per-host bounds. +The [offline Adaption adapter](docs/incident-adaption-backend.md) exercises a +distinct hosted-response contract and synthetic accounting, without credentials, +network requests, or live execution authority. +The [one-request hosted runner](docs/incident-hosted-runner.md) adds fixed-origin +HTTPS code and a bounded child process, tested only with mocked HTTP and canned +receipts. Live use requires separate approval and a verified billing bound. ## Implemented contracts diff --git a/docs/hosted-integration.md b/docs/hosted-integration.md index 8ef29ad..3fe67fc 100644 --- a/docs/hosted-integration.md +++ b/docs/hosted-integration.md @@ -60,3 +60,28 @@ are unchanged. Evidence is retained in `submission-finalization-mkpl39bc/pr18/integrated/fixture-fix/`; exact-head hosted checks remain required. The injected supervisor clock is deterministic test evidence, not a real-model timing measurement. + +The PR #18 Python-previous run at `3c4a386` then exposed a separate hosted-runner +fixture assumption. Its crash/hang/partial test allowed only 200 ms for durable +setup and execution; expiry before the claim correctly retained zero hold, so +the intended partial-response path was never reached. A 250 ms delay after the +real request write reproduces that failure and is retained with the original +hosted log. + +Hosted-runner protocol and audit fixtures now use a bounded 30-second startup +allowance. Crash and partial-response tests require one actual claim, the full +400-unit synthetic hold, the expected child exit, and the exact retained partial +body length. The separate hang test waits for the real forked child to enter +its hanging send path before advancing only the parent's test clock to expiry; +the real watchdog must kill and reap it. Dedicated deterministic tests still +exercise expiry both before and after the claim, forbid a fork at either +boundary, and require respectively zero hold or the full 400-unit hold. + +All 15 hosted-runner tests passed both normally and with the injected setup +delay. The full 440-test Python scripts suite passed with the accepted runtime +and verified existing Rust binaries. This correction changes only PR #18 tests +and this record; production deadlines, claims, holds and live authority are +unchanged. Evidence is retained in +`submission-finalization-mkpl39bc/pr18/integrated/hosted-fixture-fix/`. +Exact-head hosted acceptance remains required; these offline process fixtures +do not establish provider behavior or billing. diff --git a/docs/incident-adaption-backend.md b/docs/incident-adaption-backend.md new file mode 100644 index 0000000..3d85ce6 --- /dev/null +++ b/docs/incident-adaption-backend.md @@ -0,0 +1,143 @@ +# Offline Adaption hosted-response adapter + +This milestone implements a **hosted request/receipt contract and synthetic +accounting tests**, not a live Adaption integration. There is no HTTP client, +credential lookup, account query, model call, or paid operation. Invocation is +denied unless the caller supplies the explicit offline marker and the fixed +fixture transport. `live_admissible` remains false. +`OfflineApproval` is a fake-only caller marker, not user permission or a security +capability. + +The subsequent [one-request hosted runner](incident-hosted-runner.md) supplies +separate HTTPS and process-control code, tested offline only. The contract and +synthetic ledger described here remain unchanged; they cannot launch that runner +or supply live authority. + +This is separate from the [local Transformers backend](incident-transformers-backend.md). +Freezing an HTTP request body does not retain the provider's tokenized inputs, +measure its exact input count, pin a model deployment revision, or enforce its +billing. The local worker's token reservation and hardware profile cannot be +reused as hosted guarantees. + +## Offline commands and scope + +From the repository root: + +```sh +python scripts/incident_hosted_controller.py fixture --out /tmp/incident-hosted-fixture +python scripts/incident_hosted_controller.py audit /tmp/incident-hosted-fixture +python -m unittest discover -s scripts -p test_incident_adaption_backend.py +python -m unittest discover -s scripts -p test_incident_hosted_controller.py +``` + +The destination must not exist. The fixture command currently runs **one canned +`cal-01` response** as a contract smoke test. Its placeholder model ID is not an +available or selected Adaption model. It performs zero model calls; synthetic +usage and estimated charges are not measurements of provider performance or cost. +There is no live flag or arbitrary transport option. A ledger permits at most +four claimed initial requests; the smoke command uses only one. + +The renderer supports the four frozen calibration cases in +[incident-routing/1](incident-routing-workload.md). The proposed hosted comparison +is eight probes: four initial requests, then four fixed protocol-feedback +requests. That comparison is not implemented by this one-case smoke command or +approved for execution. Held-out evaluation is separate future work; no held-out +results, accuracy claims, or production latency profile are produced here. + +## Request and response contract + +`incident_adaption_backend.render_case` emits canonical UTF-8 request bytes, +their hash, and workload/evidence provenance for `POST /api/v1/responses`. It +requires an explicit model ID, refuses stack IDs, and renders only the frozen +prompt and selected calibration evidence. Gold labels and case-routing metadata +do not enter the HTTP body. Missing, stale, or refused evidence causes refusal, +not silent filtering. These eligibility declarations remain fixture/replay +attestations, not authenticated Store admission. + +Requests set `max_output_tokens=96`, `temperature=0`, `stream=false`, +`background=false`, `tools=[]`, `tool_choice="none"`, `agentic_memory=false`, and +`store=false`, with no prior-response linkage. These are requested controls, not +proof of provider behavior. The public [Responses creation reference](https://docs.adaptionlabs.ai/api/resources/responses/methods/create) +documents the endpoint and fields; this adapter does not assume OpenAI-compatible +features beyond its reviewed subset. + +`parse_response` keeps response attribution, completion state, output protocol +validity, and reported usage separate. It distinguishes refusals, errors, +unfinished output, unexpected tools, malformed envelopes, and invalid model JSON. +Assistant text parts must agree with the aggregate text; reasoning is not an +answer. The unchanged incident validator checks format and known IDs, **not +semantic correctness against gold**. A well-formed wrong answer can be +protocol-valid, and an invalid answer can still have accountable usage. + +Usage counts must be nonnegative integers with a consistent total. Missing, +partial, contradictory, or over-cap counts remain visible and are never repaired +or replaced with zero. Cached/reasoning detail is retained separately rather than +added to totals. Provider-reported counts are not independently measured tokens +or a verified invoice. + +## Durable fixture accounting + +`incident_hosted_controller.HostedLedger` provides operation creation, +reservation, claim, dispatch, receipt recording, reconciliation, cancellation, +and read-only audit. Operation keys bind immutable request bytes, provenance, +model, and quote. A SQLite transaction commits the single-use claim **before** +the fixture transport is invoked. Duplicate dispatches cannot resend that claim, +including after reopening the ledger. + +`OfflinePriceBound` uses integer **synthetic micro-USD** rates, fees, and maximum +holds. These are test inputs, not Adaption prices or an enforceable spending +limit. Bound, terminal receipts with complete valid usage can settle a synthetic +estimate once, including invalid answers or late receipts. Each hold must cover +the quote's worst-case synthetic cost. Missing or uncertain +usage retains the full hold and prevents further dispatch while unresolved. +Usage above the synthetic bound is preserved and flagged, not clipped or charged +as zero. A missing receipt clock is recorded as unknown, not a fabricated time. +Only planned or reserved, unclaimed work can be cancelled locally. Cancellation +of that ledger state is not cancellation of remote work. + +Artifacts are `manifest.json`, `ledger.sqlite`, and `receipt-.json`. +Receipts preserve bounded raw body bytes, HTTP status, request binding, receipt +tick, and allowlisted response headers. The receipt file and SQLite are separate +persistence routes before parsing/reconciliation. If both fail, a response held +only in memory may be lost; the last claim remains uncertain. No credential +configuration or authorization headers are implemented. Audits rebuild frozen +requests, check source/manifest bindings, and recompute parsing and estimates +from durable receipts. They report estimated spending, held bounds, parsed usage, +and issues; `observed_provider_cost` remains null. These are internal-consistency +checks, not authenticated provider provenance or tamper-proof storage. An audit +result is not a live run result, a correctness score, or a bill. +`audit_complete` means the consistency checks succeeded; operations may still +have unresolved holds. + +The fixture clock checks admission against a stored window. This milestone does +not provide a remote cancellation mechanism or a live hard-deadline controller. +A client timeout, disconnect, or local process kill would not prove that a +provider stopped computing or charging. The documented +[response deletion endpoint](https://docs.adaptionlabs.ai/api/resources/responses/methods/delete) +deletes stored responses and does not cancel in-progress work. No automatic +retries, redirects, retrieval, or retransmission are implemented. + +## Blockers before any live run + +- Verify account access to a selected hosted model through the account's model + catalogue. Public examples are not proof of availability to this account. + [Models list reference](https://docs.adaptionlabs.ai/api/resources/models/methods/list) +- Verify credits, applicable pricing, credit-to-currency conversion, and all + billable input/output/reasoning categories. The public catalogue lists credit + rates, but no account balance, effective price, or safe spending ceiling has + been established here. [Hosted model catalogue](https://docs.adaptionlabs.ai/inference-stacks/models/) +- Establish a defensible pre-send input and reasoning/output billing bound. The + reviewed API reference does not establish an exact input-count preflight, + idempotent-send contract, or provider-enforced strict JSON-schema output for + this adapter. Request hashes and `max_output_tokens` alone do not fill those + gaps. +- Obtain explicit approval for the selected service/model, transmitted data, + call count, total currency limit, and timeout/uncertainty handling. Supply + credentials through an approved secure mechanism only after that decision. +- Review the separate real transport and bounded controller for the approved run. + Freeze the final source/request configuration, preserve partial receipts, and + never treat a local timeout as permission to release a hold or resend. + +Current tests exercise the offline contract. They do not establish live API +compatibility, service availability, remote completion guarantees, or model +efficacy. Prior frozen workloads, local workers, and archived results are unchanged. diff --git a/docs/incident-calibration-controller.md b/docs/incident-calibration-controller.md index f04186f..51736e7 100644 --- a/docs/incident-calibration-controller.md +++ b/docs/incident-calibration-controller.md @@ -119,3 +119,18 @@ receipt, settlement, and artifact synchronization; late fallback receipts; supervisor death; duplicate event/conflicting receipt; changed manifest and corrupt accounting; and existing-destination refusal. Synthetic fixtures test the protocol, not Qwen preparation measurements or language-model efficacy. + +The receipt-hang accounting test waits for a real receipt artifact and a committed +SQLite `received` row with identical bytes and observation time, then advances only +the supervisor's injected clock to the original deadline. It verifies actual group +termination, SIGKILL/reaping, one claim, zero settlements, 180 known receipt units +and the full 269-unit hold. A separate regression delays process startup by two +seconds; the startup-expiry refusal test remains unchanged. These are deterministic +accounting/watchdog checks, not real-time latency measurements. Child/worker clocks, +production deadlines, receipts and accounting logic are unchanged. + +This boundary replaced a 1.5-second test window after integration CI run +`43819750749`, job `133139861895`, expired before the intended receipt existed. +A delayed-launch reproduction failed with unknown accounting before the correction; +the original CI failure and RED/GREEN receipts are retained in the local +`kairotensor-artifacts/pr-integration-20261116-2x6z4skr` packet. diff --git a/docs/incident-hosted-runner.md b/docs/incident-hosted-runner.md new file mode 100644 index 0000000..cd44046 --- /dev/null +++ b/docs/incident-hosted-runner.md @@ -0,0 +1,130 @@ +# One-request Adaption HTTP runner + +This milestone adds actual fixed-origin HTTPS transport code and a bounded +one-request process runner. **Only offline fixtures have been executed.** It +creates no credentials, reads no credentials in fixture mode, and offers no live +CLI switch. It does not establish provider availability, credit balances, +pricing, API compatibility, or model quality. + +## Offline verification + +```sh +python3 scripts/incident_hosted_runner.py fixture --out /tmp/incident-hosted-runner +python3 scripts/incident_hosted_runner.py audit /tmp/incident-hosted-runner +python3 -m unittest discover -s scripts -p test_incident_adaption_transport.py +python3 -m unittest discover -s scripts -p test_incident_hosted_runner.py +``` + +Use a fresh destination. The smoke runs a real child process with a canned HTTP +receipt, not a socket. Transport unit tests inject connection/response objects +into the same HTTP exchange logic without opening network connections. Tests +cover redirects, HTTP failures, partial responses, timeout, response-size bounds, +credential validation, deadline expiry, process crashes and hangs, duplicate +run refusal, and artifact consistency checks. + +The parent also reaps its child before propagating `KeyboardInterrupt` or +`SystemExit`. This matters when an embedding caller catches the interruption +and keeps the parent alive: parent-death protection would not fire. The claimed +reservation remains held and the interrupted directory cannot be reused. An +absent parent seal remains an incomplete outcome, not proof of nonexecution. +The separate synthetic ledger now closes SQLite connections after transaction +completion, including early returns and rollback; it does not wait for garbage +collection to release file descriptors. + +The previous `incident_hosted_controller.py` remains the separate synthetic +multi-operation accounting experiment. Its artifacts and guarantees are not +silently reused as live authority. This runner uses +`incident-hosted-single-request/1` and retains one full reservation throughout. + +## Transport boundary + +`incident_adaption_transport.send_https` sends exactly one POST to +`https://api.prod.adaptionlabs.ai/api/v1/responses`. It uses direct standard-library +HTTPS with default certificate and hostname verification. There are no proxy +lookups, redirects, automatic retries, connection reuse, tools, or model downloads. +The request comes from the frozen calibration renderer with tools, memory, +storage, and streaming disabled. This is a requested API configuration, not proof +that a remote provider obeyed it. + +The transport allows at most 64 KiB of request bytes and preserves at most 1 MiB +of response bytes. The HTTP parser's standard header-size/count limits also +apply. Socket inactivity timeouts are set to at most ten seconds and reduced to the +remaining window where the connection socket is available. The independent +parent deadline is authoritative, including when the HTTP library transfers a +closing connection to the response object. A parent process watchdog covers DNS, connection, +slow headers/body and blocking child code. Raw allowlisted response headers and +body chunks are fsynced and atomically renamed before the next read. Incomplete +`.tmp` records remain forensic evidence but are not trusted by the audit. Exceptions record their class, +never their text or request headers. Credential bytes are not put in manifests, +process arguments, queues, or logs. + +## One-shot admission and authority + +The library entry point `run_live` defaults to denial. A trusted caller must +supply a `LiveApproval` obtained from an explicit external approval process: + +- Exact request SHA-256 and model ID. +- An integer all-in request ceiling in micro-USD, within the run budget. +- Approval and conservative billing-bound references. +- Affirmative credential-use, data-transmission and all-billable-category scope. +- An approved maximum elapsed duration, at most twenty minutes. + +Only USD all-in request ceilings are supported. This is deliberately not a token +rate calculator. The ceiling must already cover all input, output, reasoning, +fees and uncertain remote completion, using verified account-specific terms. +A reference string or boolean is a **trusted caller attestation, not independently +verified pricing evidence, a security capability or user permission**. Artifact +JSON and environment variables are never loaded as launch authorization. Do not +construct an approval to bypass missing user approval or unverified billing. + +The parent validates those bindings before starting the child. SQLite commits a +single claim and full reservation before any possible request. The child checks +the original deadline, arms Linux parent-death termination, and only then reads +`ADAPTION_API_KEY` from the process environment for the fixed-origin HTTPS call. +The environment must be provisioned through an approved secure mechanism; never +paste keys into chat or put them in a command, repository, or artifact. + +The runner requires Linux `CLOCK_BOOTTIME` and uses the original startup window, +including rendering and setup. It does not restart, resume or resend. The output +directory must be new. A crashed claimed run is not a reason to create another +run without resolving its possible remote charge and obtaining any required +approval. + +## Accounting and failure behavior + +Artifacts include an immutable manifest and request, SQLite claim, numbered raw +transport records, and the parent outcome. Offline fixtures identify their +reservation as synthetic. Authorized live runs identify it as an approved +conservative USD bound. Neither is an invoice. + +The full reservation remains held even after a complete response. Provider usage +is retained separately from verified cost; `verified_provider_cost` stays null. +This milestone does not implement invoice reconciliation or release of live +holds. A transport failure, worker kill, missing receipt or partial response +therefore cannot create free budget. At most one request is attempted; no second +request can spend a released remainder in this runner. + +A local timeout or killed process **does not cancel remote computation or +billing**. The amount bound must cover that possibility independently. The +watchdog bounds local child work under normal OS scheduling, not kernel stalls, +remote server execution, or final parent artifact/audit time. If durable storage +fails, some returned bytes may be lost; the committed claim still reserves the +full ceiling. Auditing rebuilds the frozen request, validates source and claim +bindings, checks ordered raw records, and reparses any retained response. These +are internal-consistency checks, not authenticated provider provenance or +protection against an attacker rewriting the entire bundle. + +## Before the first live smoke + +Verify the account's selected model, inference credit eligibility and effective +billing terms; establish a conservative all-category one-request ceiling; obtain +specific approval for that model, synthetic data, credential use and spending; +securely provision the key; then review and launch one request. No such live +approval, price verification, credential setup or execution is supplied by these +fixtures. The larger eight-probe calibration and fresh held-out comparison remain +separate future work. Hosted client latency cannot be substituted for a local +Qwen hardware timing profile. + +The smoke records the overall startup/deadline/parent-completion window, not +individual token or per-request latency traces. It supplies no usable production +latency profile or provider cold-start measurement. diff --git a/docs/reliability-review-guide.md b/docs/reliability-review-guide.md index 33e0c52..5f79fa6 100644 --- a/docs/reliability-review-guide.md +++ b/docs/reliability-review-guide.md @@ -98,4 +98,3 @@ All new semantic integration, replay, policy activation and ablation evidence he - `7c7ca36` docs: remove unrelated integration references - `6278767` Separate prepared adapter operations and preserve existing verification logs - `c7501c4` Document warning cleanup and log preservation verification - diff --git a/scripts/incident_adaption_backend.py b/scripts/incident_adaption_backend.py new file mode 100644 index 0000000..04e881a --- /dev/null +++ b/scripts/incident_adaption_backend.py @@ -0,0 +1,163 @@ +"""Pure offline Adaption contract. No HTTP, credentials, pricing, or live path. + +Schema reference: https://docs.adaptionlabs.ai/api/resources/responses/methods/create +Reviewed 2026-11-14. Provider-reported counts are not verified local tokens, a bill, +a cancellation guarantee, or sufficient authority to release a monetary hold. +""" +import copy +import hashlib + +from incident_routing import (FREEZE_SHA256, canonical, load_workload, prepare, + validate_output, require, text as require_text, _loads) + +VERSION='incident-adaption/1' +METHOD='POST' +PATH='/api/v1/responses' +OUTPUT_CAP=96 +MAX_RECEIPT_BYTES=1_048_576 +STATUSES={'completed','failed','in_progress','cancelled','queued','incomplete'} + + +def _model(value): + require_text(value,'explicit hosted model') + require(value==value.strip() and not value.startswith('stack:') and len(value)<=256,'explicit hosted model ID required, not a stack') + require(not any(ord(c)<33 for c in value),'invalid hosted model ID') + + +def _case(case_id): + require(type(case_id) is str,'case ID required') + workload=load_workload() + matches=[c for c in workload['cases']['cases'] if c['case_id']==case_id and c['split']=='calibration'] + require(len(matches)==1,'only frozen calibration cases are supported') + return workload,matches[0] + + +def render_case(case_id, model, states=None): + """Frozen allowlisted evidence renderer; no labels enter the hosted body. + + states follows incident_routing.prepare's complete calibration-set schema. + Its eligibility is a fixture/replay attestation, not authenticated Store state. + """ + _model(model);workload,case=_case(case_id) + prepared=prepare(workload,'calibration',states) + record=next(r for r in prepared['requests'] if r['case_id']==case_id) + require(record['request'] is not None,'evidence_unavailable') + body={'model':model,'input':_loads(record['request'])['messages'], + 'max_output_tokens':OUTPUT_CAP,'temperature':0,'stream':False,'background':False, + 'tools':[],'tool_choice':'none','agentic_memory':False,'store':False} + raw=canonical(body).encode('utf-8') + return {'version':VERSION,'method':METHOD,'path':PATH,'case_id':case_id,'model':model, + 'request_bytes':raw,'request_sha256':hashlib.sha256(raw).hexdigest(), + 'freeze_sha256':FREEZE_SHA256,'evidence_sha256':record['evidence_sha256'], + 'eligibility_authority':prepared['eligibility_authority'],'live_admissible':False} + + +def _count(value):return type(value) is int and 0<=value<2**63 + + +def _usage(value): + result={'status':'missing','input_tokens':None,'output_tokens':None,'total_tokens':None, + 'details':{},'raw':copy.deepcopy(value),'issues':[]} + if value is None:return result + if type(value) is not dict: + result.update(status='invalid',issues=['usage must be an object']);return result + present=[] + for key in ('input_tokens','output_tokens','total_tokens'): + if key not in value:continue + present.append(key) + if _count(value[key]):result[key]=value[key] + else:result['issues'].append(f'{key}: nonnegative integer required') + for key,parent,subtotal in (('input_tokens_details','input_tokens','cached_tokens'),('output_tokens_details','output_tokens','reasoning_tokens')): + if key not in value:continue + result['details'][key]=copy.deepcopy(value[key]) + detail=value[key] + if type(detail) is not dict:result['issues'].append(f'{key}: object required');continue + if subtotal in detail: + count=detail[subtotal] + if not _count(count):result['issues'].append(f'{subtotal}: nonnegative integer required') + elif result[parent] is not None and count>result[parent]:result['issues'].append(f'{subtotal}: exceeds parent count') + if all(result[key] is not None for key in ('input_tokens','output_tokens','total_tokens')): + if result['input_tokens']+result['output_tokens']!=result['total_tokens']: + result['issues'].append('usage total disagrees with input plus output') + if result['output_tokens'] is not None and result['output_tokens']>OUTPUT_CAP: + result['issues'].append('reported output exceeds requested cap') + result['status']='invalid' if result['issues'] else 'complete' if len(present)==3 else 'partial' + return result + + +def parse_response(http_status, raw, *, expected_model, case_id): + """Classify a previously persisted receipt without semantic gold or pricing. + + Usage remains separately available for refusals/errors. A complete usage + object alone does not establish response attribution or authorize settlement. + Missing and invalid counts are never repaired, inferred, or replaced by zero. + """ + _model(expected_model);_,case=_case(case_id) + result={'version':VERSION,'disposition':'malformed_response','protocol_valid':False, + 'response_bound':False,'text':None,'validated_output':None,'response_id':None, + 'response_status':None,'usage':_usage(None),'issues':[]} + try: + require(type(http_status) is int and 100<=http_status<=599,'invalid HTTP status') + require(type(raw) is bytes and len(raw)<=MAX_RECEIPT_BYTES,'bounded raw bytes required') + value=_loads(raw) + require(type(value) is dict,'response must be an object') + result['usage']=_usage(value.get('usage')) + if not 200<=http_status<300: + result.update(disposition='http_error',issues=['HTTP response was not successful']);return result + require(type(value.get('id')) is str and bool(value['id']),'response ID required') + value['id'].encode('utf-8');result['response_id']=value['id'] + require(value.get('object')=='response','response object type') + require(type(value.get('model')) is str and value['model']==expected_model,'response model mismatch') + require(type(value.get('status')) is str and value['status'] in STATUSES,'response status') + result['response_status']=value['status'];result['response_bound']=True + if value.get('previous_response_id') is not None: + result['issues'].append('unexpected prior response linkage');result['response_bound']=False + if 'store' in value and value['store'] is not False: + result['issues'].append('unexpected response retention');result['response_bound']=False + if value.get('error') is not None: + error=value['error'];require(type(error) is dict and type(error.get('code')) is str and type(error.get('message')) is str,'response error shape') + result['issues'].append('provider reported an error') + if value.get('incomplete_details') is not None: + detail=value['incomplete_details'];require(type(detail) is dict and (detail.get('reason') is None or type(detail['reason']) is str),'incomplete detail shape') + result['issues'].append('provider reported incomplete output') + require(type(value.get('output')) is list,'output items required') + require(type(value.get('output_text')) is str,'aggregate output text required') + value['output_text'].encode('utf-8') + parts=[];refused=False;unexpected=False;unfinished=False + for item in value['output']: + require(type(item) is dict and type(item.get('type')) is str,'output item shape') + kind=item['type'] + if item.get('status') not in (None,'completed'):unfinished=True + if kind=='reasoning':continue + if kind!='message':unexpected=True;continue + require(item.get('role')=='assistant' and type(item.get('content')) is list,'assistant message required') + if item.get('status') not in (None,'completed'):unfinished=True + for part in item['content']: + require(type(part) is dict and type(part.get('type')) is str,'content part shape') + if part['type']=='refusal': + require(type(part.get('refusal')) is str,'refusal text required');part['refusal'].encode('utf-8');refused=True + elif part['type']=='output_text': + require(part.get('refusal') is None,'ambiguous text/refusal content') + require(type(part.get('text')) is str,'text content required');part['text'].encode('utf-8');parts.append(part['text']) + else:unexpected=True + answer=''.join(parts) + require(answer==value['output_text'],'aggregate output text mismatch') + result['text']=answer + if value['status']!='completed': + result['disposition']={'in_progress':'pending','queued':'pending'}.get(value['status'],value['status']) + elif result['issues'] or unfinished: + if unfinished:result['issues'].append('message item is not completed') + result['disposition']='incomplete' if unfinished or value.get('incomplete_details') is not None else 'failed' + elif unexpected: + result.update(disposition='unexpected_output',issues=['tool or unknown output despite tools disabled']) + elif refused:result['disposition']='refusal' + elif result['usage']['output_tokens'] is not None and result['usage']['output_tokens']>OUTPUT_CAP: + result.update(disposition='incomplete',issues=['reported output exceeds requested cap']) + else: + validated=validate_output(answer,case) + result.update(disposition='valid_output' if validated is not None else 'invalid_output', + protocol_valid=validated is not None,validated_output=validated) + return result + except (ValueError,TypeError,KeyError,UnicodeError,OverflowError,RecursionError) as error: + result.update(disposition='malformed_response',protocol_valid=False,response_bound=False) + result['issues'].append(str(error));return result diff --git a/scripts/incident_adaption_transport.py b/scripts/incident_adaption_transport.py new file mode 100644 index 0000000..8efe424 --- /dev/null +++ b/scripts/incident_adaption_transport.py @@ -0,0 +1,125 @@ +"""Single fixed-origin HTTPS request. No retries, redirects, SDK or tokenization. + +The network function is only called by an explicitly authorized runner child. +Timeouts bound local waits, not remote computation or billing. Tests inject a +connection object into the private transport core and never open a socket. +""" +import http.client +import ssl +import time +from dataclasses import dataclass +from incident_routing import require + +HOST = 'api.prod.adaptionlabs.ai' +PATH = '/api/v1/responses' +MAX_BODY = 1_048_576 +MAX_REQUEST = 65_536 +SAFE_HEADERS = {'content-type', 'request-id', 'x-request-id', 'retry-after'} + +@dataclass(frozen=True) +class TransportResult: + status: int | None + body: bytes + complete: bool + error: str | None + + +def _remaining(deadline, clock): + value = deadline - clock() + require(0 < value <= 1200, 'transport deadline exhausted or invalid') + return min(value, 10.0) + + +def _exchange(request, key, *, deadline, persist, connection_factory, clock=time.monotonic): + """Private injection seam: persist metadata/chunks before continuing reads. + + The caller supplies only frozen bytes and an in-memory credential. Never + write the key or exception messages to artifacts. A failure after entry is + conservatively delivery-uncertain, including a connection failure. + """ + require(type(request) is bytes and 0 < len(request) <= MAX_REQUEST, 'request size') + require(type(key) is str and 0 < len(key) <= 4096 and key.isascii() + and all(33 <= ord(c) < 127 for c in key), 'invalid credential') + connection = None + status = None + body = bytearray() + try: + connection = connection_factory(HOST, timeout=_remaining(deadline, clock), + context=ssl.create_default_context()) + # No connection reuse, redirect following or retry policy. + connection.request('POST', PATH, body=request, + headers={'Authorization': 'Bearer ' + key, + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Accept-Encoding': 'identity', + 'Connection': 'close'}) + if connection.sock is not None: + connection.sock.settimeout(_remaining(deadline, clock)) + response = connection.getresponse() + status = response.status + require(type(status) is int and 100 <= status <= 599, 'HTTP status') + headers = [] + expected_length = None + all_headers = response.getheaders() + require(len(all_headers) <= 100, 'header count ceiling') + lengths = [v for k, v in all_headers if k.lower() == 'content-length'] + require(len(lengths) <= 1, 'ambiguous content length') + if lengths: + require(lengths[0].isascii() and lengths[0].isdigit(), 'content length') + expected_length = int(lengths[0]) + encodings = [v.lower() for k, v in all_headers if k.lower() == 'content-encoding'] + require(all(v == 'identity' for v in encodings), 'encoded response unsupported') + for name, value in all_headers: + if name.lower() in SAFE_HEADERS: + headers.append([name.lower(), value[:4096]]) + persist('headers', {'status': status, 'headers': headers}) + # A 3xx is a final response, never a new destination/request. + while True: + remaining = _remaining(deadline, clock) + if connection.sock is not None: + connection.sock.settimeout(remaining) + # read1 returns already-received bytes without waiting for an entire + # requested chunk; the outer process watchdog also bounds drip feeds. + chunk = response.read1(min(16384, MAX_BODY + 1 - len(body))) + require(type(chunk) is bytes, 'HTTP body bytes') + if not chunk: + require(expected_length is None or len(body) == expected_length, 'truncated HTTP body') + persist('complete', {'status': status}) + return TransportResult(status, bytes(body), True, None) + allowed = chunk[:MAX_BODY - len(body)] + if allowed: + persist('chunk', allowed) + body.extend(allowed) + if len(chunk) > len(allowed): + raise ValueError('response byte ceiling') + except Exception as error: + # Preserve only the exception class. Network errors may embed URLs, + # headers or credentials in their message. + kind = type(error).__name__ + # http.client can carry bytes received before a premature EOF on the + # exception itself; preserve that prefix without treating it as complete. + partial = error.partial if isinstance(error, http.client.IncompleteRead) else None + if type(partial) is bytes and partial and len(body) < MAX_BODY: + prefix = partial[:MAX_BODY - len(body)] + try: + persist('chunk', prefix) + body.extend(prefix) + except Exception: + pass + try: + persist('error', {'kind': kind}) + except Exception: + pass + return TransportResult(status, bytes(body), False, kind) + finally: + if connection is not None: + try: + connection.close() + except Exception: + pass + + +def send_https(request, key, *, deadline, persist): + """Actual network code path; authorization belongs to the trusted runner.""" + return _exchange(request, key, deadline=deadline, persist=persist, + connection_factory=http.client.HTTPSConnection) diff --git a/scripts/incident_hosted_controller.py b/scripts/incident_hosted_controller.py new file mode 100644 index 0000000..0c46511 --- /dev/null +++ b/scripts/incident_hosted_controller.py @@ -0,0 +1,370 @@ +"""Offline hosted accounting experiment. No HTTP client, credentials or live CLI.""" +import argparse +import base64 +from contextlib import closing +from dataclasses import dataclass,asdict +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import threading +from incident_routing import canonical,_loads,require,FREEZE_SHA256 +from incident_adaption_backend import render_case,parse_response + +LIMIT=2**63-1 +HOLD_STATES=('reserved','claimed','received','uncertain') +SAFE_HEADERS={'content-type','request-id','x-request-id','retry-after'} + + +def integer(value,name,minimum=0): + require(type(value) is int and minimum<=value<=LIMIT,f'{name}: integer bound') + return value + +@dataclass(frozen=True) +class OfflineApproval: + """Explicit fake-only caller marker, not user permission or a capability.""" + kind:str='offline_fixture' + +@dataclass(frozen=True) +class OfflinePriceBound: + maximum:int + input_rate:int + output_rate:int + fixed_fee:int + maximum_input:int + maximum_output:int=96 + def validate(self): + for key,value in asdict(self).items():integer(value,key) + require(self.maximum>0 and self.maximum_input>0 and self.maximum_output==96,'fixture quote bounds') + worst=integer(self.maximum_input*self.input_rate+self.maximum_output*self.output_rate+self.fixed_fee,'worst-case synthetic cost') + require(self.maximum>=worst,'hold does not cover worst-case synthetic quote') + def estimate(self,usage): + result=usage['input_tokens']*self.input_rate+usage['output_tokens']*self.output_rate+self.fixed_fee + return integer(result,'estimated synthetic cost') + +@dataclass(frozen=True) +class HttpReceipt: + status:int + headers:tuple + body:bytes + +class FixtureTransport: + """Fixed canned outcome, no redirects/retries/network implementation.""" + def __init__(self,outcome): + require(type(outcome) is HttpReceipt or isinstance(outcome,Exception),'fixture outcome required') + self.outcome=outcome;self.calls=0;self.requests=[];self.lock=threading.Lock() + def send(self,request): + with self.lock:self.calls+=1;self.requests.append(request) + if isinstance(self.outcome,Exception):raise self.outcome + return self.outcome + + +def _approval(value): + if type(value) is not OfflineApproval or value.kind!='offline_fixture': + raise PermissionError('only explicit offline fixture invocation is implemented; live inadmissible') + + +def _write(path,value): + data=canonical(value).encode()+b'\n';temp=path.with_name(path.name+'.tmp') + with temp.open('xb') as f:f.write(data);f.flush();os.fsync(f.fileno()) + if path.exists():raise FileExistsError(path) + os.rename(temp,path) + fd=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + try:os.fsync(fd) + finally:os.close(fd) + + +def _connect(path,readonly=False): + if readonly: + header=path.read_bytes()[:20] + require(header[18:20]==b'\x01\x01' and not any(Path(str(path)+s).exists() for s in ('-wal','-shm')),'only rollback-journal snapshots supported') + db=sqlite3.connect(path.as_uri()+'?mode=ro',uri=True,timeout=5) + db.execute('PRAGMA query_only=ON') + else: + db=sqlite3.connect(path,timeout=10) + db.execute('PRAGMA journal_mode=DELETE');db.execute('PRAGMA synchronous=FULL') + db.row_factory=sqlite3.Row + return db + + +def _assess(parsed,quote): + quote.validate();usage=parsed['usage'];estimate=None;reason=None + terminal=parsed['response_status'] in ('completed','failed','cancelled','incomplete') + if parsed['response_bound'] and terminal and usage['status']=='complete': + if usage['input_tokens']>quote.maximum_input or usage['output_tokens']>quote.maximum_output:reason='usage_exceeds_synthetic_bound' + else: + try:estimate=quote.estimate(usage) + except ValueError:reason='estimated_cost_overflow' + if estimate is not None and estimate>quote.maximum:estimate=None;reason='estimated_cost_exceeds_hold' + else:reason='usage_or_attribution_unknown' + return estimate,reason + + +class HostedLedger: + def __init__(self,root): + self.root=Path(root).resolve();self.path=self.root/'ledger.sqlite' + require(self.path.is_file(),'existing ledger required') + @classmethod + def create(cls,root,*,budget,clock_domain,start,deadline,max_claims=4): + integer(budget,'budget',1);integer(start,'start');integer(deadline,'deadline');integer(max_claims,'max_claims',1) + require(max_claims<=4,'at most four initial calibration calls') + require(deadline>start and type(clock_domain) is str and clock_domain.strip() and clock_domain not in ('unknown','none'),'fixture clock window') + root=Path(root).resolve();root.mkdir(mode=0o700) + db=_connect(root/'ledger.sqlite') + db.executescript(''' + CREATE TABLE meta(id INTEGER PRIMARY KEY CHECK(id=1),budget INTEGER NOT NULL,domain TEXT NOT NULL,start INTEGER NOT NULL,deadline INTEGER NOT NULL,floor INTEGER NOT NULL,max_claims INTEGER NOT NULL); + CREATE TABLE operations(id INTEGER PRIMARY KEY,key TEXT UNIQUE NOT NULL,case_id TEXT NOT NULL,model TEXT NOT NULL,request BLOB NOT NULL,sha TEXT NOT NULL,provenance TEXT NOT NULL,quote TEXT NOT NULL,state TEXT NOT NULL,estimated INTEGER,parsed TEXT,issue TEXT); + CREATE TABLE claims(operation INTEGER PRIMARY KEY,tick INTEGER NOT NULL); + CREATE TABLE receipts(operation INTEGER PRIMARY KEY,payload TEXT NOT NULL); + CREATE TABLE events(id INTEGER PRIMARY KEY,operation INTEGER,event TEXT NOT NULL,payload TEXT NOT NULL); + CREATE TRIGGER immutable_meta BEFORE UPDATE OF budget,domain,start,deadline,max_claims ON meta BEGIN SELECT RAISE(ABORT,'immutable window');END; + CREATE TRIGGER immutable_events_update BEFORE UPDATE ON events BEGIN SELECT RAISE(ABORT,'immutable event');END; + CREATE TRIGGER immutable_events_delete BEFORE DELETE ON events BEGIN SELECT RAISE(ABORT,'immutable event');END; + CREATE TRIGGER immutable_binding BEFORE UPDATE OF key,case_id,model,request,sha,provenance,quote ON operations BEGIN SELECT RAISE(ABORT,'immutable request binding');END; + CREATE TRIGGER immutable_claim_update BEFORE UPDATE ON claims BEGIN SELECT RAISE(ABORT,'immutable claim');END; + CREATE TRIGGER immutable_claim_delete BEFORE DELETE ON claims BEGIN SELECT RAISE(ABORT,'immutable claim');END; + CREATE TRIGGER immutable_receipt BEFORE UPDATE ON receipts BEGIN SELECT RAISE(ABORT,'immutable receipt');END; + ''') + with db:db.execute('INSERT INTO meta VALUES(1,?,?,?,?,?,?)',(budget,clock_domain,start,deadline,start,max_claims)) + db.close() + _write(root/'manifest.json',{'version':'incident-hosted-fixture/1','mode':'offline_fixture','live_admissible':False,'new_model_calls':0,'monetary_unit':'synthetic_micro_usd','budget':budget,'max_claims':max_claims,'freeze_sha256':FREEZE_SHA256,'clock_domain':clock_domain,'start':start,'deadline':deadline,'source_sha256':{name:hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() for name in ('incident_hosted_controller.py','incident_adaption_backend.py','incident_routing.py')}}) + return cls(root) + def _event(self,db,op,name,payload):db.execute('INSERT INTO events(operation,event,payload) VALUES(?,?,?)',(op,name,canonical(payload))) + def create_operation(self,key,case_id,model,quote,states=None): + require(type(key) is str and key.strip(),'operation key');require(type(quote) is OfflinePriceBound,'synthetic quote required');quote.validate() + rendered=render_case(case_id,model,states=states) + request=rendered['request_bytes'];provenance={k:v for k,v in rendered.items() if k!='request_bytes'} + binding=(case_id,model,request,rendered['request_sha256'],canonical(provenance),canonical(asdict(quote))) + with closing(_connect(self.path)) as db, db: + db.execute('BEGIN IMMEDIATE') + prior=db.execute('SELECT * FROM operations WHERE key=?',(key,)).fetchone() + if prior: + require(tuple(prior[k] for k in ('case_id','model','request','sha','provenance','quote'))==binding,'key bindings differ') + return prior['id'] + op=db.execute("INSERT INTO operations(key,case_id,model,request,sha,provenance,quote,state) VALUES(?,?,?,?,?,?,?,'planned')",(key,)+binding).lastrowid + self._event(db,op,'planned',{'request_sha256':binding[3]});return op + @staticmethod + def _totals(db): + spent=held=0 + for row in db.execute('SELECT state,estimated,quote FROM operations'): + if row['estimated'] is not None:spent+=integer(row['estimated'],'estimate') + if row['state'] in HOLD_STATES:held+=_loads(row['quote'])['maximum'] + return integer(spent,'aggregate estimate'),integer(held,'aggregate hold') + def reserve(self,op): + with closing(_connect(self.path)) as db, db: + db.execute('BEGIN IMMEDIATE');r=db.execute('SELECT * FROM operations WHERE id=?',(op,)).fetchone();require(r is not None,'operation missing') + if r['state']!='planned':return r['state']=='reserved' + spent,held=self._totals(db);maximum=_loads(r['quote'])['maximum'];budget=db.execute('SELECT budget FROM meta').fetchone()[0] + accepted=spent+held+maximum<=budget;state='reserved' if accepted else 'denied' + db.execute('UPDATE operations SET state=? WHERE id=?',(state,op));self._event(db,op,state,{'bound':maximum});return accepted + def claim(self,op,*,approval=None,clock_domain=None,clock=None): + _approval(approval);require(callable(clock),'trusted fixture clock required') + with closing(_connect(self.path)) as db, db: + db.execute('BEGIN IMMEDIATE');r=db.execute('SELECT * FROM operations WHERE id=?',(op,)).fetchone();require(r is not None,'operation missing') + if r['state']!='reserved':return None + m=db.execute('SELECT * FROM meta').fetchone() + if clock_domain!=m['domain']:self._event(db,op,'refused',{'reason':'clock_domain'});return None + if db.execute("SELECT 1 FROM operations WHERE id<>? AND state IN ('claimed','received','uncertain')",(op,)).fetchone(): + self._event(db,op,'refused',{'reason':'prior_unresolved'});return None + if db.execute('SELECT count(*) FROM claims').fetchone()[0]>=m['max_claims']: + self._event(db,op,'refused',{'reason':'call_limit'});return None + spent,held=self._totals(db);require(spent+held<=m['budget'],'budget invariant') + require(hashlib.sha256(r['request']).hexdigest()==r['sha'],'request binding invalid') + now=integer(clock(),'clock') + if now=m['deadline']: + self._event(db,op,'refused',{'reason':'clock_window','tick':now});return None + db.execute('UPDATE meta SET floor=?',(now,));db.execute('INSERT INTO claims VALUES(?,?)',(op,now)) + db.execute("UPDATE operations SET state='claimed' WHERE id=?",(op,));self._event(db,op,'claimed',{'tick':now}) + return bytes(r['request']) + def cancel(self,op): + with closing(_connect(self.path)) as db, db: + db.execute('BEGIN IMMEDIATE');r=db.execute('SELECT state FROM operations WHERE id=?',(op,)).fetchone();require(r is not None,'operation missing') + if r[0]=='cancelled':return True + if r[0] not in ('planned','reserved'):return False + db.execute("UPDATE operations SET state='cancelled' WHERE id=?",(op,));self._event(db,op,'cancelled',{});return True + def record_receipt(self,op,receipt,*,observed,minimum_tick=None): + require(type(receipt) is HttpReceipt and type(receipt.body) is bytes,'HTTP fixture receipt required') + if observed is not None:integer(observed,'receipt clock') + require(len(receipt.body)<=1024*1024,'receipt exceeds storage bound') + headers=[] + for key,value in receipt.headers: + require(type(key) is str and type(value) is str,'header strings required') + if key.lower() in SAFE_HEADERS:headers.append([key.lower(),value[:4096]]) + db=_connect(self.path);durable=False;issues=[] + try: + db.execute('BEGIN IMMEDIATE') + row=db.execute('SELECT c.tick,o.sha,m.floor,m.deadline FROM claims c JOIN operations o ON o.id=c.operation CROSS JOIN meta m WHERE operation=?',(op,)).fetchone() + require(row is not None,'receipt without prior claim') + send_tick=row['tick'] if minimum_tick is None else integer(minimum_tick,'send clock') + require(row['tick']<=send_tick=payload['send_tick']),'receipt observation window') + if row['state']=='settled':return + db.execute('UPDATE meta SET floor=max(floor,?)',(payload['send_tick'],)) + if payload['observed'] is not None:db.execute('UPDATE meta SET floor=max(floor,?)',(payload['observed'],)) + parsed=parse_response(payload['status'],base64.b64decode(payload['body_base64'],validate=True),expected_model=row['model'],case_id=row['case_id']) + estimate,reason=_assess(parsed,OfflinePriceBound(**_loads(row['quote']))) + state='settled' if estimate is not None else 'uncertain' + db.execute('UPDATE operations SET state=?,estimated=?,parsed=? WHERE id=?',(state,estimate,canonical(parsed),op)) + if reason:db.execute('UPDATE operations SET issue=? WHERE id=?',(canonical([reason]),op)) + self._event(db,op,'reconciled',{'estimated':estimate,'reason':reason}) + def dispatch(self,op,transport,*,approval=None,clock_domain=None,clock=None): + _approval(approval) + if type(transport) is not FixtureTransport:raise PermissionError('only the fixed fake transport is implemented') + self.reserve(op);request=self.claim(op,approval=approval,clock_domain=clock_domain,clock=clock) + if request is None:return self.audit() + try: + with closing(_connect(self.path)) as db, db: + m=db.execute('SELECT * FROM meta').fetchone() + final=integer(clock(),'final clock') + require(clock_domain==m['domain'] and m['floor']<=final=m['start'] and 1<=m['max_claims']<=4,'window invalid') + spent=held=0;claims=0 + allowed={'planned','reserved','claimed','received','settled','uncertain','denied','cancelled'} + for r in db.execute('SELECT * FROM operations ORDER BY id'): + require(r['state'] in allowed,'unknown operation state') + quote=OfflinePriceBound(**_loads(r['quote']));quote.validate() + rendered=render_case(r['case_id'],r['model']) + require(r['request']==rendered['request_bytes'] and r['sha']==rendered['request_sha256'] and _loads(r['provenance'])=={k:v for k,v in rendered.items() if k!='request_bytes'},'frozen request/provenance mismatch') + ev=[(e['event'],_loads(e['payload'])) for e in db.execute('SELECT event,payload FROM events WHERE operation=? ORDER BY id',(r['id'],))] + require(ev and ev[0]==('planned',{'request_sha256':r['sha']}),'planned event missing') + require(all(name in {'planned','reserved','denied','claimed','received','reconciled','cancelled','refused','failed','send_boundary'} and type(value) is dict for name,value in ev),'event shape') + names=[name for name,_ in ev] + require(names.count('planned')==1 and names.count('reserved')<=1 and names.count('denied')<=1 and not ('reserved' in names and 'denied' in names),'allocation event chain') + if r['state']=='planned':require(len(ev)==1,'planned state after allocation') + if 'claimed' in names:require(names.index('reserved')=payload['send_tick']),'receipt clock') + if record and observed is not None:require(observed<=m['floor'],'receipt floor regression') + require(type(payload['headers']) is list and all(type(pair) is list and len(pair)==2 and pair[0] in SAFE_HEADERS and type(pair[1]) is str for pair in payload['headers']),'safe receipt headers') + body=base64.b64decode(payload['body_base64'],validate=True) + parsed=parse_response(payload['status'],body,expected_model=r['model'],case_id=r['case_id']) + recomputed,_=_assess(parsed,quote) + if r['parsed'] is not None:require(parsed is not None and _loads(r['parsed'])==parsed,'stored parse differs from raw') + if r['state']=='settled': + require(r['parsed'] is not None and recomputed is not None and r['estimated']==recomputed,'settled estimate lacks raw evidence') + integer(r['estimated'],'estimate');spent+=r['estimated'] + else:require(r['estimated'] is None,'unsettled estimated spend') + if r['state'] in HOLD_STATES:held+=quote.maximum + if r['state']=='received':require(record is not None,'received state missing receipt') + late=payload['observed']>=m['deadline'] if payload and payload['observed'] is not None else None + result['operations'].append({'id':r['id'],'state':r['state'],'request_sha256':r['sha'],'estimated':r['estimated'],'parsed':parsed,'recomputed_estimate':recomputed,'issue':_loads(r['issue']) if r['issue'] else None,'late':late,'raw_receipt_available':payload is not None}) + require(claims==db.execute('SELECT count(*) FROM claims').fetchone()[0]<=m['max_claims'],'call-count bound') + require(spent+held<=m['budget'],'accounting invariant') + result.update(estimated_spent=spent,held_bound=held,claimed_count=claims,audit_complete=True,max_claims=m['max_claims']) + except (ValueError,TypeError,KeyError,OSError,sqlite3.Error,IndexError,OverflowError,RecursionError) as error: + result.update(audit_complete=False,estimated_spent=None,held_bound=None,audit_error=str(error),operations=[]) + return result + + +def main(argv=None): + p=argparse.ArgumentParser(description=__doc__);sub=p.add_subparsers(dest='command',required=True) + fixture=sub.add_parser('fixture');fixture.add_argument('--out',type=Path,required=True) + audit=sub.add_parser('audit');audit.add_argument('out',type=Path) + a=p.parse_args(argv) + if a.command=='audit':report=HostedLedger(a.out).audit() + else: + ledger=HostedLedger.create(a.out,budget=400,clock_domain='fixture-clock',start=0,deadline=1000) + op=ledger.create_operation('fixture-case','cal-01','fixture-hosted-model/1',OfflinePriceBound(400,2,3,10,20)) + body=canonical({'id':'fixture-response','object':'response','model':'fixture-hosted-model/1','status':'completed','output':[{'type':'message','role':'assistant','content':[{'type':'output_text','text':'{}'}]}],'output_text':'{}','usage':{'input_tokens':10,'output_tokens':5,'total_tokens':15}}).encode() + report=ledger.dispatch(op,FixtureTransport(HttpReceipt(200,(),body)),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + print(canonical(report));return 0 if report['audit_complete'] else 1 + +if __name__=='__main__': + try:raise SystemExit(main()) + except Exception as error:raise SystemExit(f'Offline hosted controller refused: {type(error).__name__}: {error}') diff --git a/scripts/incident_hosted_runner.py b/scripts/incident_hosted_runner.py new file mode 100644 index 0000000..4bcdde0 --- /dev/null +++ b/scripts/incident_hosted_runner.py @@ -0,0 +1,346 @@ +"""One-shot hosted runner. Only fixture/audit CLI; no live launch switch. + +Live execution is a library operation requiring a separately obtained, trusted +approval and billing-bound attestation. Neither artifact JSON nor an environment +variable grants that authority. No live operation is performed by the tests. +""" +import argparse +import base64 +from dataclasses import dataclass, asdict +import hashlib +import json +import multiprocessing +import os +from pathlib import Path +import signal +import sqlite3 +import time + +from incident_adaption_backend import render_case, parse_response +from incident_adaption_transport import send_https, MAX_BODY +from incident_routing import canonical, _loads, require, FREEZE_SHA256 + +VERSION = 'incident-hosted-single-request/1' + +@dataclass(frozen=True) +class LiveApproval: + """Trusted caller's attestation, not a cryptographic authorization token. + + A human/operator must approve this exact request and the monetary ceiling + outside this library. Merely constructing this object is not that approval. + """ + request_sha256: str + model: str + maximum_micro_usd: int + approval_reference: str + billing_bound_reference: str + credential_use_approved: bool = False + data_transmission_approved: bool = False + all_billable_categories_bounded: bool = False + currency: str = 'USD' + bound_basis: str = 'all_in_request_ceiling' + maximum_duration_ms: int = 60_000 + + +def _now(): + require(hasattr(time, 'CLOCK_BOOTTIME'), 'Linux boottime required') + return time.clock_gettime_ns(time.CLOCK_BOOTTIME) + + +def _sources(): + return {name: hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() + for name in ('incident_hosted_runner.py', 'incident_adaption_transport.py', + 'incident_adaption_backend.py', 'incident_routing.py', + 'incident_calibration_process.py')} + + +def _write(path, payload): + # Exclusive immutable artifact; parent and child never target the same file. + temporary = path.with_name(path.name + '.tmp') + with temporary.open('xb') as handle: + handle.write(canonical(payload).encode() + b'\n') + handle.flush() + os.fsync(handle.fileno()) + if path.exists(): + raise FileExistsError(path) + os.rename(temporary, path) + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _db(root): + db = sqlite3.connect(Path(root) / 'ledger.sqlite', timeout=1) + db.execute('PRAGMA journal_mode=DELETE') + db.execute('PRAGMA synchronous=FULL') + return db + + +def _validate_approval(approval, rendered, budget): + if type(approval) is not LiveApproval: + raise PermissionError('separate live approval and billing attestation required') + require(approval.request_sha256 == rendered['request_sha256'] + and approval.model == rendered['model'], 'approval request binding') + require(type(approval.maximum_micro_usd) is int + and 0 < approval.maximum_micro_usd <= budget, 'approved monetary bound') + require(approval.currency == 'USD' and approval.bound_basis == 'all_in_request_ceiling', + 'only explicit all-in USD request ceilings are supported') + require(type(approval.maximum_duration_ms) is int + and 100 <= approval.maximum_duration_ms <= 1_200_000, 'approved time bound') + require(all(type(v) is str and bool(v.strip()) and len(v) <= 512 for v in + (approval.approval_reference, approval.billing_bound_reference)), + 'approval and conservative billing-bound references required') + require(approval.credential_use_approved is True + and approval.data_transmission_approved is True + and approval.all_billable_categories_bounded is True, + 'credential/data/billing scope not approved') + + +def _persistor(root): + sequence = 0 + def persist(kind, value): + nonlocal sequence + payload = {'sequence': sequence, 'kind': kind, + 'value': base64.b64encode(value).decode() if kind == 'chunk' else value} + _write(root / f'transport-{sequence:05d}.json', payload) + sequence += 1 + return persist + + +def _child(root, request, mode, deadline_ns, parent_pid, fault): + # Forked child has no unrelated process group; Linux parent-death protection + # is armed before any potential network or credential operation. + from incident_calibration_process import arm_parent_death + arm_parent_death(parent_pid) + persist = _persistor(root) + if _now() >= deadline_ns: + persist('error', {'kind': 'DeadlineBeforeSend'}) + return + if mode == 'offline_process_fixture': + if fault == 'hang': + while True: + time.sleep(1) + if fault == 'crash': + os._exit(7) + value = _loads(request) + body = canonical({'id': 'fixture-response', 'object': 'response', + 'model': value['model'], 'status': 'completed', + 'output': [{'type': 'message', 'role': 'assistant', + 'content': [{'type': 'output_text', 'text': '{}'}]}], + 'output_text': '{}', + 'usage': {'input_tokens': 10, 'output_tokens': 5, + 'total_tokens': 15}}).encode() + persist('headers', {'status': 200, 'headers': []}) + persist('chunk', body if fault != 'partial' else body[:50]) + if fault == 'partial': + os._exit(8) + persist('complete', {'status': 200}) + return + # Not reached by offline tests or CLI. The parent validated exact approval + # before forking; reading a credential is deferred until this point. + try: + key = os.environ['ADAPTION_API_KEY'] + deadline = time.monotonic() + max(0, (deadline_ns - _now()) / 1e9) + send_https(request, key, deadline=deadline, persist=persist) + except Exception as error: + persist('error', {'kind': type(error).__name__}) + + +def _run(root, case_id, model, *, mode, budget_micro_usd, duration_ms, + approval=None, fault='none'): + start = _now() + require(type(duration_ms) is int and 100 <= duration_ms <= 1_200_000, 'bounded duration') + require(type(budget_micro_usd) is int and 0 < budget_micro_usd <= 2**63 - 1, 'budget') + require(mode in ('offline_process_fixture', 'live_https'), 'mode') + require(fault in ('none', 'hang', 'crash', 'partial'), 'fixture fault') + require(mode == 'offline_process_fixture' or fault == 'none', 'no live faults') + rendered = render_case(case_id, model) + if mode == 'live_https': + _validate_approval(approval, rendered, budget_micro_usd) + require(duration_ms <= approval.maximum_duration_ms, 'duration exceeds approval') + hold = approval.maximum_micro_usd + else: + require(approval is None, 'fixture cannot consume live authority') + hold = budget_micro_usd + deadline = start + duration_ms * 1_000_000 + root = Path(root).resolve() + root.mkdir(mode=0o700) + manifest = {'version': VERSION, 'mode': mode, 'request_sha256': rendered['request_sha256'], + 'case_id': case_id, 'model': model, 'freeze_sha256': FREEZE_SHA256, + 'sources': _sources(), 'start_ns': start, 'deadline_ns': deadline, + 'maximum_calls': 1, 'budget_micro_usd': budget_micro_usd, + 'reserved_micro_usd': hold, 'monetary_basis': 'approved_conservative_bound' if mode == 'live_https' else 'synthetic_fixture_bound', + 'approval': asdict(approval) if approval else None, + 'remote_cancellation_guaranteed': False, 'resume_allowed': False} + _write(root / 'manifest.json', manifest) + _write(root / 'request.json', {'body_base64': base64.b64encode(rendered['request_bytes']).decode()}) + db = _db(root) + db.executescript('CREATE TABLE claim(id INTEGER PRIMARY KEY CHECK(id=1),request_sha256 TEXT NOT NULL,tick INTEGER NOT NULL,hold INTEGER NOT NULL); CREATE TRIGGER immutable_claim BEFORE UPDATE ON claim BEGIN SELECT RAISE(ABORT,"immutable claim");END;') + claimed = False + process = None + reason = None + try: + # SQLite is the durable single-use claim. Reusing a directory is denied + # before this point; no restart path can automatically send again. + with db: + tick = _now() + require(tick < deadline, 'deadline before claim') + db.execute('INSERT INTO claim VALUES(1,?,?,?)', (rendered['request_sha256'], tick, hold)) + claimed = True + require(_now() < deadline, 'deadline after claim') + process = multiprocessing.get_context('fork').Process( + target=_child, args=(root, rendered['request_bytes'], mode, deadline, os.getpid(), fault)) + process.start() + while process.is_alive() and _now() < deadline: + process.join(timeout=min(.02, max(0, (deadline - _now()) / 1e9))) + if process.is_alive(): + reason = 'local_watchdog_expired_remote_outcome_unknown' + process.kill() + process.join(timeout=1) + require(not process.is_alive(), 'child termination not confirmed') + if reason is None and process.exitcode != 0: + reason = 'child_exit_without_complete_receipt' + except Exception as error: + reason = type(error).__name__ + finally: + # A caller may catch KeyboardInterrupt/SystemExit and keep this parent + # alive, so Linux parent-death protection alone is insufficient. Reap + # the local child on every exit while preserving interrupt propagation + # and the durable potentially-sent claim/hold. + try: + if process is not None and process.pid is not None: + if process.is_alive(): + process.kill() + process.join(timeout=1) + require(not process.is_alive(), 'child termination not confirmed') + finally: + db.close() + _write(root / 'parent.json', {'claimed': claimed, 'reason': reason, + 'finished_ns': _now(), + 'child_exitcode': process.exitcode if process is not None and process.pid else None}) + return audit_run(root) + + +def run_fixture(root, *, duration_ms=1000, fault='none'): + return _run(root, 'cal-01', 'fixture-hosted-model/1', mode='offline_process_fixture', + budget_micro_usd=400, duration_ms=duration_ms, fault=fault) + + +def run_live(root, case_id, model, *, approval=None, budget_micro_usd, + duration_ms=60_000): + """No CLI exposure. Caller must obtain exact external approval first. + + The asserted bound must cover every billable category, even if the client + dies. This code cannot verify provider prices or enforce provider billing. + """ + return _run(root, case_id, model, mode='live_https', approval=approval, + budget_micro_usd=budget_micro_usd, duration_ms=duration_ms) + + +def audit_run(root): + root = Path(root).resolve() + result = {'version': VERSION, 'audit_complete': False, 'provider_reported_usage': None, + 'verified_provider_cost': None, 'held_micro_usd': None, + 'remote_cancellation_guaranteed': False} + try: + m = _loads((root / 'manifest.json').read_bytes()) + require(m['version'] == VERSION and m['sources'] == _sources(), 'source identity') + require(m['mode'] in ('offline_process_fixture', 'live_https'), 'manifest mode') + rendered = render_case(m['case_id'], m['model']) + require(rendered['request_sha256'] == m['request_sha256'] and m['freeze_sha256'] == FREEZE_SHA256, 'request provenance') + raw = base64.b64decode(_loads((root / 'request.json').read_bytes())['body_base64'], validate=True) + require(raw == rendered['request_bytes'], 'frozen request mismatch') + require(type(m['start_ns']) is int and type(m['deadline_ns']) is int + and 100_000_000 <= m['deadline_ns'] - m['start_ns'] <= 1_200_000_000_000, 'deadline window') + require(type(m['maximum_calls']) is int and m['maximum_calls'] == 1 + and m['resume_allowed'] is False + and type(m['budget_micro_usd']) is int + and type(m['reserved_micro_usd']) is int + and 0 < m['reserved_micro_usd'] <= m['budget_micro_usd'] <= 2**63 - 1, 'bound manifest') + if m['mode'] == 'live_https': + _validate_approval(LiveApproval(**m['approval']), rendered, m['budget_micro_usd']) + require(m['deadline_ns'] - m['start_ns'] <= m['approval']['maximum_duration_ms'] * 1_000_000, 'duration exceeds approval') + require(m['reserved_micro_usd'] == m['approval']['maximum_micro_usd'] + and m['monetary_basis'] == 'approved_conservative_bound', 'live hold binding') + else: + require(m['approval'] is None and m['monetary_basis'] == 'synthetic_fixture_bound', 'fixture authority') + ledger = root / 'ledger.sqlite' + with ledger.open('rb') as handle: + header = handle.read(20) + require(header[18:20] == b'\x01\x01' + and not any(Path(str(ledger) + suffix).exists() + for suffix in ('-wal', '-shm', '-journal')), + 'only clean rollback-journal snapshots supported') + db = sqlite3.connect(ledger.as_uri() + '?mode=ro', uri=True) + db.execute('PRAGMA query_only=ON') + try: + claims = db.execute('SELECT request_sha256,tick,hold FROM claim').fetchall() + finally: + db.close() + require(len(claims) <= 1, 'duplicate claim') + if claims: + sha, tick, hold = claims[0] + require(sha == m['request_sha256'] and m['start_ns'] <= tick < m['deadline_ns'] + and hold == m['reserved_micro_usd'], 'claim binding') + body = bytearray() + status = None + complete = False + errors = [] + records = sorted(root.glob('transport-*.json')) + for index, path in enumerate(records): + require(path.name == f'transport-{index:05d}.json', 'transport sequence gap') + record = _loads(path.read_bytes()) + require(record['sequence'] == index and not complete, 'transport record order') + kind, value = record['kind'], record['value'] + if kind == 'headers': + require(index == 0 and type(value['status']) is int, 'headers order') + status = value['status'] + elif kind == 'chunk': + require(status is not None, 'body without headers') + body.extend(base64.b64decode(value, validate=True)) + require(len(body) <= MAX_BODY, 'body ceiling') + elif kind == 'complete': + require(status is not None and value['status'] == status, 'completion status') + complete = True + elif kind == 'error': + require(type(value) is dict and type(value['kind']) is str, 'error record') + errors.append(value['kind']) + else: + raise ValueError('unknown transport record') + require(not records or claims, 'transport without claim') + parsed = parse_response(status, bytes(body), expected_model=m['model'], case_id=m['case_id']) if status is not None else None + seal = _loads((root / 'parent.json').read_bytes()) if (root / 'parent.json').exists() else None + if seal: + require(seal['claimed'] == bool(claims) and type(seal['finished_ns']) is int + and seal['finished_ns'] >= m['start_ns'], 'parent seal') + result.update(audit_complete=True, mode=m['mode'], claimed_count=len(claims), + new_model_calls=0 if m['mode'] == 'offline_process_fixture' else None, + potential_model_calls=len(claims) if m['mode'] == 'live_https' else 0, + monetary_basis=m['monetary_basis'], held_micro_usd=m['reserved_micro_usd'] if claims else 0, + receipt_complete=complete, usable_protocol_output=bool(complete and not errors and parsed and parsed['protocol_valid']), + raw_body_bytes=len(body), parsed=parsed, + provider_reported_usage=parsed['usage'] if parsed else None, + transport_errors=errors, parent=seal, + billing_reconciliation='not_implemented_hold_retained' if claims else 'unclaimed', + state='receipt_recorded_hold_retained' if complete else 'delivery_uncertain' if claims else 'not_sent') + except Exception as error: + result.update(audit_error=type(error).__name__, held_micro_usd=None) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + fixture = sub.add_parser('fixture') + fixture.add_argument('--out', type=Path, required=True) + audit = sub.add_parser('audit') + audit.add_argument('out', type=Path) + args = parser.parse_args(argv) + report = run_fixture(args.out) if args.command == 'fixture' else audit_run(args.out) + print(canonical(report)) + return 0 if report['audit_complete'] else 1 + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/test_incident_adaption_backend.py b/scripts/test_incident_adaption_backend.py new file mode 100644 index 0000000..a861d7f --- /dev/null +++ b/scripts/test_incident_adaption_backend.py @@ -0,0 +1,100 @@ +import copy +import json +import unittest +from incident_adaption_backend import render_case, parse_response +from incident_routing import canonical, load_workload, prepare + +MODEL='fixture-hosted-model/1' +TEXT='{"decision":"abstain","owner":null,"reason":"undetermined_cause","evidence_ids":["e01"]}' + +def response(**changes): + value={'id':'resp-fixture-1','object':'response','status':'completed','model':MODEL, + 'output':[{'type':'message','role':'assistant','status':'completed','content':[{'type':'output_text','text':TEXT}]}], + 'output_text':TEXT,'usage':{'input_tokens':101,'output_tokens':12,'total_tokens':113}} + value.update(changes);return value + +def parse(value,status=200):return parse_response(status,canonical(value).encode(),expected_model=MODEL,case_id='cal-01') + +class AdaptionTests(unittest.TestCase): + def test_frozen_renderer_explicit_controls_and_no_labels(self): + r=render_case('cal-01',MODEL);body=json.loads(r['request_bytes']) + self.assertEqual(body['input'],json.loads(prepare(load_workload())['requests'][0]['request'])['messages']) + self.assertEqual(set(body),{'model','input','max_output_tokens','temperature','stream','background','tools','tool_choice','agentic_memory','store'}) + self.assertEqual(body['max_output_tokens'],96);self.assertEqual(body['temperature'],0) + for key in ('stream','background','agentic_memory','store'):self.assertIs(body[key],False) + self.assertEqual(body['tools'],[]);self.assertEqual(body['tool_choice'],'none') + self.assertNotIn(b'case_id',r['request_bytes']);self.assertNotIn(b'expected',r['request_bytes']) + self.assertEqual(r,render_case('cal-01',MODEL));self.assertEqual(r['path'],'/api/v1/responses') + def test_renderer_rejects_model_ambiguity_and_heldout(self): + for model in ('',' ','stack:test',None,True): + with self.subTest(model=model),self.assertRaises(ValueError):render_case('cal-01',model) + with self.assertRaises(ValueError):render_case('hold-01',MODEL) + def test_evidence_missing_stale_refused_no_silent_filter(self): + states=prepare(load_workload())['states'] + for state in ('missing','stale','refused'): + changed=copy.deepcopy(states);changed[0]['evidence'][0]['state']=state + with self.subTest(state=state),self.assertRaises(ValueError):render_case('cal-01',MODEL,changed) + def test_success_and_provider_usage_not_semantic_gold(self): + r=parse(response());self.assertTrue(r['protocol_valid']);self.assertTrue(r['response_bound']) + self.assertEqual(r['disposition'],'valid_output');self.assertEqual(r['usage']['status'],'complete') + self.assertEqual(r['usage']['total_tokens'],113) + def test_malformed_output_is_measured_but_not_valid(self): + r=parse(response(output_text='oops',output=[{'type':'message','role':'assistant','content':[{'type':'output_text','text':'oops'}]}])) + self.assertEqual(r['disposition'],'invalid_output');self.assertFalse(r['protocol_valid']) + self.assertEqual(r['usage']['status'],'complete') + def test_refusal_incomplete_failure_pending_and_cancelled(self): + r=parse(response(output_text='',output=[{'type':'message','role':'assistant','content':[{'type':'refusal','refusal':'no'}]}])) + self.assertEqual(r['disposition'],'refusal') + for status in ('incomplete','failed','in_progress','queued','cancelled'): + r=parse(response(status=status,incomplete_details={'reason':'max_output_tokens'} if status=='incomplete' else None)) + self.assertFalse(r['protocol_valid']);self.assertNotEqual(r['disposition'],'valid_output') + self.assertEqual(r['usage']['total_tokens'],113) + def test_contradictory_text_tools_and_wrong_model_refused(self): + for value in (response(output_text='different'),response(model='other'),response(output=[{'type':'function_call','name':'x'}])): + r=parse(value);self.assertFalse(r['protocol_valid']);self.assertTrue(r['issues']) + self.assertFalse(parse(response(model='other'))['response_bound']) + def test_partial_and_missing_usage_not_zero(self): + for usage,status in ((None,'missing'),({},'partial'),({'input_tokens':10},'partial')): + r=parse(response(usage=usage));self.assertEqual(r['usage']['status'],status) + self.assertIsNone(r['usage']['output_tokens']);self.assertIsNone(r['usage']['total_tokens']) + def test_bad_usage_is_invalid_and_retained(self): + for usage in ({'input_tokens':True,'output_tokens':2,'total_tokens':3}, + {'input_tokens':-1,'output_tokens':2,'total_tokens':1}, + {'input_tokens':1.0,'output_tokens':2,'total_tokens':3}, + {'input_tokens':2**63,'output_tokens':2,'total_tokens':2**63+2}, + {'input_tokens':1,'output_tokens':2,'total_tokens':4}): + r=parse(response(usage=usage));self.assertEqual(r['usage']['status'],'invalid');self.assertEqual(r['usage']['raw'],usage) + def test_details_retained_not_added_to_totals(self): + usage={'input_tokens':101,'output_tokens':12,'total_tokens':113,'input_tokens_details':{'cached_tokens':50},'output_tokens_details':{'reasoning_tokens':9}} + r=parse(response(usage=usage));self.assertEqual(r['usage']['total_tokens'],113) + self.assertEqual(r['usage']['details']['output_tokens_details'],{'reasoning_tokens':9}) + for bad in (True,-1,13,'9'): + usage['output_tokens_details']['reasoning_tokens']=bad + self.assertEqual(parse(response(usage=usage))['usage']['status'],'invalid') + def test_duplicate_keys_bad_utf8_deep_json_and_http_error(self): + for raw in (b'{"id":1,"id":2}',b'\xff',b'['*100+b'0'+b']'*100,b'not json'): + r=parse_response(200,raw,expected_model=MODEL,case_id='cal-01') + self.assertEqual(r['disposition'],'malformed_response');self.assertFalse(r['protocol_valid']) + r=parse(response(),500);self.assertEqual(r['disposition'],'http_error');self.assertFalse(r['response_bound']) + def test_response_constraints_and_cap_violation(self): + for change in ({'store':True},{'previous_response_id':'old'},{'error':{'code':'x','message':'bad'}},{'usage':{'input_tokens':1,'output_tokens':97,'total_tokens':98}}): + r=parse(response(**change));self.assertFalse(r['protocol_valid']);self.assertTrue(r['issues']) + def test_reasoning_separate_from_answer(self): + value=response();value['output'].insert(0,{'type':'reasoning','summary':[{'type':'summary_text','text':'not answer'}]}) + r=parse(value);self.assertTrue(r['protocol_valid']);self.assertEqual(r['text'],TEXT) + def test_unfinished_reasoning_and_ambiguous_refusal_not_valid(self): + value=response();value['output'].insert(0,{'type':'reasoning','status':'in_progress','summary':[]}) + self.assertFalse(parse(value)['protocol_valid']) + value=response();value['output'][0]['content'][0]['refusal']='declined' + self.assertFalse(parse(value)['protocol_valid']) + def test_false_text_never_from_usage_or_reasoning(self): + value=response(output=[],output_text=TEXT) + self.assertFalse(parse(value)['protocol_valid']) + value=response(output=[{'type':'reasoning','summary':[{'text':TEXT}]}],output_text=TEXT) + self.assertFalse(parse(value)['protocol_valid']) + + def test_unicode_and_escaped_duplicate_model_output(self): + value=response(output_text='{"decision":"route","\\u0064ecision":"route"}',output=[{'type':'message','role':'assistant','content':[{'type':'output_text','text':'{"decision":"route","\\u0064ecision":"route"}'}]}]) + self.assertEqual(parse(value)['disposition'],'invalid_output') + +if __name__=='__main__':unittest.main() diff --git a/scripts/test_incident_adaption_transport.py b/scripts/test_incident_adaption_transport.py new file mode 100644 index 0000000..018fe9b --- /dev/null +++ b/scripts/test_incident_adaption_transport.py @@ -0,0 +1,74 @@ +import unittest +import http.client +from incident_adaption_transport import _exchange, HOST, PATH, MAX_BODY + +class Socket: + def __init__(self):self.timeouts=[] + def settimeout(self,value):self.timeouts.append(value) +class Response: + def __init__(self,chunks,status=200,headers=()):self.chunks=list(chunks);self.status=status;self.headers=headers + def getheaders(self):return self.headers + def read1(self,size): + if not self.chunks:return b'' + item=self.chunks.pop(0) + if isinstance(item,Exception):raise item + return item +class Connection: + def __init__(self,response):self.response=response;self.sock=Socket();self.requests=[];self.closed=False + def request(self,*args,**kwargs):self.requests.append((args,kwargs)) + def getresponse(self):return self.response + def close(self):self.closed=True +class TransportTests(unittest.TestCase): + def exchange(self,response,clock=lambda:1): + connection=Connection(response);records=[];factories=[] + def factory(*args,**kwargs):factories.append((args,kwargs));return connection + result=_exchange(b'{"frozen":true}','TEST_ONLY_NOT_A_SECRET',deadline=5,clock=clock, + persist=lambda k,v:records.append((k,v)),connection_factory=factory) + return result,records,connection,factories + def test_fixed_origin_tls_single_post_and_safe_headers(self): + r,events,c,f=self.exchange(Response([b'hello'],headers=[('Authorization','secret'),('X-Request-Id','r')])) + self.assertTrue(r.complete);self.assertEqual(r.body,b'hello');self.assertEqual(len(c.requests),1) + self.assertEqual(f[0][0],(HOST,));self.assertTrue(f[0][1]['context'].check_hostname) + self.assertEqual(c.requests[0][0],('POST',PATH));self.assertTrue(c.closed) + self.assertEqual(events[0][1]['headers'],[['x-request-id','r']]) + def test_redirect_and_http_errors_never_retry(self): + for code in (301,307,429,500): + with self.subTest(code=code): + r,e,c,f=self.exchange(Response([b'error'],status=code,headers=[('Location','https://evil.test')])) + self.assertEqual(r.status,code);self.assertEqual(len(c.requests),1);self.assertEqual(len(f),1) + def test_partial_body_survives_error_no_exception_message(self): + r,e,c,_=self.exchange(Response([b'partial',TimeoutError('secret appears here')])) + self.assertFalse(r.complete);self.assertEqual(r.body,b'partial');self.assertEqual(r.error,'TimeoutError') + self.assertNotIn('secret appears here',repr(e));self.assertTrue(c.closed) + def test_incomplete_read_exception_preserves_partial_bytes(self): + r,e,c,_=self.exchange(Response([b'first',http.client.IncompleteRead(b'partial',99)])) + self.assertFalse(r.complete);self.assertEqual(r.body,b'firstpartial') + self.assertEqual(r.error,'IncompleteRead') + def test_size_cap_preserves_bounded_prefix(self): + r,e,c,_=self.exchange(Response([b'a'*(MAX_BODY+1)])) + self.assertFalse(r.complete);self.assertEqual(len(r.body),MAX_BODY) + def test_exact_body_ceiling_is_complete(self): + r,e,c,_=self.exchange(Response([b'a'*MAX_BODY],headers=[('Content-Length',str(MAX_BODY))])) + self.assertTrue(r.complete);self.assertEqual(len(r.body),MAX_BODY) + def test_expired_clock_never_connects(self): + r,e,c,f=self.exchange(Response([]),clock=lambda:5) + self.assertFalse(r.complete);self.assertEqual(f,[]);self.assertEqual(c.requests,[]) + def test_chunk_persistence_failure_stops_reads(self): + c=Connection(Response([b'one',b'two']));calls=[] + def persist(k,v): + calls.append(k) + if k=='chunk':raise OSError('secret path') + r=_exchange(b'{}','test',deadline=5,clock=lambda:1,persist=persist,connection_factory=lambda *a,**k:c) + self.assertFalse(r.complete);self.assertEqual(c.response.chunks,[b'two']) + self.assertNotIn('secret path',repr(r)) + def test_short_content_length_is_partial_not_complete(self): + r,e,c,_=self.exchange(Response([b'abc'],headers=[('Content-Length','9')])) + self.assertFalse(r.complete);self.assertEqual(r.body,b'abc') + self.assertFalse(any(k=='complete' for k,v in e)) + def test_invalid_credential_refused_before_factory(self): + for key in ('','abc\r\nInjected:yes','test space','é'): + with self.subTest(key=key),self.assertRaises(ValueError): + _exchange(b'{}',key,deadline=5,clock=lambda:1,persist=lambda *a:None, + connection_factory=lambda *a,**k:self.fail('network factory')) + +if __name__=='__main__':unittest.main() diff --git a/scripts/test_incident_calibration_controller.py b/scripts/test_incident_calibration_controller.py index 0d5e841..4194909 100644 --- a/scripts/test_incident_calibration_controller.py +++ b/scripts/test_incident_calibration_controller.py @@ -89,11 +89,70 @@ def test_receipt_survives_crash_and_sqlite_failures_without_replay(self): for fault in ['after_receipt','hang_after_receipt','sqlite_receipt_failure','sqlite_settle_failure']: with self.subTest(fault=fault): self.out=Path(self.tmp.name)/fault - report=self.execute(fault,1500 if fault=='hang_after_receipt' else 30000) + report=self.execute_receipt_hang() if fault=='hang_after_receipt' else self.execute(fault) with self.subTest(**self.diagnostics(report)): self.assertFalse(report['complete']);self.assertEqual(report['unresolved_hold_units'],269) self.assertEqual(report['known_receipt_units'],180) self.assertEqual(report['reconciliation'][0]['action'],'review_known_receipt_no_resend') + def execute_receipt_hang(self): + import base64 + import signal + from contextlib import closing + from unittest.mock import patch + import incident_calibration_controller as controller + clock=controller.now_ns + observed_at=None;expired_at=None + def expire_after_committed_receipt(mode): + nonlocal observed_at,expired_at + if expired_at is not None:return expired_at + artifact=self.out/'receipt-0.json';ledger=self.out/'ledger.sqlite' + if artifact.exists() and ledger.exists(): + receipt=json.loads(artifact.read_text()) + try: + with closing(sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True,timeout=0)) as db: + row=db.execute('SELECT state,raw_reply,receipt_ns FROM slots WHERE slot=0').fetchone() + except sqlite3.OperationalError: + row=None + if row==('received',base64.b64decode(receipt['raw_reply_base64']),receipt['observed_ns']): + observed_at=time.monotonic() + expired_at=json.loads((self.out/'manifest.json').read_text())['deadline_ns'] + return expired_at + return clock(mode) + # The child and worker are real subprocesses with unchanged clocks. + # Only the parent's supervisor clock advances, after both durable + # receipt copies agree. Startup does not race a 1.5-second receipt test. + with patch.object(controller,'now_ns',expire_after_committed_receipt): + report=self.execute('hang_after_receipt') + diagnostic=self.diagnostics(report) + self.assertIsNotNone(observed_at,diagnostic) + self.assertLess(time.monotonic()-observed_at,5,diagnostic) + self.assertTrue(report['audit_complete'],diagnostic) + self.assertFalse(report['complete'],diagnostic) + self.assertEqual(report['claimed_slots'],1,diagnostic) + self.assertEqual(report['settled_slots'],0,diagnostic) + self.assertEqual(report['ledger_charged_units'],0,diagnostic) + self.assertEqual(report['unresolved_hold_units'],269,diagnostic) + self.assertEqual(report['known_receipt_units'],180,diagnostic) + self.assertEqual(report['new_model_calls'],0,diagnostic) + seal=report['supervisor'] + self.assertTrue(seal['timed_out'],diagnostic) + self.assertTrue(seal['deadline_overrun'],diagnostic) + self.assertTrue(seal['child_reaped'],diagnostic) + self.assertTrue(seal['group_terminated'],diagnostic) + self.assertEqual(seal['exit_code'],-signal.SIGKILL,diagnostic) + self.assertEqual(report['reconciliation'][0]['action'],'review_known_receipt_no_resend',diagnostic) + with self.assertRaises(FileExistsError):self.execute() + return report + def test_receipt_hang_accounting_survives_slow_process_start(self): + from unittest.mock import patch + import incident_calibration_controller as controller + launch=controller.subprocess.Popen + def delayed_launch(*args,**kwargs): + # Exceeds the former receipt test's entire 1.5-second window. + time.sleep(2) + return launch(*args,**kwargs) + with patch.object(controller.subprocess,'Popen',delayed_launch): + self.execute_receipt_hang() def test_wrong_quote_runtime_and_malformed_usage_fail_closed(self): for fault in ['wrong_quote','runtime_mismatch','oversized_reply']: with self.subTest(fault=fault): diff --git a/scripts/test_incident_hosted_controller.py b/scripts/test_incident_hosted_controller.py new file mode 100644 index 0000000..be7bb6f --- /dev/null +++ b/scripts/test_incident_hosted_controller.py @@ -0,0 +1,187 @@ +import json +import tempfile +import unittest +from pathlib import Path +from incident_hosted_controller import HostedLedger,OfflineApproval,OfflinePriceBound,FixtureTransport,HttpReceipt + +MODEL='fixture-hosted-model/1' +def reply(usage=None,status='completed',text='{}'): + return HttpReceipt(200,(('content-type','application/json'),),json.dumps({'id':'resp-fixture','object':'response','model':MODEL,'status':status,'output':[{'type':'message','role':'assistant','content':[{'type':'output_text','text':text}]}],'output_text':text,'usage':usage if usage is not None else {'input_tokens':10,'output_tokens':5,'total_tokens':15}}).encode()) + +class HostedTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.root=Path(self.temp.name)/'run' + self.ledger=HostedLedger.create(self.root,budget=800,clock_domain='fixture-clock',start=0,deadline=1000) + self.quote=OfflinePriceBound(400,2,3,10,20,96) + def op(self,key='a'): + return self.ledger.create_operation(key,'cal-01',MODEL,self.quote) + def dispatch(self,op,transport,now=20): + return self.ledger.dispatch(op,transport,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:now) + def test_defaults_deny_and_exact_key_binding(self): + op=self.op();self.assertEqual(op,self.op()) + with self.assertRaises(ValueError):self.ledger.create_operation('a','cal-02',MODEL,self.quote) + with self.assertRaises(PermissionError):self.ledger.dispatch(op,FixtureTransport(reply())) + self.assertEqual(self.ledger.audit()['claimed_count'],0) + def test_charge_is_synthetic_estimate_and_duplicate_never_resends(self): + op=self.op();t=FixtureTransport(reply());self.dispatch(op,t) + a=self.ledger.audit();self.assertEqual(a['estimated_spent'],45);self.assertEqual(a['held_bound'],0) + self.assertIsNone(a['observed_provider_cost']);self.assertFalse(a['live_admissible']) + self.dispatch(op,t);self.assertEqual(t.calls,1);self.assertEqual(self.ledger.audit()['estimated_spent'],45) + def test_ambiguous_send_survives_restart_blocks_continuation(self): + op=self.op();self.dispatch(op,FixtureTransport(TimeoutError('unknown'))) + reopened=HostedLedger(self.root) + second=reopened.create_operation('b','cal-02',MODEL,self.quote) + t=FixtureTransport(reply());reopened.dispatch(second,t,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:30) + self.assertEqual(t.calls,0);self.assertEqual(reopened.audit()['held_bound'],800) + self.assertFalse(reopened.cancel(op)) + def test_partial_usage_preserved_without_invented_charge(self): + op=self.op();self.dispatch(op,FixtureTransport(reply({'input_tokens':10}))) + a=self.ledger.audit();self.assertEqual(a['held_bound'],400);self.assertEqual(a['estimated_spent'],0) + self.assertEqual(a['operations'][0]['parsed']['usage']['input_tokens'],10) + def test_late_receipt_still_reconciles_once(self): + ticks=iter((20,20,20,1200));op=self.op() + self.ledger.dispatch(op,FixtureTransport(reply()),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:next(ticks)) + a=self.ledger.audit();self.assertEqual(a['estimated_spent'],45);self.assertTrue(a['operations'][0]['late']) + self.ledger.reconcile(op);self.assertEqual(self.ledger.audit()['estimated_spent'],45) + def test_budget_denial_and_cancel_only_known_unsent(self): + self.ledger=HostedLedger.create(Path(self.temp.name)/'small',budget=399,clock_domain='fixture-clock',start=0,deadline=1000) + op=self.op();t=FixtureTransport(reply());self.dispatch(op,t) + self.assertEqual(t.calls,0);self.assertEqual(self.ledger.audit()['held_bound'],0) + self.assertEqual(self.ledger.audit()['operations'][0]['state'],'denied') + def test_concurrent_duplicate_claim_single_invocation(self): + from concurrent.futures import ThreadPoolExecutor + op=self.op();self.ledger.reserve(op) + def claim(_):return HostedLedger(self.root).claim(op,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + with ThreadPoolExecutor(max_workers=4) as pool:claims=list(pool.map(claim,range(4))) + self.assertEqual(sum(x is not None for x in claims),1) + self.assertEqual(self.ledger.audit()['held_bound'],400) + + def test_inconsistent_bound_rejected_and_max_claims_persisted(self): + with self.assertRaises(ValueError):self.ledger.create_operation('bad','cal-01',MODEL,OfflinePriceBound(100,2,3,10,20)) + limited=HostedLedger.create(Path(self.temp.name)/'limited',budget=800,clock_domain='fixture-clock',start=0,deadline=1000,max_claims=1) + first=limited.create_operation('a','cal-01',MODEL,self.quote) + limited.dispatch(first,FixtureTransport(reply()),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + second=limited.create_operation('b','cal-02',MODEL,self.quote);t=FixtureTransport(reply()) + limited.dispatch(second,t,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:30) + self.assertEqual(t.calls,0);self.assertEqual(limited.audit()['claimed_count'],1) + def test_final_clock_expiry_skips_transport_preserves_hold(self): + op=self.op();t=FixtureTransport(reply());ticks=iter((20,1000)) + self.ledger.dispatch(op,t,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:next(ticks)) + self.assertEqual(t.calls,0);self.assertEqual(self.ledger.audit()['held_bound'],400) + def test_postcommit_expiry_never_invokes_transport(self): + op=self.op();t=FixtureTransport(reply());ticks=iter((20,20,1000)) + self.ledger.dispatch(op,t,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:next(ticks)) + self.assertEqual(t.calls,0);self.assertEqual(self.ledger.audit()['held_bound'],400) + def test_receipt_clock_failure_preserves_body_and_usage(self): + op=self.op();ticks=iter((20,20,20)) + report=self.ledger.dispatch(op,FixtureTransport(reply()),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:next(ticks)) + self.assertEqual(report['estimated_spent'],45,report);self.assertIsNone(report['operations'][0]['late']) + self.assertTrue((self.root/f'receipt-{op}.json').exists()) + def test_clock_floor_survives_late_receipt_and_restart(self): + op=self.op();ticks=iter((20,20,20,1200)) + self.ledger.dispatch(op,FixtureTransport(reply()),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:next(ticks)) + reopened=HostedLedger(self.root);second=reopened.create_operation('b','cal-02',MODEL,self.quote);reopened.reserve(second) + self.assertIsNone(reopened.claim(second,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:60)) + def test_over_bound_and_bad_usage_retains_raw_counts(self): + for i,usage in enumerate(({'input_tokens':21,'output_tokens':5,'total_tokens':26},{'input_tokens':True,'output_tokens':5,'total_tokens':6},{'input_tokens':10,'output_tokens':5,'total_tokens':99})): + with self.subTest(usage=usage): + ledger=HostedLedger.create(Path(self.temp.name)/f'bad-{i}',budget=400,clock_domain='fixture-clock',start=0,deadline=1000) + op=ledger.create_operation('a','cal-01',MODEL,self.quote) + a=ledger.dispatch(op,FixtureTransport(reply(usage)),approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + self.assertEqual(a['held_bound'],400,a);self.assertEqual(a['estimated_spent'],0,a) + self.assertEqual(a['operations'][0]['parsed']['usage']['raw'],usage) + def test_terminal_invalid_output_still_has_estimated_cost(self): + op=self.op();a=self.dispatch(op,FixtureTransport(reply(status='incomplete'))) + self.assertEqual(a['estimated_spent'],45,a);self.assertFalse(a['operations'][0]['parsed']['protocol_valid']) + def test_duplicate_concurrent_receipts_do_not_double_estimate(self): + from concurrent.futures import ThreadPoolExecutor + op=self.op();self.ledger.reserve(op);self.ledger.claim(op,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + def record(i):HostedLedger(self.root).record_receipt(op,reply(),observed=30+i) + with ThreadPoolExecutor(max_workers=4) as pool:list(pool.map(record,range(4))) + self.ledger.reconcile(op);self.ledger.reconcile(op) + self.assertEqual(self.ledger.audit()['estimated_spent'],45) + with self.assertRaises(ValueError):self.ledger.record_receipt(op,reply(text='different'),observed=99) + def test_independent_receipt_persistence_and_header_allowlist(self): + from unittest.mock import patch + import incident_hosted_controller as controller + op=self.op();r=reply();r=HttpReceipt(r.status,(('authorization','secret'),('x-request-id','safe')),r.body) + original=controller._write + def fail_receipt(path,value): + if path.name.startswith('receipt-'):raise OSError('injected') + return original(path,value) + with patch.object(controller,'_write',fail_receipt):a=self.dispatch(op,FixtureTransport(r)) + self.assertEqual(a['estimated_spent'],45,a) + import sqlite3 + db=sqlite3.connect(self.root/'ledger.sqlite');raw=db.execute('SELECT payload FROM receipts').fetchone()[0];db.close() + self.assertNotIn('secret',raw);self.assertIn('safe',raw) + def test_audit_recomputes_raw_and_rejects_tampering(self): + import sqlite3 + op=self.op();self.dispatch(op,FixtureTransport(reply())) + before={p.name:p.read_bytes() for p in self.root.iterdir() if p.is_file()} + self.ledger.audit();self.assertEqual(before,{p.name:p.read_bytes() for p in self.root.iterdir() if p.is_file()}) + db=sqlite3.connect(self.root/'ledger.sqlite');db.execute('UPDATE operations SET estimated=1 WHERE id=?',(op,));db.commit();db.close() + a=self.ledger.audit();self.assertFalse(a['audit_complete'],a);self.assertIsNone(a['estimated_spent']) + def test_fallback_only_receipt_preserves_late_and_reconciles(self): + import sqlite3 + op=self.op();self.ledger.reserve(op);self.ledger.claim(op,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + db=sqlite3.connect(self.root/'ledger.sqlite');db.execute("CREATE TRIGGER block_receipt BEFORE INSERT ON receipts BEGIN SELECT RAISE(ABORT,'injected'); END");db.commit();db.close() + self.ledger.record_receipt(op,reply(),observed=1200) + a=self.ledger.audit();self.assertEqual(a['held_bound'],400,a);self.assertTrue(a['operations'][0]['late']);self.assertEqual(a['operations'][0]['recomputed_estimate'],45) + self.ledger.reconcile(op);a=self.ledger.audit();self.assertEqual(a['estimated_spent'],45,a) + + def test_conflicting_receipt_cannot_release_hold_or_enable_continuation(self): + import sqlite3 + op=self.op();self.ledger.reserve(op);self.ledger.claim(op,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + self.ledger.record_receipt(op,reply(),observed=30) + path=self.root/f'receipt-{op}.json';value=json.loads(path.read_text());value['status']=201;path.write_text(json.dumps(value)) + with self.assertRaises(ValueError):self.ledger.reconcile(op) + db=sqlite3.connect(self.root/'ledger.sqlite') + self.assertEqual(db.execute('SELECT state,estimated FROM operations WHERE id=?',(op,)).fetchone(),('received',None));db.close() + second=self.ledger.create_operation('b','cal-02',MODEL,self.quote);t=FixtureTransport(reply()) + self.dispatch(second,t);self.assertEqual(t.calls,0) + def test_fallback_send_clock_checked_before_reconciliation(self): + import sqlite3 + op=self.op();self.ledger.reserve(op);self.ledger.claim(op,approval=OfflineApproval(),clock_domain='fixture-clock',clock=lambda:20) + db=sqlite3.connect(self.root/'ledger.sqlite');db.execute("CREATE TRIGGER block_receipt BEFORE INSERT ON receipts BEGIN SELECT RAISE(ABORT,'injected'); END");db.commit();db.close() + self.ledger.record_receipt(op,reply(),observed=30) + path=self.root/f'receipt-{op}.json';value=json.loads(path.read_text());value['send_tick']=1000;value['observed']=1001;path.write_text(json.dumps(value)) + with self.assertRaises(ValueError):self.ledger.reconcile(op) + db=sqlite3.connect(self.root/'ledger.sqlite') + self.assertEqual(db.execute('SELECT state,estimated FROM operations WHERE id=?',(op,)).fetchone(),('claimed',None));db.close() + + def test_connections_close_on_success_early_return_and_rollback(self): + import sqlite3 + from unittest.mock import patch + import incident_hosted_controller as controller + original = controller._connect + opened = [] + + def track(*args, **kwargs): + db = original(*args, **kwargs) + opened.append(db) + return db + + try: + with patch.object(controller, '_connect', track): + ledger = HostedLedger.create(Path(self.temp.name) / 'closed', budget=800, + clock_domain='fixture-clock', start=0, deadline=1000) + with patch.object(ledger, '_event', side_effect=RuntimeError('event failure')): + with self.assertRaises(RuntimeError): + ledger.create_operation('rolled-back', 'cal-01', MODEL, self.quote) + self.assertEqual(ledger.audit()['operations'], []) + op = ledger.create_operation('a', 'cal-01', MODEL, self.quote) + self.assertEqual(ledger.create_operation('a', 'cal-01', MODEL, self.quote), op) + report = ledger.dispatch(op, FixtureTransport(reply()), approval=OfflineApproval(), + clock_domain='fixture-clock', clock=lambda:20) + self.assertEqual(report['estimated_spent'], 45) + self.assertFalse(ledger.cancel(op)) + ledger.reconcile(op) + for index, db in enumerate(opened): + with self.subTest(connection=index), self.assertRaises(sqlite3.ProgrammingError): + db.execute('SELECT 1') + finally: + for db in opened: + db.close() + +if __name__=='__main__':unittest.main() diff --git a/scripts/test_incident_hosted_runner.py b/scripts/test_incident_hosted_runner.py new file mode 100644 index 0000000..b73e58e --- /dev/null +++ b/scripts/test_incident_hosted_runner.py @@ -0,0 +1,225 @@ +import hashlib +import multiprocessing +import os +import signal +import subprocess +import sys +import time +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from incident_hosted_runner import run_fixture, run_live, audit_run, LiveApproval +from incident_adaption_backend import render_case + +class RunnerTests(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup);self.root=Path(self.temp.name)/'run' + def execute(self,root=None,*,fault='none'): + # Protocol/audit assertions must reach their intended path despite slow + # durable setup. Explicit clock-boundary tests below cover expiry. + return run_fixture(self.root if root is None else root,duration_ms=30000,fault=fault) + def test_live_default_deny_before_credentials_or_network(self): + with patch('incident_hosted_runner.os.environ',{}),patch('incident_hosted_runner.send_https',side_effect=AssertionError('network')): + with self.assertRaises(PermissionError):run_live(self.root,'cal-01','some-model',budget_micro_usd=1000) + self.assertFalse(self.root.exists()) + def test_wrong_request_and_missing_billing_attestation_denied(self): + rendered=render_case('cal-01','some-model') + for sha,bounded in [('wrong',True),(rendered['request_sha256'],False)]: + approval=LiveApproval(sha,'some-model',100,'approval','price',True,True,bounded) + with self.subTest(sha=sha,bounded=bounded),self.assertRaises(ValueError): + run_live(self.root,'cal-01','some-model',approval=approval,budget_micro_usd=100) + self.assertFalse(self.root.exists()) + def test_fixture_complete_usage_but_full_hold_not_invoice(self): + result=self.execute() + self.assertTrue(result['audit_complete']);self.assertTrue(result['receipt_complete']) + self.assertEqual(result['claimed_count'],1);self.assertEqual(result['new_model_calls'],0) + self.assertEqual(result['held_micro_usd'],400);self.assertIsNone(result['verified_provider_cost']) + self.assertEqual(result['provider_reported_usage']['total_tokens'],15) + self.assertEqual(result['monetary_basis'],'synthetic_fixture_bound') + def test_rerun_cannot_resend_and_audit_does_not_mutate(self): + self.execute() + before={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in self.root.iterdir()} + with self.assertRaises(FileExistsError):self.execute() + self.assertTrue(audit_run(self.root)['audit_complete']) + after={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in self.root.iterdir()} + self.assertEqual(before,after) + def test_crash_and_partial_keep_uncertain_hold(self): + for fault,exitcode,raw_bytes in (('crash',7,0),('partial',8,50)): + with self.subTest(fault=fault): + result=self.execute(self.root.with_name(fault),fault=fault) + self.assertTrue(result['audit_complete'],result);self.assertEqual(result['claimed_count'],1,result) + self.assertEqual(result['held_micro_usd'],400,result);self.assertEqual(result['new_model_calls'],0,result) + self.assertFalse(result['receipt_complete'],result);self.assertEqual(result['state'],'delivery_uncertain',result) + self.assertEqual(result['raw_body_bytes'],raw_bytes,result) + self.assertEqual(result['parent']['child_exitcode'],exitcode,result) + self.assertEqual(result['parent']['reason'],'child_exit_without_complete_receipt',result) + def test_hanging_child_is_reaped_after_watchdog_expiry_with_hold(self): + import incident_hosted_runner as runner + hanging=multiprocessing.get_context('fork').Event() + parent_pid=os.getpid();original_sleep=time.sleep;original_clock=runner._now + expired_at=None;observed_at=None + def observe_hang(seconds): + # This forked fixture's only sleep is its real hanging send path. + if os.getpid()!=parent_pid:hanging.set() + return original_sleep(seconds) + def expire_when_hanging(): + nonlocal expired_at,observed_at + if os.getpid()==parent_pid and hanging.is_set(): + if expired_at is None: + observed_at=time.monotonic() + expired_at=json.loads((self.root/'manifest.json').read_text())['deadline_ns'] + return expired_at + return original_clock() + with patch.object(runner.time,'sleep',observe_hang),patch.object(runner,'_now',expire_when_hanging): + result=self.execute(fault='hang') + self.assertTrue(hanging.is_set(),result);self.assertIsNotNone(observed_at,result) + self.assertLess(time.monotonic()-observed_at,5,result) + self.assertTrue(result['audit_complete'],result);self.assertEqual(result['claimed_count'],1,result) + self.assertEqual(result['held_micro_usd'],400,result);self.assertEqual(result['new_model_calls'],0,result) + self.assertFalse(result['receipt_complete'],result);self.assertEqual(result['raw_body_bytes'],0,result) + self.assertEqual(result['state'],'delivery_uncertain',result) + self.assertEqual(result['parent']['reason'],'local_watchdog_expired_remote_outcome_unknown',result) + self.assertEqual(result['parent']['child_exitcode'],-signal.SIGKILL,result) + def test_deadline_before_and_after_claim_never_forks(self): + import sqlite3 + from contextlib import closing + import incident_hosted_runner as runner + for boundary,claims,hold,state in (('before',0,0,'not_sent'),('after',1,400,'delivery_uncertain')): + root=self.root.with_name(boundary) + def clock(): + expire=(root/'request.json').exists() if boundary=='before' else False + if boundary=='after' and (root/'ledger.sqlite').exists(): + with closing(sqlite3.connect((root/'ledger.sqlite').as_uri()+'?mode=ro',uri=True)) as db: + expire=db.execute('SELECT count(*) FROM claim').fetchone()[0]==1 + return json.loads((root/'manifest.json').read_text())['deadline_ns'] if expire else 10_000_000 + with self.subTest(boundary=boundary),patch.object(runner,'_now',clock), \ + patch.object(multiprocessing.process.BaseProcess,'start',side_effect=AssertionError('expired request forked')): + result=run_fixture(root,duration_ms=200) + self.assertTrue(result['audit_complete'],result);self.assertEqual(result['claimed_count'],claims,result) + self.assertEqual(result['held_micro_usd'],hold,result);self.assertEqual(result['state'],state,result) + self.assertFalse(result['receipt_complete'],result);self.assertEqual(result['raw_body_bytes'],0,result) + self.assertEqual(result['new_model_calls'],0,result) + self.assertEqual(result['parent']['reason'],'ValueError',result) + self.assertIsNone(result['parent']['child_exitcode'],result) + self.assertFalse(any(root.glob('transport-*.json'))) + def test_raw_request_and_manifest_tampering_refused(self): + self.execute() + request=self.root/'request.json';request.write_text('{"body_base64":"e30="}') + self.assertFalse(audit_run(self.root)['audit_complete']) + def test_incomplete_temporary_record_does_not_destroy_committed_prefix(self): + self.execute() + (self.root/'transport-00003.json.tmp').write_bytes(b'{partial') + result=audit_run(self.root) + self.assertTrue(result['audit_complete']);self.assertEqual(result['provider_reported_usage']['total_tokens'],15) + def test_transport_sequence_gap_refused(self): + self.execute() + (self.root/'transport-00001.json').unlink() + self.assertFalse(audit_run(self.root)['audit_complete']) + def test_audit_strict_integer_budget(self): + self.execute() + path=self.root/'manifest.json';original=json.loads(path.read_text()) + for field in ('maximum_calls','budget_micro_usd','reserved_micro_usd'): + value=dict(original);value[field]=True;path.write_text(json.dumps(value)) + with self.subTest(field=field):self.assertFalse(audit_run(self.root)['audit_complete']) + def test_rejected_wal_and_journal_never_mutate_source(self): + self.execute() + ledger=self.root/'ledger.sqlite' + for suffix in ('-wal','-shm','-journal'): + sidecar=Path(str(ledger)+suffix);sidecar.write_bytes(b'untrusted sidecar') + before={p.name:p.read_bytes() for p in self.root.iterdir()} + self.assertFalse(audit_run(self.root)['audit_complete']) + self.assertEqual(before,{p.name:p.read_bytes() for p in self.root.iterdir()}) + sidecar.unlink() + raw=bytearray(ledger.read_bytes());raw[18:20]=b'\x02\x02';ledger.write_bytes(raw) + before={p.name:p.read_bytes() for p in self.root.iterdir()} + self.assertFalse(audit_run(self.root)['audit_complete']) + self.assertEqual(before,{p.name:p.read_bytes() for p in self.root.iterdir()}) + def test_supervisor_death_kills_fork_child(self): + script=Path(__file__).resolve().parent + code="import sys;sys.path.insert(0,sys.argv[1]);from incident_hosted_runner import run_fixture;run_fixture(sys.argv[2],duration_ms=10000,fault='hang')" + process=subprocess.Popen([sys.executable,'-c',code,str(script),str(self.root)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + child=None + def proc_for(pid): + for path in Path('/proc').glob('[0-9]*'): + try: + lines=path.joinpath('status').read_text().splitlines() + found=int(next(v.split()[-1] for v in lines if v.startswith('NSpid:'))) + if found==pid:return path + except (OSError,StopIteration,ValueError):pass + return None + def running(pid): + path=proc_for(pid) + if path is None:return False + try:return path.joinpath('stat').read_text().split()[2] not in ('Z','X') + except (FileNotFoundError,ProcessLookupError):return False + try: + deadline=time.monotonic()+5 + while time.monotonic()