diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index 0f25c0188..7ed5eb5f1 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -1,5 +1,44 @@ basePath: / definitions: + adminaudit.Operation: + properties: + action: + type: string + actor_label: + type: string + admin_credential_id: + type: string + created_at: + type: string + id: + type: string + project_id: + type: string + x-nullable: true + request_id: + type: string + resource_id: + type: string + resource_type: + type: string + result_ids: + items: + type: object + type: array + trace_id: + type: string + type: object + adminaudit.Page: + properties: + data: + items: + $ref: '#/definitions/adminaudit.Operation' + type: array + has_more: + type: boolean + next_cursor: + type: string + type: object api.AdminRuntimeObservation: properties: observation: @@ -486,7 +525,7 @@ definitions: properties: data: items: - $ref: '#/definitions/store.ResourceOwner' + $ref: '#/definitions/writeaudit.ResourceOwner' type: array type: object api.RuntimeDiskObservation: @@ -931,58 +970,6 @@ definitions: vaults: type: integer type: object - store.AdminAuditOperation: - properties: - action: - type: string - actor_label: - type: string - admin_credential_id: - type: string - created_at: - type: string - id: - type: string - project_id: - type: string - x-nullable: true - request_id: - type: string - resource_id: - type: string - resource_type: - type: string - result_ids: - items: - type: object - type: array - trace_id: - type: string - type: object - store.AdminAuditPage: - properties: - data: - items: - $ref: '#/definitions/store.AdminAuditOperation' - type: array - has_more: - type: boolean - next_cursor: - type: string - type: object - store.AuditAPIKey: - properties: - id: - type: string - kind: - type: string - name: - type: string - prefix: - type: string - revoked_at: - type: string - type: object store.ExecutorCredential: properties: created_at: @@ -1098,17 +1085,6 @@ definitions: has_more: type: boolean type: object - store.ResourceOwner: - properties: - admin_audit_id: - type: string - api_key: - $ref: '#/definitions/store.AuditAPIKey' - resource_id: - type: string - source: - type: string - type: object store.RuntimeDeploymentView: properties: configuration: @@ -1461,38 +1437,6 @@ definitions: retention_seconds: type: integer type: object - store.WriteOperation: - properties: - action: - type: string - api_key: - $ref: '#/definitions/store.AuditAPIKey' - created_at: - type: string - id: - type: string - parent_id: - type: string - request_id: - type: string - resource_id: - type: string - resource_type: - type: string - trace_id: - type: string - type: object - store.WriteOperationPage: - properties: - data: - items: - $ref: '#/definitions/store.WriteOperation' - type: array - has_more: - type: boolean - next_cursor: - type: string - type: object v1.Agent: properties: id: @@ -3372,6 +3316,62 @@ definitions: required: - type type: object + writeaudit.APIKey: + properties: + id: + type: string + kind: + type: string + name: + type: string + prefix: + type: string + revoked_at: + type: string + type: object + writeaudit.Operation: + properties: + action: + type: string + api_key: + $ref: '#/definitions/writeaudit.APIKey' + created_at: + type: string + id: + type: string + parent_id: + type: string + request_id: + type: string + resource_id: + type: string + resource_type: + type: string + trace_id: + type: string + type: object + writeaudit.Page: + properties: + data: + items: + $ref: '#/definitions/writeaudit.Operation' + type: array + has_more: + type: boolean + next_cursor: + type: string + type: object + writeaudit.ResourceOwner: + properties: + admin_audit_id: + type: string + api_key: + $ref: '#/definitions/writeaudit.APIKey' + resource_id: + type: string + source: + type: string + type: object info: contact: {} description: Deployment and operations routes under /core/v1 for Core Web's server and operator scripts. Every operation requires the Core key; Project API keys and machine credentials are not accepted. @@ -3426,7 +3426,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.AdminAuditPage' + $ref: '#/definitions/adminaudit.Page' "400": description: Bad Request schema: @@ -6128,7 +6128,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/store.WriteOperationPage' + $ref: '#/definitions/writeaudit.Page' "400": description: Bad Request schema: diff --git a/services/core/IMPLEMENTATION.md b/services/core/IMPLEMENTATION.md index 1d6fe4414..bcb109794 100644 --- a/services/core/IMPLEMENTATION.md +++ b/services/core/IMPLEMENTATION.md @@ -6,7 +6,11 @@ These are the code-level rules of `services/core` that no contract states. Contr `api.NewHandler` takes one `api.Dependencies` value, built only in `cmd/server`. Each application area is one field typed as an interface declared in `api` beside its handlers, listing exactly the methods they call. Every field is required and `NewHandler` rejects a missing one, except the optional groups whose comments say what nil means: `Execution` is nil without an execution Worker, `Sandboxes` is nil without a managed sandbox installation and requires `Execution`, and `Execution.NativeInstaller` is nil for a build without a source revision. Handlers never discover a capability by type assertion or fall back to another implementation. API tests use one strict fake per area, `fake`, which fails the test on any call the test did not set. -`internal/persistence/postgres/pgunit` owns Core's PostgreSQL transaction and execution-lease mechanics: pooled read-write and snapshot transactions, the lease's dedicated connection and its gate, the ownership check, the cancellation fence, close, and the execution deadline. Persistence code runs every transaction through it. Outside `persistence` and `store`, only `cmd/server`, which acquires the lease, and test fixtures import it. `internal/persistence/postgres/pgtest` is test support: it opens the dedicated test database under the `oac_*_tests` guard, applies the migrations, and creates isolated databases for database-wide state such as the execution lease. Only test files import it. +`internal/persistence/postgres/pgunit` owns the PostgreSQL mechanics that adapters share: pooled read-write and snapshot transactions; pool-bound queries, used only for a single-statement read that needs no transaction, such as the per-request key lookup; the execution lease (its dedicated connection and gate, the ownership check, the cancellation fence, close, and the execution deadline); identifier parsing (`ParseID`, `PathID`, `LookupCursor`); and detection of text PostgreSQL cannot store (`IsUnstorableText`). Persistence code runs every transaction through it. Outside `persistence` and `store`, only `cmd/server`, which acquires the lease and builds the adapters' pool, and test fixtures import it. `internal/persistence/postgres/pgtest` is test support: it opens the dedicated test database under the `oac_*_tests` guard, applies the migrations, and creates isolated databases for database-wide state such as the execution lease. Only test files import it. + +`internal/persistence/postgres/auditpg` is the one adapter that other adapters call directly. Audit rows are written inside the business transaction, so each adapter calls `RecordWriteAudit`, `RecordAdminMutation` or `RecordDeploymentMutation` with its own transaction's queries; the audit provenance travels in the context. `writeaudit` and `adminaudit` own the sources, their validation, `ErrInvalidSource` and the read models, and `auditpg.Store` serves the audit reads. + +Two errors are shared across domains, each with one `api` helper: `textvalue.ErrUnstorable` (400, `writeTextValueError`) for text PostgreSQL cannot store, and `credentialcrypto.ErrUnavailable` (503, `writeCredentialUnavailableError`) for a missing credential key. The audit `ErrInvalidSource` errors pass through adapters unchanged, and `writeAuditSourceError` maps both to 400. Shared vocabulary has one owner each, and domains use it rather than copy it. `internal/environmentconfig` owns Environment setup, Skills, Plugins and initial files with their validation and public metadata; `Setup.Validate` checks requested configuration, where a Skill may be an unresolved reference, and `Setup.ValidateInstalled` checks frozen, installable configuration. `internal/skills` owns `ParseVersion`, the canonical positive decimal Skill version. `internal/metadata` owns the metadata rules: `Validate` for the pair, key and value limits and U+0000, `ValidateStorable` for U+0000 alone, and `Encode` with its 64 KiB bound. `internal/jsonobject` owns `Normalize`, the stable encoding of stored JSON objects that snapshots and retry identities compare. These packages import no persistence. @@ -18,7 +22,7 @@ Shared vocabulary has one owner each, and domains use it rather than copy it. `i Every Agents API JSON route reads its body through `readJSONObject` before decoding, validation or lookup. The gate requires a JSON Content-Type, applies the route's body limit and rejects invalid UTF-8, malformed JSON (including unpaired surrogate escapes), repeated keys and non-object roots with the official messages; an empty body or `null` becomes `{}`. DELETE, multipart, Core extension and internal routes keep their own readers. Member names match exactly: decode request objects with `decodeInputObject`, or check `inexactMember` before another decoder, so `encoding/json` never matches a case variant. -Report a validation failure that has official evidence through the typed field error, which emits `invalid_request_error` with the observed param and message; keep other local codes until their official fields are sampled. Saved and inline Agent configuration pass one path-tracking validator of the pinned shapes before their parsers and harness admission; do not grow it into a JSON Schema engine. A malformed path identifier must produce exactly the response of a well-formed missing one on that route, including for invalid bodies, queries and storage availability: resolve it to the never-assigned maximum UUID and let the missing path run, or reject it directly only where the lookup is the next check. An `after` cursor that does not resolve inside its already resolved parent, malformed ones included, returns that list family's observed error, and foreign and missing cursors stay identical. U+0000 is rejected explicitly only in metadata (`metadata.`), by the `metadata` package; other stored strings rely on the PostgreSQL error mapping, so keep each request's writes in one transaction. +Report a validation failure that has official evidence through the typed field error, which emits `invalid_request_error` with the observed param and message; keep other local codes until their official fields are sampled. Saved and inline Agent configuration pass one path-tracking validator of the pinned shapes before their parsers and harness admission; do not grow it into a JSON Schema engine. A malformed path identifier must produce exactly the response of a well-formed missing one on that route, including for invalid bodies, queries and storage availability: the handler passes it unchanged, the storage call resolves it with `pgunit.PathID` to the never-assigned maximum UUID, and the missing path runs, or reject it directly only where the lookup is the next check. An `after` cursor that does not resolve inside its already resolved parent, malformed ones included, returns that list family's observed error, and foreign and missing cursors stay identical. U+0000 is rejected explicitly only in metadata (`metadata.`), by the `metadata` package; other stored strings rely on the PostgreSQL error mapping, so keep each request's writes in one transaction. List queries reuse the shared parser and error serializer while keeping each family's limit bounds and error fields. The Environment Files list keeps its own path and cursor parsing but follows the same unknown-key and duplicate-key rules, and still rejects malformed query encoding that the shared lists drop. Change page bounds, cursor ownership or parent lookup order only with evidence for that family, and never reproduce an observed upstream server failure as compatibility behavior. @@ -26,8 +30,8 @@ Requests are served on their canonical path and never redirected. `api.Canonical On the Beta group, the OpenAI-Beta check (exactly one `agents=v1` value) runs before authentication, and authentication precedes every Beta handler, 404 and 405. The API router's own middleware, not the shared log middleware, sets a fresh `X-Request-Id` (also in the log context), `OpenAI-Version`, `OpenAI-Processing-Ms` and `nosniff`. HEAD runs GET routes, except that streaming, content-download, live directory, Runtime observation and Runtime history routes register an explicit HEAD 405. Every 405 of the API router, unknown methods included, has the JSON body and lists the route's methods in `Allow`. -- Stored strings other than metadata rely on PostgreSQL rejecting U+0000 and invalid UTF-8: map SQLSTATE `22021` (text parameter) and `22P05` (`\u0000` in jsonb) to the 400 unstorable-text error, including query filters such as `agent_id`. The failing statement aborts its transaction, so keep each request's writes in one transaction. -- An `after` cursor that cannot name a resource on a lookup list (Agents, Sessions, Turns, Templates, Vaults, Credentials) resolves to the never-assigned maximum UUID and runs the normal lookup, so storage failures and missing rows behave as for a well-formed cursor. Resolve every cursor only inside its already resolved parent and tenant. +- Stored strings other than metadata rely on PostgreSQL rejecting U+0000 and invalid UTF-8: `pgunit.IsUnstorableText` detects SQLSTATE `22021` (text parameter) and `22P05` (`\u0000` in jsonb), and the adapter returns `textvalue.ErrUnstorable`, the 400 unstorable-text error, including for query filters such as `agent_id`. The failing statement aborts its transaction, so keep each request's writes in one transaction. +- An `after` cursor that cannot name a resource on a lookup list (Agents, Sessions, Turns, Templates, Vaults, Credentials) resolves through `pgunit.LookupCursor` to the never-assigned maximum UUID and runs the normal lookup, so storage failures and missing rows behave as for a well-formed cursor. Resolve every cursor only inside its already resolved parent and tenant. - Lists whose parent and cursor lookups are separate statements (Artifacts, Skill versions) re-check the parent before reporting a cursor 400, so a parent deleted in between still returns its 404. Item and Subagent lists read both inside one locked Session transaction. The Skill version cursor lookup is tenant-wide so another Skill's version can be told apart from a missing one; another tenant's version stays missing. ## Source Files and Artifacts @@ -105,7 +109,7 @@ Provider input validation uses the adapter rules in `internal/harnessconfig`: on [Vaults and credentials](../../contracts/agents-api/vaults.md) describes the resources, selection rules, refresh and deletion. The store implements them under these rules: - Credentials are children of tenant-owned Vaults. Creation admits the owner in the same SQL statement as the insert; retrieval joins the owning Vault; listing enforces Project and Vault ownership on the parent, cursor and row query. Metadata queries never select ciphertext and need no encryption key. -- Secret values are encrypted before they reach SQL, with Core's separately configured random 32-byte key and the standard library's random-nonce AES-GCM. The versioned authenticated binding covers tenant, Vault, Credential, authentication purpose and exact destination. Never reuse daemon transport encryption for this storage. A missing key disables credential writes; a malformed configured key fails startup. +- Secret values are encrypted before they reach SQL, with Core's separately configured random 32-byte key and the standard library's random-nonce AES-GCM. The versioned authenticated binding covers tenant, Vault, Credential, authentication purpose and exact destination. Never reuse daemon transport encryption for this storage. A missing key disables credential writes with `credentialcrypto.ErrUnavailable`; a malformed configured key fails startup. - A static replacement is one SQL mutation scoped by tenant, Vault, Credential, auth type and destination, reusing the safe metadata for the immutable binding; it never decrypts the previous token, and a failed write keeps the old row. - OAuth refresh and replacement serialize on the Credential row lock, authenticate the stored grant metadata against its encrypted copy before using an endpoint, and persist the refreshed grant before returning an access token, so a stale refresh cannot undo a deletion or Vault cascade. - Credential deletion is one mutation checked by tenant, Vault and ID; Vault deletion removes the parent and its Credentials through the foreign-key cascade in one SQL statement, without decrypting, needing the key or calling providers. @@ -175,5 +179,5 @@ The [managed lifecycle](../../docs/sandbox-provider.md#managed-lifecycle) descri - Core error details are scoped by the `/core/v1` router's writer mark, never by a request path test. Write Core errors with `writeCoreError` and typed `CoreErrorDetails` values (string, number, null and string-array constructors); invalid or empty details are omitted as a whole. The mark preserves error observation, flushing and `http.ResponseController` access. A shared handler or a Core-looking path alone never changes a public or machine error envelope. Core authentication runs before operation configuration checks, and unknown paths keep their status and admission rules. When adding a code with details, document its fixed keys in `contracts/agents-api/core-errors.md`, and pass only safe Core-owned facts: never submitted values, secrets, native text or provider bodies. - Operation validators keep their original error text, sentinel identity and validation precedence. Package-owned typed errors carry fixed field metadata; only the marked Core error mapper translates it into operation codes and safe bound or catalog details. Keep Project and key rune limits separate from node byte limits. Sandbox validation metadata travels through its store wrapper without changing transaction or provider authority. Public Session provider validation stays byte-for-byte unchanged; cover it with handler-level golden responses. The Core clients ignore malformed optional details and never retry a write. - Core metrics instrument the existing worker and job owners without changing scheduling, lease or retention behavior. Count `execution_unavailable` at the HTTP error writer, once per rejected response; never capture request or response bodies and never infer the count from other 503s or failed Turns. Process CPU, RSS and cgroup limits are sampled by the 30-second Core metrics loop into the same bounded in-memory ring; the first CPU interval and restart gaps stay null, and host usage never substitutes for process usage. Root Turn history is queried read-only from PostgreSQL with native timestamps. Builds inject the source commit with `-ldflags` into `main.buildRevision`. Keep the response shape aligned with `packages/agents-client/src/core-metrics.ts`. -- Public resource writes carry the authenticated key's provenance separately from the execution principal. Record the operation and any creation ownership in the business transaction, never in response middleware or an asynchronous queue; a failed record rolls back the write. Internal lifecycle and refresh work never acquires public provenance, and retries never replace ownership. Environment uploads persist the safe request origin before dispatch and record success with the confirmed Runtime receipt, not the native filesystem call. Never put payloads, paths or secrets in audit metadata. Do not confuse key identity with the Session creator identity used for retries. -- Administrator writes reuse the public resource deletion and serialization code and record their administrator audit entry in the same transaction. +- Public resource writes carry the authenticated key's provenance separately from the execution principal. Record the operation and any creation ownership with `auditpg.RecordWriteAudit` in the business transaction, never in response middleware or an asynchronous queue; a failed record rolls back the write. Internal lifecycle and refresh work never acquires public provenance, and retries never replace ownership. Environment uploads persist the safe request origin before dispatch and record success with the confirmed Runtime receipt, not the native filesystem call. Never put payloads, paths or secrets in audit metadata. Do not confuse key identity with the Session creator identity used for retries. +- Administrator writes reuse the public resource deletion and serialization code and record their administrator audit entry with `auditpg.RecordAdminMutation`, or `RecordDeploymentMutation` for a deployment-wide write, in the same transaction. Administrator provenance takes precedence over a key's. diff --git a/services/core/cmd/server/http_routes_test.go b/services/core/cmd/server/http_routes_test.go index 2b61b2a05..e64d696ec 100644 --- a/services/core/cmd/server/http_routes_test.go +++ b/services/core/cmd/server/http_routes_test.go @@ -92,7 +92,7 @@ func daemonComposition(t testing.TB) http.Handler { Agents: struct{ api.Agents }{}, Sessions: struct{ api.Sessions }{}, SessionEvents: struct{ api.SessionEvents }{}, SessionHistory: struct{ api.SessionHistory }{}, Subagents: struct{ api.Subagents }{}, Artifacts: struct{ api.Artifacts }{}, SessionAdmin: struct{ api.SessionAdmin }{}, Environments: struct{ api.Environments }{}, ExecutorConnections: struct{ api.ExecutorConnections }{}, - Admin: struct{ api.Admin }{}, WriteAudit: struct{ api.WriteAudit }{}, Metrics: struct{ api.Metrics }{}, + Admin: struct{ api.Admin }{}, AdminAudit: struct{ api.AdminAudit }{}, WriteAudit: struct{ api.WriteAudit }{}, Metrics: struct{ api.Metrics }{}, RuntimeObservations: struct{ api.RuntimeObservations }{}, RuntimeHistory: struct{ api.RuntimeHistory }{}, Execution: &api.Execution{ExecutorURL: "wss://core.example/api/v1/agent-daemon/ws", Admission: struct{ api.Admission }{}, SessionArchive: struct{ api.SessionArchive }{}, Workspaces: struct{ api.EnvironmentWorkspaces }{}}, diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index 20cf8f058..126aac7b5 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -36,6 +36,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/databaseurl" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/nativeinstaller" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtime" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeenrollment" @@ -106,6 +107,8 @@ func run() error { } executionStore := store.NewWithCredentialCipherAndOAuthRefresh(pool, credentialKey, oauthClient) executionStore.SetPublicURL(public) + units := pgunit.NewPool(pool) + auditStore := auditpg.New(units) installation, err := installationFacts(public) if err != nil { return err @@ -120,7 +123,7 @@ func run() error { auditCleanupDone := make(chan struct{}) go func() { defer close(auditCleanupDone) - runWriteAuditCleanup(auditCleanupCtx, executionStore, auditRetention, metrics) + runWriteAuditCleanup(auditCleanupCtx, auditStore, auditRetention, metrics) }() defer func() { cancelAuditCleanup(); <-auditCleanupDone }() var workerDone chan error @@ -297,7 +300,7 @@ func run() error { Sessions: executionStore, SessionEvents: executionStore, SessionHistory: executionStore, Subagents: executionStore, Artifacts: executionStore, SessionAdmin: executionStore, Environments: executionStore, ExecutorConnections: executorConnections{store: executionStore, registry: registry}, - Admin: executionStore, WriteAudit: executionStore, Metrics: metrics, + Admin: executionStore, AdminAudit: auditStore, WriteAudit: auditStore, Metrics: metrics, RuntimeObservations: observationService, RuntimeHistory: historyService, } if worker != nil { diff --git a/services/core/internal/adminaudit/context.go b/services/core/internal/adminaudit/context.go index 337820994..dd6fe9829 100644 --- a/services/core/internal/adminaudit/context.go +++ b/services/core/internal/adminaudit/context.go @@ -1,4 +1,6 @@ -// Package adminaudit carries non-secret administrator provenance into business transactions. +// Package adminaudit carries non-secret administrator provenance into business +// transactions, and owns the rules for recording it and the audit log's read +// models. persistence/postgres/auditpg stores it. package adminaudit import "context" diff --git a/services/core/internal/adminaudit/reader.go b/services/core/internal/adminaudit/reader.go new file mode 100644 index 000000000..1c4408259 --- /dev/null +++ b/services/core/internal/adminaudit/reader.go @@ -0,0 +1,62 @@ +package adminaudit + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +// ErrInvalidQuery reports an audit log query with an invalid filter, page size +// or cursor. +var ErrInvalidQuery = errors.New("invalid administrator audit query") + +// Reader reads committed administrator mutations as safe metadata, never +// request bodies or secrets. +type Reader interface { + ListAdminAudit(ctx context.Context, filter Filter) (Page, error) +} + +// Filter selects committed administrator mutations, newest first. A zero Limit +// is the default page size. +type Filter struct { + ProjectID, ResourceType, ResourceID, Action, After string + CreatedAfter, CreatedBefore *time.Time + Limit int +} + +// Operation is one administrator write. ProjectID is null for +// deployment-wide writes, such as deployment default model providers. +type Operation struct { + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + AdminCredentialID string `json:"admin_credential_id"` + ActorLabel string `json:"actor_label"` + Action string `json:"action"` + ProjectID *string `json:"project_id" extensions:"x-nullable"` + ResourceType string `json:"resource_type"` + ResourceID string `json:"resource_id"` + ResultIDs json.RawMessage `json:"result_ids" swaggertype:"array,object"` + RequestID string `json:"request_id"` + TraceID string `json:"trace_id"` +} + +type Page struct { + Data []Operation `json:"data"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor"` +} + +// Validate checks f and returns it with the default page size applied. The +// cursor is checked where it is resolved. +func (f Filter) Validate() (Filter, error) { + if f.Limit == 0 { + f.Limit = 50 + } + if f.Limit < 1 || f.Limit > 100 || !writeaudit.ValidText(f.ProjectID, 128, false) || !writeaudit.ValidText(f.ResourceType, 64, false) || !writeaudit.ValidText(f.ResourceID, 256, false) || !writeaudit.ValidText(f.Action, 64, false) || f.CreatedAfter != nil && f.CreatedBefore != nil && !f.CreatedAfter.Before(*f.CreatedBefore) { + return Filter{}, ErrInvalidQuery + } + return f, nil +} diff --git a/services/core/internal/adminaudit/record.go b/services/core/internal/adminaudit/record.go new file mode 100644 index 000000000..ea207e35b --- /dev/null +++ b/services/core/internal/adminaudit/record.go @@ -0,0 +1,38 @@ +package adminaudit + +import ( + "errors" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +// ErrInvalidSource reports an administrator mutation record that cannot be +// recorded: missing or malformed provenance, or a malformed action or resource. +// The mutation it belongs to fails closed. +var ErrInvalidSource = errors.New("invalid administrator audit source") + +// ValidateProjectMutation checks the record of a mutation in the Project that +// s names. +func (s Source) ValidateProjectMutation(action, resourceType, resourceID string) error { + if !writeaudit.ValidText(s.ProjectID, 128, true) { + return ErrInvalidSource + } + return s.validate(action, resourceType, resourceID) +} + +// ValidateDeploymentMutation checks the record of a deployment-wide mutation, +// which has no Project, so s names none. +func (s Source) ValidateDeploymentMutation(action, resourceType, resourceID string) error { + if s.ProjectID != "" { + return ErrInvalidSource + } + return s.validate(action, resourceType, resourceID) +} + +func (s Source) validate(action, resourceType, resourceID string) error { + if !writeaudit.ValidText(s.CredentialID, 64, true) || !writeaudit.ValidText(s.ActorLabel, 128, false) || !writeaudit.ValidText(s.RequestID, 128, true) || !writeaudit.ValidText(s.TraceID, 128, true) || + !writeaudit.ValidText(action, 64, true) || !writeaudit.ValidText(resourceType, 64, true) || !writeaudit.ValidText(resourceID, 256, true) { + return ErrInvalidSource + } + return nil +} diff --git a/services/core/internal/adminaudit/record_test.go b/services/core/internal/adminaudit/record_test.go new file mode 100644 index 000000000..139e18a33 --- /dev/null +++ b/services/core/internal/adminaudit/record_test.go @@ -0,0 +1,53 @@ +package adminaudit + +import ( + "errors" + "strings" + "testing" + "time" +) + +func TestValidateMutation(t *testing.T) { + project := Source{CredentialID: "12345678", ActorLabel: "console", RequestID: "request", TraceID: "trace", ProjectID: "project"} + deployment := project + deployment.ProjectID = "" + if err := project.ValidateProjectMutation("update", "agent", "agent"); err != nil { + t.Fatal(err) + } + if err := deployment.ValidateDeploymentMutation("set", "deployment_model_provider", "codex"); err != nil { + t.Fatal(err) + } + if err := deployment.ValidateProjectMutation("update", "agent", "agent"); !errors.Is(err, ErrInvalidSource) { + t.Fatal("Project mutation without a Project accepted", err) + } + if err := project.ValidateDeploymentMutation("set", "deployment_model_provider", "codex"); !errors.Is(err, ErrInvalidSource) { + t.Fatal("deployment mutation with a Project accepted", err) + } + for name, change := range map[string]func(*Source, *[3]string){ + "credential": func(s *Source, _ *[3]string) { s.CredentialID = "" }, + "actor": func(s *Source, _ *[3]string) { s.ActorLabel = strings.Repeat("x", 129) }, + "request": func(s *Source, _ *[3]string) { s.RequestID = "" }, + "trace": func(s *Source, _ *[3]string) { s.TraceID = "bad\x01" }, + "action": func(_ *Source, r *[3]string) { r[0] = "" }, + "type": func(_ *Source, r *[3]string) { r[1] = strings.Repeat("x", 65) }, + "resource": func(_ *Source, r *[3]string) { r[2] = "" }, + } { + source, record := project, [3]string{"update", "agent", "agent"} + change(&source, &record) + if err := source.ValidateProjectMutation(record[0], record[1], record[2]); !errors.Is(err, ErrInvalidSource) { + t.Errorf("%s accepted: %v", name, err) + } + } +} + +func TestFilterValidate(t *testing.T) { + if f, err := (Filter{}).Validate(); err != nil || f.Limit != 50 { + t.Fatal(f, err) + } + now := time.Now() + for _, f := range []Filter{{Limit: 101}, {ProjectID: strings.Repeat("x", 129)}, {Action: "bad\x00"}, {CreatedAfter: &now, CreatedBefore: &now}} { + if _, err := f.Validate(); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("filter %+v accepted", f) + } + } +} diff --git a/services/core/internal/api/admin_history.go b/services/core/internal/api/admin_history.go index 8e1a8e178..3bce3e882 100644 --- a/services/core/internal/api/admin_history.go +++ b/services/core/internal/api/admin_history.go @@ -1,13 +1,19 @@ package api import ( + "context" "net/http" "net/url" "strconv" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" ) +// AdminAudit reads the administrator audit log. +type AdminAudit interface { + ListAdminAudit(context.Context, adminaudit.Filter) (adminaudit.Page, error) +} + // @Summary Query committed administrator mutations // @Description Core key only. Newest-first cursor pagination of safe metadata. Actor labels are unverified console labels, not authorization identities. Request bodies and secrets are never recorded. // @Tags Core Administration @@ -21,48 +27,48 @@ import ( // @Param created_before query string false "Exclusive RFC3339 timestamp" // @Param limit query int false "Page size" minimum(1) maximum(100) default(50) // @Param after query string false "Opaque next_cursor from the preceding page" -// @Success 200 {object} store.AdminAuditPage +// @Success 200 {object} adminaudit.Page // @Failure 400,401,500 {object} CoreErrorResponse // @Router /core/v1/audit-log [get] func (h *Handler) listAdminAudit(w http.ResponseWriter, r *http.Request) { values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } for name, entries := range values { switch name { case "project_id", "resource_type", "resource_id", "action", "created_after", "created_before", "limit", "after": default: - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } if len(entries) != 1 || entries[0] == "" { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } } - filter := store.AdminAuditFilter{ProjectID: values.Get("project_id"), ResourceType: values.Get("resource_type"), ResourceID: values.Get("resource_id"), Action: values.Get("action"), After: values.Get("after"), Limit: 50} + filter := adminaudit.Filter{ProjectID: values.Get("project_id"), ResourceType: values.Get("resource_type"), ResourceID: values.Get("resource_id"), Action: values.Get("action"), After: values.Get("after"), Limit: 50} if limit := values.Get("limit"); limit != "" { filter.Limit, err = strconv.Atoi(limit) if err != nil || filter.Limit < 1 || filter.Limit > 100 { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } } filter.CreatedAfter, err = adminSummaryTime(r, "created_after") if err != nil { - writeStoreError(w, r, err) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } filter.CreatedBefore, err = adminSummaryTime(r, "created_before") if err != nil { - writeStoreError(w, r, err) + writeAuditError(w, r, adminaudit.ErrInvalidQuery) return } - page, err := h.Admin.ListAdminAudit(r.Context(), filter) + page, err := h.AdminAudit.ListAdminAudit(r.Context(), filter) if err != nil { - writeStoreError(w, r, err) + writeAuditError(w, r, err) return } writeJSON(w, http.StatusOK, page) diff --git a/services/core/internal/api/admin_resources.go b/services/core/internal/api/admin_resources.go index 51e310b62..f7a9b877c 100644 --- a/services/core/internal/api/admin_resources.go +++ b/services/core/internal/api/admin_resources.go @@ -12,12 +12,11 @@ import ( // Only Core-key-authenticated project resource handlers receive it. type adminTenantContextKey struct{} -// Admin reads the administrator's cross-Project views: the asset summary, the -// Sessions whose Runtime is observed and the administrator audit log. +// Admin reads the administrator's cross-Project views: the asset summary and +// the Sessions whose Runtime is observed. type Admin interface { ReadAdminSummary(context.Context, string, store.AdminSummaryFilter, func(store.Session, *string) error) (store.AdminAssetCounts, error) ListAdminRuntimeTargets(context.Context, []string, string, int, bool) (store.AdminRuntimeTargetPage, error) - ListAdminAudit(context.Context, store.AdminAuditFilter) (store.AdminAuditPage, error) } func (h *Handler) adminResourceScope(next http.Handler) http.Handler { diff --git a/services/core/internal/api/agents_update.go b/services/core/internal/api/agents_update.go index 9321c0363..ffd9e756c 100644 --- a/services/core/internal/api/agents_update.go +++ b/services/core/internal/api/agents_update.go @@ -34,12 +34,7 @@ func (h *Handler) updateAgent(w http.ResponseWriter, r *http.Request) { } return } - id := chi.URLParam(r, "agent_id") - if !validAgentID(id) { - // Storage validation precedes the lookup; follow the missing-Agent path. - id = store.UnknownResourceID - } - updated, err := h.Agents.UpdateAgent(r.Context(), tenantID(r), id, input) + updated, err := h.Agents.UpdateAgent(r.Context(), tenantID(r), chi.URLParam(r, "agent_id"), input) if err != nil { writeStoreError(w, r, err) return diff --git a/services/core/internal/api/credentials.go b/services/core/internal/api/credentials.go index 6d06fe108..639f018e2 100644 --- a/services/core/internal/api/credentials.go +++ b/services/core/internal/api/credentials.go @@ -23,7 +23,7 @@ import ( // @Failure 400,401,404,413,500,503 {object} v1.ErrorResponse // @Router /vaults/{vault_id}/credentials [post] func (h *Handler) createCredential(w http.ResponseWriter, r *http.Request) { - vaultID := credentialPathID(r, "vault_id") + vaultID := chi.URLParam(r, "vault_id") raw, ok := readJSONObject(w, r) if !ok { return @@ -107,16 +107,6 @@ func credentialResourceID(w http.ResponseWriter, r *http.Request, param string) return id.String(), true } -// credentialPathID resolves a malformed identifier to one that never exists, -// so body, query and storage checks run exactly as for a missing identifier. -func credentialPathID(r *http.Request, param string) string { - id, err := uuid.Parse(chi.URLParam(r, param)) - if err != nil || id == uuid.Nil { - return store.UnknownResourceID - } - return id.String() -} - func credentialResponse(c store.Credential) v1.Credential { auth := v1.CredentialAuth{Type: c.AuthType, MCPServerURL: c.MCPServerURL} if c.OAuth != nil { diff --git a/services/core/internal/api/credentials_list.go b/services/core/internal/api/credentials_list.go index 9a240f3d0..dbca67869 100644 --- a/services/core/internal/api/credentials_list.go +++ b/services/core/internal/api/credentials_list.go @@ -4,6 +4,7 @@ import ( "net/http" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/go-chi/chi/v5" ) // @Summary List safe Vault Credential metadata @@ -22,7 +23,7 @@ import ( // @Failure 400,401,404,500 {object} v1.ErrorResponse // @Router /vaults/{vault_id}/credentials [get] func (h *Handler) listCredentials(w http.ResponseWriter, r *http.Request) { - vaultID := credentialPathID(r, "vault_id") + vaultID := chi.URLParam(r, "vault_id") options, statuses, ok := readVaultPage(w, r) if !ok { return diff --git a/services/core/internal/api/credentials_list_test.go b/services/core/internal/api/credentials_list_test.go index db6881c35..5896245b8 100644 --- a/services/core/internal/api/credentials_list_test.go +++ b/services/core/internal/api/credentials_list_test.go @@ -63,10 +63,11 @@ func TestCredentialListRejectsInvalidInputBeforeStorage(t *testing.T) { t.Fatal("invalid query reached storage", suffix, w.Code) } } - // A malformed parent follows the missing-Vault path, after query validation. + // A malformed parent reaches storage unchanged after query validation, and + // storage reports it as a missing Vault. h, f, _ := credentialHandler(t) f.err = store.ErrNotFound - if w := credentialRequest(h, "GET", "/v1/vaults/invalid/credentials", ""); w.Code != 404 || f.vault != store.UnknownResourceID { + if w := credentialRequest(h, "GET", "/v1/vaults/invalid/credentials", ""); w.Code != 404 || f.vault != "invalid" { t.Fatal("invalid parent was not resolved as a missing Vault", w.Code, f.vault) } } diff --git a/services/core/internal/api/credentials_oauth_test.go b/services/core/internal/api/credentials_oauth_test.go index 2e81dcb8e..d11d4ecbd 100644 --- a/services/core/internal/api/credentials_oauth_test.go +++ b/services/core/internal/api/credentials_oauth_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" ) @@ -147,7 +148,7 @@ func TestOAuthCredentialStoreFailuresUseSafeExistingErrors(t *testing.T) { for _, tc := range []struct { err error code int - }{{store.ErrNotFound, 404}, {store.ErrInvalidInput, 400}, {store.ErrCredentialStorageUnavailable, 503}, {errors.New("access-canary"), 500}} { + }{{store.ErrNotFound, 404}, {store.ErrInvalidInput, 400}, {credentialcrypto.ErrUnavailable, 503}, {errors.New("access-canary"), 500}} { for _, update := range []bool{false, true} { h, f, _ := credentialHandler(t) f.err = tc.err diff --git a/services/core/internal/api/credentials_test.go b/services/core/internal/api/credentials_test.go index ccb81466f..d8224533a 100644 --- a/services/core/internal/api/credentials_test.go +++ b/services/core/internal/api/credentials_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -112,7 +113,7 @@ func TestCredentialStorageErrorsStaySafe(t *testing.T) { for _, test := range []struct { err error status int - }{{store.ErrNotFound, 404}, {store.ErrCredentialStorageUnavailable, 503}, {errors.New("credential-canary"), 500}} { + }{{store.ErrNotFound, 404}, {credentialcrypto.ErrUnavailable, 503}, {errors.New("credential-canary"), 500}} { h, f, _ := credentialHandler(t) f.err = test.err w := credentialRequest(h, "POST", "/v1/vaults/"+f.credential.VaultID+"/credentials", `{"name":"n","auth":{"type":"static_bearer","mcp_server_url":"https://example.invalid","token":"credential-canary"}}`) diff --git a/services/core/internal/api/credentials_update.go b/services/core/internal/api/credentials_update.go index 673e78b73..7b77aca7a 100644 --- a/services/core/internal/api/credentials_update.go +++ b/services/core/internal/api/credentials_update.go @@ -6,6 +6,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/go-chi/chi/v5" ) // @Summary Replace Vault Credential authentication secrets @@ -22,7 +23,7 @@ import ( // @Failure 400,401,404,413,500,503 {object} v1.ErrorResponse // @Router /vaults/{vault_id}/credentials/{credential_id} [post] func (h *Handler) updateCredential(w http.ResponseWriter, r *http.Request) { - vaultID, id := credentialPathID(r, "vault_id"), credentialPathID(r, "credential_id") + vaultID, id := chi.URLParam(r, "vault_id"), chi.URLParam(r, "credential_id") raw, ok := readJSONObject(w, r) if !ok { return diff --git a/services/core/internal/api/credentials_update_test.go b/services/core/internal/api/credentials_update_test.go index 48da83e14..0d0ff6461 100644 --- a/services/core/internal/api/credentials_update_test.go +++ b/services/core/internal/api/credentials_update_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -69,8 +70,8 @@ func TestCredentialUpdateUsesExistingBoundariesAndSafeErrors(t *testing.T) { h, f, _ := credentialHandler(t) path := "/v1/vaults/" + f.credential.VaultID + "/credentials/" + f.credential.ID method, status := "POST", http.StatusBadRequest - // Malformed identifiers reach storage only as the never-assigned ID, - // after body validation, exactly like a well-formed missing identifier. + // Malformed identifiers reach storage unchanged, after body validation, + // and storage reports them like a well-formed missing identifier. malformed := strings.Contains(mode, "invalid") || strings.Contains(mode, "zero") if malformed { f.err = store.ErrNotFound @@ -99,14 +100,14 @@ func TestCredentialUpdateUsesExistingBoundariesAndSafeErrors(t *testing.T) { } w := httptest.NewRecorder() h.ServeHTTP(w, r) - if w.Code != status || !malformed && f.calls != 0 || malformed && (f.calls != 1 || f.vault != store.UnknownResourceID && f.id != store.UnknownResourceID) { + if w.Code != status || !malformed && f.calls != 0 || malformed && (f.calls != 1 || !strings.Contains(path, "/vaults/"+f.vault+"/credentials/"+f.id)) { t.Fatal("update boundary changed", mode, w.Code, f.vault, f.id) } } for _, tc := range []struct { err error status int - }{{store.ErrNotFound, 404}, {store.ErrCredentialStorageUnavailable, 503}, {errors.New("credential-canary"), 500}} { + }{{store.ErrNotFound, 404}, {credentialcrypto.ErrUnavailable, 503}, {errors.New("credential-canary"), 500}} { h, f, _ := credentialHandler(t) f.err = tc.err w := credentialRequest(h, "POST", "/v1/vaults/"+f.credential.VaultID+"/credentials/"+f.credential.ID, body) diff --git a/services/core/internal/api/dependencies.go b/services/core/internal/api/dependencies.go index ef0cc5bcd..94227f306 100644 --- a/services/core/internal/api/dependencies.go +++ b/services/core/internal/api/dependencies.go @@ -45,6 +45,7 @@ type Dependencies struct { Environments Environments ExecutorConnections ExecutorConnections Admin Admin + AdminAudit AdminAudit WriteAudit WriteAudit Metrics Metrics RuntimeObservations RuntimeObservations @@ -113,7 +114,7 @@ func (d Dependencies) validate() error { field{"EnvironmentTemplates", d.EnvironmentTemplates}, field{"Agents", d.Agents}, field{"Sessions", d.Sessions}, field{"SessionEvents", d.SessionEvents}, field{"SessionHistory", d.SessionHistory}, field{"Subagents", d.Subagents}, field{"Artifacts", d.Artifacts}, field{"SessionAdmin", d.SessionAdmin}, field{"Environments", d.Environments}, - field{"ExecutorConnections", d.ExecutorConnections}, field{"Admin", d.Admin}, field{"WriteAudit", d.WriteAudit}, + field{"ExecutorConnections", d.ExecutorConnections}, field{"Admin", d.Admin}, field{"AdminAudit", d.AdminAudit}, field{"WriteAudit", d.WriteAudit}, field{"Metrics", d.Metrics}, field{"RuntimeObservations", d.RuntimeObservations}, field{"RuntimeHistory", d.RuntimeHistory}, ); err != nil { return err diff --git a/services/core/internal/api/dependencies_test.go b/services/core/internal/api/dependencies_test.go index 71912e4c4..2d09e6b9f 100644 --- a/services/core/internal/api/dependencies_test.go +++ b/services/core/internal/api/dependencies_test.go @@ -31,6 +31,7 @@ type testFakes struct { environments *fakeEnvironments executorConnections *fakeExecutorConnections admin *fakeAdmin + adminAudit *fakeAdminAudit writeAudit *fakeWriteAudit metrics *fakeMetrics runtimeObservations *fakeRuntimeObservations @@ -57,7 +58,7 @@ func testDependencies(t testing.TB) (Dependencies, *testFakes) { agents: &fakeAgents{t: t}, sessions: &fakeSessions{t: t}, sessionEvents: &fakeSessionEvents{t: t}, sessionHistory: &fakeSessionHistory{t: t}, subagents: &fakeSubagents{t: t}, artifacts: &fakeArtifacts{t: t}, sessionAdmin: &fakeSessionAdmin{t: t}, environments: &fakeEnvironments{t: t}, executorConnections: &fakeExecutorConnections{t: t}, - admin: &fakeAdmin{t: t}, writeAudit: &fakeWriteAudit{t: t}, metrics: &fakeMetrics{t: t}, + admin: &fakeAdmin{t: t}, adminAudit: &fakeAdminAudit{t: t}, writeAudit: &fakeWriteAudit{t: t}, metrics: &fakeMetrics{t: t}, runtimeObservations: &fakeRuntimeObservations{t: t}, runtimeHistory: &fakeRuntimeHistory{t: t}, installationBindings: &fakeInstallationBindings{t: t}, admission: &fakeAdmission{t: t}, sessionArchive: &fakeSessionArchive{t: t}, workspaces: &fakeEnvironmentWorkspaces{t: t}, deployment: &fakeDeployment{t: t}, deploymentChanges: &fakeDeploymentChanges{t: t}, configurationDiscovery: &fakeConfigurationDiscovery{t: t}, @@ -67,7 +68,7 @@ func testDependencies(t testing.TB) (Dependencies, *testFakes) { Projects: f.projects, Vaults: f.vaults, ModelProviders: f.modelProviders, Files: f.files, Skills: f.skills, EnvironmentTemplates: f.environmentTemplates, Agents: f.agents, Sessions: f.sessions, SessionEvents: f.sessionEvents, SessionHistory: f.sessionHistory, Subagents: f.subagents, Artifacts: f.artifacts, SessionAdmin: f.sessionAdmin, - Environments: f.environments, ExecutorConnections: f.executorConnections, Admin: f.admin, WriteAudit: f.writeAudit, + Environments: f.environments, ExecutorConnections: f.executorConnections, Admin: f.admin, AdminAudit: f.adminAudit, WriteAudit: f.writeAudit, Metrics: f.metrics, RuntimeObservations: f.runtimeObservations, RuntimeHistory: f.runtimeHistory, }, f } diff --git a/services/core/internal/api/errors.go b/services/core/internal/api/errors.go index 7a05e2ed0..03dda1553 100644 --- a/services/core/internal/api/errors.go +++ b/services/core/internal/api/errors.go @@ -8,11 +8,15 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/textvalue" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) func writeJSON(w http.ResponseWriter, status int, value any) { @@ -183,8 +187,6 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusBadRequest, "invalid_value", "Cannot delete the default skill version.", "version") case errors.Is(err, store.ErrSourceFileTooLarge): writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "File exceeds this operation's content limit.") - case errors.Is(err, store.ErrCredentialStorageUnavailable): - writeError(w, http.StatusServiceUnavailable, "credential_storage_unavailable", "Credential encryption is not configured on this service.") case errors.Is(err, store.ErrModelProviderRequired): writeError(w, http.StatusBadRequest, "model_provider_required", "This Session was created without a model provider and cannot run. Create a new Session with x_agents_core.model_provider or an Agent that has one saved.") case errors.Is(err, store.ErrHostedEnvironmentFailed): @@ -237,14 +239,56 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound case errors.Is(err, store.ErrIdempotencyConflict): writeError(w, http.StatusConflict, "idempotency_conflict", "This idempotency key was used with different input.") case errors.Is(err, store.ErrInvalidInput), errors.Is(err, environmentconfig.ErrInvalid): - writeError(w, http.StatusBadRequest, "invalid_request", "Invalid resource identifier or request limits.") - case store.UnstorableText(err): - // A documented local limit: PostgreSQL text and jsonb cannot store U+0000, - // and text parameters, including query filters, reject invalid UTF-8. - writeError(w, http.StatusBadRequest, "invalid_request_error", unstorableTextMessage) + writeError(w, http.StatusBadRequest, "invalid_request", invalidInputMessage) default: - // Driver errors can include submitted values; do not log the raw error. - log.Ctx(r.Context()).Error("oac-core persistence operation failed") - writeError(w, http.StatusInternalServerError, "internal_error", "The operation could not be completed.") + if writeAuditSourceError(w, r, err) || writeTextValueError(w, r, err) || writeCredentialUnavailableError(w, r, err) { + return + } + writeInternalError(w, r) + } +} + +// invalidInputMessage accompanies the 400 invalid_request for invalid +// identifiers, limits, audit queries and audit provenance. +const invalidInputMessage = "Invalid resource identifier or request limits." + +// writeInternalError reports an unmapped failure. Driver errors can include +// submitted values, so the raw error is never logged. +func writeInternalError(w http.ResponseWriter, r *http.Request) { + log.Ctx(r.Context()).Error("oac-core persistence operation failed") + writeError(w, http.StatusInternalServerError, "internal_error", "The operation could not be completed.") +} + +// writeTextValueError reports request text that PostgreSQL cannot store and +// returns false for any other error. It is a documented local limit: text and +// jsonb cannot store U+0000, and text parameters, including query filters, +// reject invalid UTF-8. +func writeTextValueError(w http.ResponseWriter, r *http.Request, err error) bool { + if !errors.Is(err, textvalue.ErrUnstorable) && !store.UnstorableText(err) { + return false + } + writeError(w, http.StatusBadRequest, "invalid_request_error", unstorableTextMessage) + return true +} + +// writeCredentialUnavailableError reports a request that needs credential +// encryption on a service without a credential key, and returns false for any +// other error. +func writeCredentialUnavailableError(w http.ResponseWriter, r *http.Request, err error) bool { + if !errors.Is(err, credentialcrypto.ErrUnavailable) { + return false + } + writeError(w, http.StatusServiceUnavailable, "credential_storage_unavailable", "Credential encryption is not configured on this service.") + return true +} + +// writeAuditSourceError reports write or administrator provenance that cannot +// be recorded, so the write failed closed, and returns false for any other +// error. +func writeAuditSourceError(w http.ResponseWriter, r *http.Request, err error) bool { + if !errors.Is(err, writeaudit.ErrInvalidSource) && !errors.Is(err, adminaudit.ErrInvalidSource) { + return false } + writeError(w, http.StatusBadRequest, "invalid_request", invalidInputMessage) + return true } diff --git a/services/core/internal/api/errors_audit.go b/services/core/internal/api/errors_audit.go new file mode 100644 index 000000000..012dff7c7 --- /dev/null +++ b/services/core/internal/api/errors_audit.go @@ -0,0 +1,22 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +// writeAuditError maps an audit query error to its Core error response. +func writeAuditError(w http.ResponseWriter, r *http.Request, err error) { + switch { + case errors.Is(err, writeaudit.ErrInvalidQuery), errors.Is(err, adminaudit.ErrInvalidQuery): + writeError(w, http.StatusBadRequest, "invalid_request", invalidInputMessage) + default: + if writeTextValueError(w, r, err) { + return + } + writeInternalError(w, r) + } +} diff --git a/services/core/internal/api/errors_test.go b/services/core/internal/api/errors_test.go index 15abd3aff..97b571dea 100644 --- a/services/core/internal/api/errors_test.go +++ b/services/core/internal/api/errors_test.go @@ -2,6 +2,7 @@ package api import ( "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -9,8 +10,12 @@ import ( "testing" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/textvalue" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) func TestResourceNotFoundErrorSurfaces(t *testing.T) { @@ -154,3 +159,35 @@ func TestConflictErrorsUseConflictType(t *testing.T) { t.Fatal(response.Body) } } + +// The shared persistence errors keep the responses the store errors had: an +// audit source or query that cannot be used answers like invalid input. +func TestSharedPersistenceErrors(t *testing.T) { + storeError := func(w http.ResponseWriter, r *http.Request, err error) { writeStoreError(w, r, err) } + respond := func(write func(http.ResponseWriter, *http.Request, error), err error) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + write(response, httptest.NewRequest(http.MethodPost, "/v1/agents", nil), err) + return response + } + invalid := respond(storeError, store.ErrInvalidInput).Body.String() + for _, test := range []struct { + write func(http.ResponseWriter, *http.Request, error) + err error + status int + body string + }{ + {storeError, fmt.Errorf("write: %w", writeaudit.ErrInvalidSource), 400, invalid}, + {storeError, fmt.Errorf("write: %w", adminaudit.ErrInvalidSource), 400, invalid}, + {writeAuditError, writeaudit.ErrInvalidQuery, 400, invalid}, + {writeAuditError, adminaudit.ErrInvalidQuery, 400, invalid}, + {storeError, fmt.Errorf("write: %w", textvalue.ErrUnstorable), 400, unstorableTextMessage}, + {writeAuditError, textvalue.ErrUnstorable, 400, unstorableTextMessage}, + {storeError, credentialcrypto.ErrUnavailable, 503, "credential_storage_unavailable"}, + {writeAuditError, errors.New("canary"), 500, "internal_error"}, + } { + response := respond(test.write, test.err) + if response.Code != test.status || !strings.Contains(response.Body.String(), test.body) { + t.Errorf("%v: %d %s", test.err, response.Code, response.Body) + } + } +} diff --git a/services/core/internal/api/fakes_test.go b/services/core/internal/api/fakes_test.go index 61679767a..e9a20ed72 100644 --- a/services/core/internal/api/fakes_test.go +++ b/services/core/internal/api/fakes_test.go @@ -8,6 +8,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" @@ -15,6 +16,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) // Strict fakes: one per Dependencies area, with a func field per method. A @@ -33,7 +35,6 @@ type fakeAdmin struct { t testing.TB readAdminSummary func(context.Context, string, store.AdminSummaryFilter, func(store.Session, *string) error) (store.AdminAssetCounts, error) listAdminRuntimeTargets func(context.Context, []string, string, int, bool) (store.AdminRuntimeTargetPage, error) - listAdminAudit func(context.Context, store.AdminAuditFilter) (store.AdminAuditPage, error) } func (f *fakeAdmin) ReadAdminSummary(a0 context.Context, a1 string, a2 store.AdminSummaryFilter, a3 func(store.Session, *string) error) (store.AdminAssetCounts, error) { @@ -50,7 +51,12 @@ func (f *fakeAdmin) ListAdminRuntimeTargets(a0 context.Context, a1 []string, a2 return f.listAdminRuntimeTargets(a0, a1, a2, a3, a4) } -func (f *fakeAdmin) ListAdminAudit(a0 context.Context, a1 store.AdminAuditFilter) (store.AdminAuditPage, error) { +type fakeAdminAudit struct { + t testing.TB + listAdminAudit func(context.Context, adminaudit.Filter) (adminaudit.Page, error) +} + +func (f *fakeAdminAudit) ListAdminAudit(a0 context.Context, a1 adminaudit.Filter) (adminaudit.Page, error) { if f.listAdminAudit == nil { unexpectedCall(f.t, "ListAdminAudit") } @@ -1095,18 +1101,18 @@ func (f *fakeVaults) ResolveMCPCredentials(a0 context.Context, a1 string, a2 []s type fakeWriteAudit struct { t testing.TB - getResourceOwners func(context.Context, string, string, []string) ([]store.ResourceOwner, error) - listWriteOperations func(context.Context, string, store.WriteOperationFilter) (store.WriteOperationPage, error) + getResourceOwners func(context.Context, string, string, []string) ([]writeaudit.ResourceOwner, error) + listWriteOperations func(context.Context, string, writeaudit.Filter) (writeaudit.Page, error) } -func (f *fakeWriteAudit) GetResourceOwners(a0 context.Context, a1 string, a2 string, a3 []string) ([]store.ResourceOwner, error) { +func (f *fakeWriteAudit) GetResourceOwners(a0 context.Context, a1 string, a2 string, a3 []string) ([]writeaudit.ResourceOwner, error) { if f.getResourceOwners == nil { unexpectedCall(f.t, "GetResourceOwners") } return f.getResourceOwners(a0, a1, a2, a3) } -func (f *fakeWriteAudit) ListWriteOperations(a0 context.Context, a1 string, a2 store.WriteOperationFilter) (store.WriteOperationPage, error) { +func (f *fakeWriteAudit) ListWriteOperations(a0 context.Context, a1 string, a2 writeaudit.Filter) (writeaudit.Page, error) { if f.listWriteOperations == nil { unexpectedCall(f.t, "ListWriteOperations") } diff --git a/services/core/internal/api/session_model_defaults.go b/services/core/internal/api/session_model_defaults.go index ad4739d84..2a37bdf46 100644 --- a/services/core/internal/api/session_model_defaults.go +++ b/services/core/internal/api/session_model_defaults.go @@ -6,6 +6,7 @@ import ( "errors" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" ) @@ -27,7 +28,7 @@ func (h *Handler) sessionAgentDefaults(ctx context.Context, tenant string, input return nil, nil, err } if inherit && saved.XAgentsCore != nil && saved.XAgentsCore.ModelProvider != nil && provider == nil { - return nil, nil, store.ErrCredentialStorageUnavailable + return nil, nil, credentialcrypto.ErrUnavailable } return saved, provider, nil } diff --git a/services/core/internal/api/write_audit.go b/services/core/internal/api/write_audit.go index e1a311556..690a45081 100644 --- a/services/core/internal/api/write_audit.go +++ b/services/core/internal/api/write_audit.go @@ -9,23 +9,24 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) // WriteAudit reads public write provenance as safe read models, never request // bodies or credentials. type WriteAudit interface { - GetResourceOwners(context.Context, string, string, []string) ([]store.ResourceOwner, error) - ListWriteOperations(context.Context, string, store.WriteOperationFilter) (store.WriteOperationPage, error) + GetResourceOwners(context.Context, string, string, []string) ([]writeaudit.ResourceOwner, error) + ListWriteOperations(context.Context, string, writeaudit.Filter) (writeaudit.Page, error) } type ResourceOwnerList struct { - Data []store.ResourceOwner `json:"data"` + Data []writeaudit.ResourceOwner `json:"data"` } func (h *Handler) writeAuditScope(w http.ResponseWriter, r *http.Request, allowed ...string) (url.Values, string, bool) { values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return nil, "", false } for name, entries := range values { @@ -36,7 +37,7 @@ func (h *Handler) writeAuditScope(w http.ResponseWriter, r *http.Request, allowe } } if !recognized || len(entries) != 1 || entries[0] == "" { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return nil, "", false } } @@ -65,23 +66,23 @@ func (h *Handler) getResourceOwners(w http.ResponseWriter, r *http.Request) { return } ids := strings.Split(values.Get("resource_ids"), ",") - if !store.ValidAuditResourceType(values.Get("resource_type")) || len(ids) > 100 { - writeStoreError(w, r, store.ErrInvalidInput) + if !writeaudit.ValidResourceType(values.Get("resource_type")) || len(ids) > 100 { + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } for _, id := range ids { if id == "" || len(id) > 256 || strings.TrimSpace(id) != id { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } } owners, err := h.WriteAudit.GetResourceOwners(r.Context(), tenant, values.Get("resource_type"), ids) if err != nil { - writeStoreError(w, r, err) + writeAuditError(w, r, err) return } if owners == nil { - owners = []store.ResourceOwner{} + owners = []writeaudit.ResourceOwner{} } writeJSON(w, http.StatusOK, ResourceOwnerList{Data: owners}) } @@ -99,7 +100,7 @@ func (h *Handler) getResourceOwners(w http.ResponseWriter, r *http.Request) { // @Param created_before query string false "Exclusive RFC3339 timestamp" // @Param limit query int false "Page size, 1-100, default 50" // @Param after query string false "Opaque next_cursor from the preceding page" -// @Success 200 {object} store.WriteOperationPage +// @Success 200 {object} writeaudit.Page // @Failure 400,401,404,500 {object} CoreErrorResponse // @Router /core/v1/projects/{project_id}/write-operations [get] func (h *Handler) listWriteOperations(w http.ResponseWriter, r *http.Request) { @@ -107,25 +108,25 @@ func (h *Handler) listWriteOperations(w http.ResponseWriter, r *http.Request) { if !ok { return } - filter := store.WriteOperationFilter{KeyID: values.Get("key_id"), ResourceType: values.Get("resource_type"), ResourceID: values.Get("resource_id"), After: values.Get("after"), Limit: 50} - if filter.ResourceType != "" && !store.ValidAuditResourceType(filter.ResourceType) { - writeStoreError(w, r, store.ErrInvalidInput) + filter := writeaudit.Filter{KeyID: values.Get("key_id"), ResourceType: values.Get("resource_type"), ResourceID: values.Get("resource_id"), After: values.Get("after"), Limit: 50} + if filter.ResourceType != "" && !writeaudit.ValidResourceType(filter.ResourceType) { + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } for _, value := range []string{filter.KeyID, filter.ResourceID} { if len(value) > 256 { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } } if len(filter.After) > 2048 { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } if value := values.Get("limit"); value != "" { n, err := strconv.Atoi(value) if err != nil || n < 1 || n > 100 { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } filter.Limit = n @@ -134,23 +135,23 @@ func (h *Handler) listWriteOperations(w http.ResponseWriter, r *http.Request) { if value := values.Get(key); value != "" { parsed, err := time.Parse(time.RFC3339Nano, value) if err != nil { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } *target = &parsed } } if filter.CreatedAfter != nil && filter.CreatedBefore != nil && !filter.CreatedAfter.Before(*filter.CreatedBefore) { - writeStoreError(w, r, store.ErrInvalidInput) + writeAuditError(w, r, writeaudit.ErrInvalidQuery) return } page, err := h.WriteAudit.ListWriteOperations(r.Context(), tenant, filter) if err != nil { - writeStoreError(w, r, err) + writeAuditError(w, r, err) return } if page.Data == nil { - page.Data = []store.WriteOperation{} + page.Data = []writeaudit.Operation{} } writeJSON(w, http.StatusOK, page) } diff --git a/services/core/internal/api/write_audit_test.go b/services/core/internal/api/write_audit_test.go index fcc5bff76..008c131c0 100644 --- a/services/core/internal/api/write_audit_test.go +++ b/services/core/internal/api/write_audit_test.go @@ -19,26 +19,26 @@ import ( type auditQueryFixture struct { tenant, resourceType string ids []string - filter store.WriteOperationFilter + filter writeaudit.Filter calls int } -func (s *auditQueryFixture) GetResourceOwners(_ context.Context, tenant, kind string, ids []string) ([]store.ResourceOwner, error) { +func (s *auditQueryFixture) GetResourceOwners(_ context.Context, tenant, kind string, ids []string) ([]writeaudit.ResourceOwner, error) { s.calls++ s.tenant = tenant s.resourceType = kind s.ids = ids - result := make([]store.ResourceOwner, len(ids)) + result := make([]writeaudit.ResourceOwner, len(ids)) for i, id := range ids { result[i].ResourceID = id } return result, nil } -func (s *auditQueryFixture) ListWriteOperations(_ context.Context, tenant string, filter store.WriteOperationFilter) (store.WriteOperationPage, error) { +func (s *auditQueryFixture) ListWriteOperations(_ context.Context, tenant string, filter writeaudit.Filter) (writeaudit.Page, error) { s.calls++ s.tenant = tenant s.filter = filter - return store.WriteOperationPage{}, nil + return writeaudit.Page{}, nil } func TestWriteAuditQueriesDeploymentScopeAndValidation(t *testing.T) { key := callerBinding() @@ -130,7 +130,7 @@ func TestAuthenticatedWriteProvenance(t *testing.T) { func TestAuditQueryJSONSafeFields(t *testing.T) { now := time.Now().UTC() - raw, err := json.Marshal(ResourceOwnerList{Data: []store.ResourceOwner{{ResourceID: "r", APIKey: &store.AuditAPIKey{ID: "key", Name: "sdk", Prefix: "pc_prefix", Kind: "issued", RevokedAt: &now}}}}) + raw, err := json.Marshal(ResourceOwnerList{Data: []writeaudit.ResourceOwner{{ResourceID: "r", APIKey: &writeaudit.APIKey{ID: "key", Name: "sdk", Prefix: "pc_prefix", Kind: "issued", RevokedAt: &now}}}}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/credentialcrypto/cipher.go b/services/core/internal/credentialcrypto/cipher.go index 8a9136545..552bfb7fb 100644 --- a/services/core/internal/credentialcrypto/cipher.go +++ b/services/core/internal/credentialcrypto/cipher.go @@ -18,6 +18,10 @@ const ( bindingDomain = "parsar.agents-api.credential" ) +// ErrUnavailable reports that this service has no credential encryption key, +// so it can neither store nor read credential secrets. +var ErrUnavailable = errors.New("credential encryption is not configured") + var ( errInvalidKey = errors.New("credentialcrypto: invalid encryption key") errUnavailable = errors.New("credentialcrypto: cipher unavailable") diff --git a/services/core/internal/persistence/postgres/auditpg/admin_audit.go b/services/core/internal/persistence/postgres/auditpg/admin_audit.go new file mode 100644 index 000000000..61e8dfea5 --- /dev/null +++ b/services/core/internal/persistence/postgres/auditpg/admin_audit.go @@ -0,0 +1,72 @@ +package auditpg + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" +) + +// ListAdminAudit pages committed administrator mutations across the +// deployment, newest first, with the ID as the tie breaker. +func (s *Store) ListAdminAudit(ctx context.Context, filter adminaudit.Filter) (adminaudit.Page, error) { + page := adminaudit.Page{Data: []adminaudit.Operation{}} + filter, err := filter.Validate() + if err != nil { + return page, err + } + query := filter + query.After, query.Limit = "", 0 + query.CreatedAfter, query.CreatedBefore = utc(query.CreatedAfter), utc(query.CreatedBefore) + scope := cursorScope(query) + params := sqlc.ListAdminAuditLogParams{ProjectID: filter.ProjectID, ResourceType: filter.ResourceType, ResourceID: filter.ResourceID, Action: filter.Action, CreatedAfter: timestamp(filter.CreatedAfter), CreatedBefore: timestamp(filter.CreatedBefore), AfterID: pgtype.UUID{Valid: true}, PageLimit: int32(filter.Limit + 1)} + var rows []sqlc.AdminAuditLog + err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + if filter.After != "" { + id, ok := decodeCursor(filter.After, scope) + if !ok { + return adminaudit.ErrInvalidQuery + } + var err error + params.AfterID = id + params.AfterTime, err = q.AdminAuditCursor(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return adminaudit.ErrInvalidQuery + } + if err != nil { + return err + } + } + var err error + rows, err = q.ListAdminAuditLog(ctx, params) + return err + }) + if err != nil { + return page, err + } + page.HasMore = len(rows) > filter.Limit + if page.HasMore { + rows = rows[:filter.Limit] + } + for _, row := range rows { + page.Data = append(page.Data, adminaudit.Operation{ID: uuid.UUID(row.ID.Bytes).String(), CreatedAt: row.CreatedAt.Time, AdminCredentialID: row.AdminCredentialID, ActorLabel: row.ActorLabel, Action: row.Action, ProjectID: projectID(row.ProjectID), ResourceType: row.ResourceType, ResourceID: row.ResourceID, ResultIDs: row.ResultIds, RequestID: row.RequestID, TraceID: row.TraceID}) + } + if page.HasMore { + page.NextCursor = encodeCursor(page.Data[len(page.Data)-1].ID, scope) + } + return page, nil +} + +func projectID(id pgtype.UUID) *string { + if !id.Valid { + return nil + } + value := uuid.UUID(id.Bytes).String() + return &value +} diff --git a/services/core/internal/persistence/postgres/auditpg/auditpg_test.go b/services/core/internal/persistence/postgres/auditpg/auditpg_test.go new file mode 100644 index 000000000..bc079bd71 --- /dev/null +++ b/services/core/internal/persistence/postgres/auditpg/auditpg_test.go @@ -0,0 +1,467 @@ +package auditpg_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +func openAudit(t *testing.T) (*pgunit.Pool, *auditpg.Store) { + t.Helper() + pool := pgunit.NewPool(pgtest.Open(t)) + return pool, auditpg.New(pool) +} + +func uuidOf(id string) pgtype.UUID { return pgtype.UUID{Bytes: uuid.MustParse(id), Valid: true} } + +func staticSource(tenant string) writeaudit.Source { + sum := sha256.Sum256([]byte(uuid.NewString())) + digest := hex.EncodeToString(sum[:]) + return writeaudit.Source{KeyID: "static:" + digest, Prefix: digest[:8], Name: "test key", Kind: "static", TenantID: tenant, RequestID: uuid.NewString(), TraceID: uuid.NewString()} +} + +func adminSource(projectID string) adminaudit.Source { + return adminaudit.Source{CredentialID: "12345678", ActorLabel: "test", RequestID: uuid.NewString(), TraceID: uuid.NewString(), ProjectID: projectID} +} + +// record runs one audited write in its own transaction. +func record(t *testing.T, pool *pgunit.Pool, ctx context.Context, write func(context.Context, *sqlc.Queries) error) error { + t.Helper() + return pool.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { return write(ctx, sqlc.New(tx)) }) +} + +func recordWrite(t *testing.T, pool *pgunit.Pool, source writeaudit.Source, action, kind, id string, created ...writeaudit.Resource) { + t.Helper() + if err := record(t, pool, writeaudit.WithSource(t.Context(), source), func(ctx context.Context, q *sqlc.Queries) error { + return auditpg.RecordWriteAudit(ctx, q, source.TenantID, action, kind, id, "", created...) + }); err != nil { + t.Fatal(err) + } +} + +type project struct{ id, tenant string } + +func createProject(t *testing.T, pool *pgunit.Pool) project { + t.Helper() + p := project{id: uuid.NewString(), tenant: uuid.NewString()} + if err := record(t, pool, t.Context(), func(ctx context.Context, q *sqlc.Queries) error { + if _, err := q.EnsureProjectScope(ctx, sqlc.EnsureProjectScopeParams{TenantID: uuidOf(p.tenant), OrganizationID: "org_" + p.id, ProjectID: "proj_" + p.id}); err != nil { + return err + } + _, err := q.CreateProject(ctx, sqlc.CreateProjectParams{ID: uuidOf(p.id), Name: "Project", TenantID: uuidOf(p.tenant), SubjectID: p.id}) + return err + }); err != nil { + t.Fatal(err) + } + return p +} + +func createAgent(ctx context.Context, q *sqlc.Queries, tenant, id string) error { + _, err := q.CreateAgent(ctx, sqlc.CreateAgentParams{ID: uuidOf(id), TenantID: uuidOf(tenant), Metadata: []byte("{}"), Configuration: []byte("{}")}) + return err +} + +func agentExists(t *testing.T, pool *pgunit.Pool, tenant, id string) bool { + t.Helper() + var exists bool + if err := pool.Snapshot(t.Context(), func(ctx context.Context, tx pgx.Tx) error { + return tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM agents WHERE tenant_id=$1 AND id=$2)", tenant, id).Scan(&exists) + }); err != nil { + t.Fatal(err) + } + return exists +} + +func exec(t *testing.T, pool *pgunit.Pool, sql string, args ...any) { + t.Helper() + if err := pool.Transaction(t.Context(), func(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, sql, args...) + return err + }); err != nil { + t.Fatal(err) + } +} + +// Audit rows commit and roll back with the business write that records them. +func TestWriteAuditCommitsAndRollsBackWithTheBusinessWrite(t *testing.T) { + pool, audit := openAudit(t) + tenant := uuid.NewString() + source := staticSource(tenant) + for _, failure := range []string{"", "after_audit", "invalid_source", "database_audit_failure"} { + t.Run(failure, func(t *testing.T) { + id := uuid.NewString() + source.RequestID = uuid.NewString() + if failure == "invalid_source" { + source.TraceID = "" + } else { + source.TraceID = uuid.NewString() + } + err := pool.Transaction(writeaudit.WithSource(t.Context(), source), func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + if err := createAgent(ctx, q, tenant, id); err != nil { + return err + } + if failure == "database_audit_failure" { + // This transaction-local constraint deliberately rejects the audit insert. + if _, err := tx.Exec(ctx, "ALTER TABLE write_audit_operations ADD CONSTRAINT audit_test_failure CHECK (false) NOT VALID"); err != nil { + return err + } + } + if err := auditpg.RecordWriteAudit(ctx, q, tenant, "create", "agent", id, "", writeaudit.Resource{Type: "agent", ID: id}); err != nil { + return err + } + if failure == "after_audit" { + return errors.New("business failure after audit") + } + return nil + }) + if (err != nil) != (failure != "") || failure == "invalid_source" && !errors.Is(err, writeaudit.ErrInvalidSource) { + t.Fatalf("commit result: %v", err) + } + if agentExists(t, pool, tenant, id) != (failure == "") { + t.Fatal("business write did not follow the audit outcome") + } + page, err := audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{ResourceID: id}) + want := 0 + if failure == "" { + want = 1 + } + if err != nil || len(page.Data) != want { + t.Fatalf("audit count %d want %d: %v", len(page.Data), want, err) + } + owners, err := audit.GetResourceOwners(t.Context(), tenant, "agent", []string{id}) + if err != nil || (owners[0].APIKey != nil) != (failure == "") { + t.Fatalf("ownership rollback: %+v %v", owners, err) + } + }) + } +} + +// A write without provenance is intentional internal work: no row, no error. +func TestWriteWithoutProvenanceStaysUnattributed(t *testing.T) { + pool, audit := openAudit(t) + tenant, id := uuid.NewString(), uuid.NewString() + if err := record(t, pool, t.Context(), func(ctx context.Context, q *sqlc.Queries) error { + if err := createAgent(ctx, q, tenant, id); err != nil { + return err + } + return auditpg.RecordWriteAudit(ctx, q, tenant, "create", "agent", id, "", writeaudit.Resource{Type: "agent", ID: id}) + }); err != nil { + t.Fatal(err) + } + page, err := audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{}) + if err != nil || len(page.Data) != 0 || !agentExists(t, pool, tenant, id) { + t.Fatalf("unattributed write: %+v %v", page, err) + } + owners, err := audit.GetResourceOwners(t.Context(), tenant, "agent", []string{id}) + if err != nil || owners[0].APIKey != nil || owners[0].Source != nil { + t.Fatalf("unattributed owner: %+v %v", owners, err) + } +} + +// Malformed provenance fails closed before any statement runs, so a nil q +// shows that nothing reached the database. +func TestMalformedProvenanceFailsClosed(t *testing.T) { + valid := staticSource(uuid.NewString()) + for _, field := range []string{"tenant", "key", "prefix", "kind", "request", "trace", "name", "action", "resource_type", "created_type", "resource_id"} { + source, action, kind, id := valid, "create", "agent", "resource" + created := []writeaudit.Resource{{Type: "agent", ID: "resource"}} + switch field { + case "tenant": + source.TenantID = uuid.NewString() + case "key": + source.KeyID = "static:abcd" + case "prefix": + source.Prefix = "bad" + case "kind": + source.Kind = "unknown" + case "request": + source.RequestID = "" + case "trace": + source.TraceID = "" + case "name": + source.Name = strings.Repeat("x", 81) + case "action": + action = "copy" + case "resource_type": + kind = "project" + case "created_type": + created[0].Type = "project" + case "resource_id": + id = "" + } + ctx := writeaudit.WithSource(t.Context(), source) + if err := auditpg.RecordWriteAudit(ctx, nil, valid.TenantID, action, kind, id, "", created...); !errors.Is(err, writeaudit.ErrInvalidSource) { + t.Fatalf("%s accepted: %v", field, err) + } + } + admin := adminSource(uuid.NewString()) + for name, ctx := range map[string]context.Context{ + "missing": t.Context(), + "no request": adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: admin.CredentialID, TraceID: admin.TraceID, ProjectID: admin.ProjectID}), + "no credential": adminaudit.WithSource(t.Context(), adminaudit.Source{RequestID: admin.RequestID, TraceID: admin.TraceID, ProjectID: admin.ProjectID}), + "no project": adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: admin.CredentialID, RequestID: admin.RequestID, TraceID: admin.TraceID}), + "invalid project": adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: admin.CredentialID, RequestID: admin.RequestID, TraceID: admin.TraceID, ProjectID: "project"}), + } { + if err := auditpg.RecordAdminMutation(ctx, nil, uuid.NewString(), "update", "agent", "resource"); !errors.Is(err, adminaudit.ErrInvalidSource) { + t.Fatalf("administrator source %s accepted: %v", name, err) + } + } + for name, ctx := range map[string]context.Context{ + "missing": t.Context(), + "project scope": adminaudit.WithSource(t.Context(), admin), + } { + if err := auditpg.RecordDeploymentMutation(ctx, nil, "set", "deployment_model_provider", "codex"); !errors.Is(err, adminaudit.ErrInvalidSource) { + t.Fatalf("deployment source %s accepted: %v", name, err) + } + } +} + +// Administrator provenance takes precedence over a write source in the same +// context, and deployment mutations record no Project. +func TestAdministratorProvenance(t *testing.T) { + pool, audit := openAudit(t) + p := createProject(t, pool) + id := uuid.NewString() + ctx := adminaudit.WithSource(writeaudit.WithSource(t.Context(), staticSource(p.tenant)), adminSource(p.id)) + if err := record(t, pool, ctx, func(ctx context.Context, q *sqlc.Queries) error { + if err := createAgent(ctx, q, p.tenant, id); err != nil { + return err + } + return auditpg.RecordWriteAudit(ctx, q, p.tenant, "create", "agent", id, "", writeaudit.Resource{Type: "agent", ID: id}) + }); err != nil { + t.Fatal(err) + } + writes, err := audit.ListWriteOperations(t.Context(), p.tenant, writeaudit.Filter{}) + if err != nil || len(writes.Data) != 0 { + t.Fatalf("write source recorded beside administrator provenance: %+v %v", writes, err) + } + page, err := audit.ListAdminAudit(t.Context(), adminaudit.Filter{ProjectID: p.id}) + if err != nil || len(page.Data) != 1 || page.Data[0].Action != "create" || page.Data[0].ResourceID != id || page.Data[0].ProjectID == nil || *page.Data[0].ProjectID != p.id { + t.Fatalf("administrator audit: %+v %v", page, err) + } + resource := "deployment-" + uuid.NewString() + if err := record(t, pool, adminaudit.WithSource(t.Context(), adminSource("")), func(ctx context.Context, q *sqlc.Queries) error { + return auditpg.RecordDeploymentMutation(ctx, q, "set", "deployment_model_provider", resource) + }); err != nil { + t.Fatal(err) + } + page, err = audit.ListAdminAudit(t.Context(), adminaudit.Filter{ResourceID: resource}) + if err != nil || len(page.Data) != 1 || page.Data[0].ProjectID != nil || page.Data[0].Action != "set" { + t.Fatalf("deployment audit: %+v %v", page, err) + } + // A failed business write leaves no administrator row. + failed := errors.New("business failure after audit") + if err := record(t, pool, adminaudit.WithSource(t.Context(), adminSource(p.id)), func(ctx context.Context, q *sqlc.Queries) error { + if err := auditpg.RecordAdminMutation(ctx, q, p.tenant, "delete", "agent", id); err != nil { + return err + } + return failed + }); !errors.Is(err, failed) { + t.Fatal(err) + } + if page, err := audit.ListAdminAudit(t.Context(), adminaudit.Filter{ProjectID: p.id, Action: "delete"}); err != nil || len(page.Data) != 0 { + t.Fatalf("rolled back administrator audit: %+v %v", page, err) + } +} + +func TestWriteAuditOwnersIdentityReplayAndRevocation(t *testing.T) { + pool, audit := openAudit(t) + tenant := uuid.NewString() + a := staticSource(tenant) + id, implicit := uuid.NewString(), uuid.NewString() + recordWrite(t, pool, a, "create", "session", id, writeaudit.Resource{Type: "session", ID: id}, writeaudit.Resource{Type: "environment", ID: implicit, ParentID: id}) + // Same request may reach a commit receipt twice but cannot create another owner. + replayID := uuid.NewString() + recordWrite(t, pool, a, "create", "session", id, writeaudit.Resource{Type: "session", ID: replayID}) + b := staticSource(tenant) + b.Kind = "console" + recordWrite(t, pool, b, "update", "session", id) + owners, err := audit.GetResourceOwners(t.Context(), tenant, "session", []string{replayID, id, id, "historical"}) + if err != nil || len(owners) != 4 || owners[0].APIKey != nil || owners[1].APIKey.ID != a.KeyID || owners[2].APIKey.ID != a.KeyID || owners[3].APIKey != nil { + t.Fatalf("owners %+v: %v", owners, err) + } + implicitOwners, err := audit.GetResourceOwners(t.Context(), tenant, "environment", []string{implicit}) + if err != nil || implicitOwners[0].APIKey.ID != a.KeyID { + t.Fatalf("implicit owner: %+v %v", implicitOwners, err) + } + foreign, err := audit.GetResourceOwners(t.Context(), uuid.NewString(), "session", []string{id}) + if err != nil || foreign[0].APIKey != nil { + t.Fatalf("foreign owner: %+v %v", foreign, err) + } + page, err := audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{ResourceID: id}) + if err != nil || len(page.Data) != 2 || page.Data[0].APIKey.Kind != "console" || page.Data[1].APIKey.ID != a.KeyID { + t.Fatalf("request dedup or key identity: %+v %v", page, err) + } + p := createProject(t, pool) + keyID := uuid.NewString() + sum := sha256.Sum256([]byte(keyID)) + if err := record(t, pool, t.Context(), func(ctx context.Context, q *sqlc.Queries) error { + _, err := q.CreateProjectAPIKey(ctx, sqlc.CreateProjectAPIKeyParams{ID: uuidOf(keyID), Name: "issued key", Prefix: "pc_" + hex.EncodeToString(sum[:4]), TokenSha256: hex.EncodeToString(sum[:]), ProjectID: uuidOf(p.id)}) + return err + }); err != nil { + t.Fatal(err) + } + c := staticSource(p.tenant) + c.KeyID, c.Name, c.Prefix, c.Kind = keyID, "issued key", "pc_"+hex.EncodeToString(sum[:4]), "issued" + fileID := "file_" + uuid.NewString() + recordWrite(t, pool, c, "create", "file", fileID, writeaudit.Resource{Type: "file", ID: fileID}) + if err := record(t, pool, t.Context(), func(ctx context.Context, q *sqlc.Queries) error { + _, err := q.RevokeProjectAPIKey(ctx, sqlc.RevokeProjectAPIKeyParams{ID: uuidOf(keyID), ProjectID: uuidOf(p.id)}) + return err + }); err != nil { + t.Fatal(err) + } + revoked, err := audit.GetResourceOwners(t.Context(), p.tenant, "file", []string{fileID}) + if err != nil || revoked[0].APIKey.RevokedAt == nil || revoked[0].APIKey.Name != "issued key" { + t.Fatalf("revocation metadata: %+v %v", revoked, err) + } + page, err = audit.ListWriteOperations(t.Context(), p.tenant, writeaudit.Filter{KeyID: c.KeyID}) + if err != nil || len(page.Data) != 1 || page.Data[0].APIKey.RevokedAt == nil { + t.Fatalf("history revocation: %+v %v", page, err) + } +} + +// The copy operation was removed; its committed provenance must stay readable. +func TestHistoricalAdminCopyProvenance(t *testing.T) { + pool, audit := openAudit(t) + p := createProject(t, pool) + auditID, agentID := uuid.NewString(), uuid.NewString() + exec(t, pool, `INSERT INTO admin_audit_log(id,tenant_id,project_id,admin_credential_id,actor_label,action,resource_type,resource_id,result_ids,request_id,trace_id) + VALUES($1,$2,$3,'digest','admin','copy','agent','source-agent',$4::jsonb,'request','trace')`, auditID, p.tenant, p.id, `[{"type":"agent","source_id":"source-agent","target_id":"`+agentID+`"}]`) + exec(t, pool, "INSERT INTO admin_resource_owners(tenant_id,resource_type,resource_id,audit_id) VALUES($1,'agent',$2,$3)", p.tenant, agentID, auditID) + owners, err := audit.GetResourceOwners(t.Context(), p.tenant, "agent", []string{agentID}) + if err != nil || len(owners) != 1 || owners[0].APIKey != nil || owners[0].Source == nil || *owners[0].Source != "admin_copy" || owners[0].AdminAuditID == nil || *owners[0].AdminAuditID != auditID { + t.Fatalf("historical copy owner: %+v %v", owners, err) + } + page, err := audit.ListAdminAudit(t.Context(), adminaudit.Filter{ProjectID: p.id, Action: "copy"}) + if err != nil || len(page.Data) != 1 || page.Data[0].ID != auditID || !strings.Contains(string(page.Data[0].ResultIDs), agentID) { + t.Fatalf("historical copy audit: %+v %v", page, err) + } +} + +func TestWriteAuditCursorFiltersAndRetention(t *testing.T) { + pool, audit := openAudit(t) + tenant, id := uuid.NewString(), uuid.NewString() + source := staticSource(tenant) + if err := record(t, pool, t.Context(), func(ctx context.Context, q *sqlc.Queries) error { return createAgent(ctx, q, tenant, id) }); err != nil { + t.Fatal(err) + } + recordWrite(t, pool, source, "create", "agent", id, writeaudit.Resource{Type: "agent", ID: id}) + for i := 0; i < 4; i++ { + source.RequestID = uuid.NewString() + recordWrite(t, pool, source, "update", "agent", id) + } + // Equal timestamps exercise the ID tie breaker, independent of insertion order. + stamp := time.Date(1800, 1, 1, 0, 0, 0, 0, time.UTC) + exec(t, pool, "UPDATE write_audit_operations SET created_at=$1 WHERE tenant_id=$2", stamp, tenant) + first, err := audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{Limit: 2}) + if err != nil || len(first.Data) != 2 || !first.HasMore || first.NextCursor == "" { + t.Fatalf("first %+v %v", first, err) + } + seen := map[string]bool{} + page := first + for { + for _, row := range page.Data { + if seen[row.ID] { + t.Fatal("duplicate cursor item") + } + seen[row.ID] = true + } + if !page.HasMore { + break + } + page, err = audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{Limit: 2, After: page.NextCursor}) + if err != nil { + t.Fatal(err) + } + } + if len(seen) != 5 { + t.Fatalf("lost rows %d", len(seen)) + } + for _, filter := range []writeaudit.Filter{{Limit: 2, After: first.NextCursor, KeyID: "different"}, {After: "malformed"}, {Limit: 101}, {ResourceType: "project"}} { + if _, err := audit.ListWriteOperations(t.Context(), tenant, filter); !errors.Is(err, writeaudit.ErrInvalidQuery) { + t.Fatalf("filter %+v: %v", filter, err) + } + } + if _, err := audit.ListWriteOperations(t.Context(), uuid.NewString(), writeaudit.Filter{After: first.NextCursor}); !errors.Is(err, writeaudit.ErrInvalidQuery) { + t.Fatalf("cross tenant cursor: %v", err) + } + for _, filter := range []writeaudit.Filter{{CreatedBefore: &stamp}, {KeyID: "missing"}, {ResourceType: "file"}, {ResourceID: "missing"}} { + page, err := audit.ListWriteOperations(t.Context(), tenant, filter) + if err != nil || len(page.Data) != 0 { + t.Fatalf("filter %+v: %+v %v", filter, page, err) + } + } + inclusive, err := audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{CreatedAfter: &stamp}) + if err != nil || len(inclusive.Data) != 5 { + t.Fatalf("inclusive lower bound: %+v %v", inclusive, err) + } + cutoff := stamp.Add(time.Hour) + n, err := audit.DeleteExpiredWriteOperations(t.Context(), cutoff, 2) + if err != nil || n != 2 { + t.Fatalf("bounded retention %d %v", n, err) + } + n, err = audit.DeleteExpiredWriteOperations(t.Context(), cutoff, 1000) + if err != nil || n != 2 { + t.Fatalf("remaining retention %d %v", n, err) + } + page, err = audit.ListWriteOperations(t.Context(), tenant, writeaudit.Filter{}) + if err != nil || len(page.Data) != 1 || page.Data[0].Action != "create" { + t.Fatalf("creator retention %+v %v", page, err) + } + // Deleting the business resource cannot cascade through provenance. + exec(t, pool, "DELETE FROM agents WHERE tenant_id=$1 AND id=$2", tenant, id) + owners, err := audit.GetResourceOwners(t.Context(), tenant, "agent", []string{id}) + if err != nil || owners[0].APIKey == nil { + t.Fatalf("deleted creator %+v %v", owners, err) + } + for _, query := range []struct { + tenant, kind string + ids []string + }{{"tenant", "agent", []string{id}}, {tenant, "project", []string{id}}, {tenant, "agent", nil}, {tenant, "agent", []string{""}}} { + if _, err := audit.GetResourceOwners(t.Context(), query.tenant, query.kind, query.ids); !errors.Is(err, writeaudit.ErrInvalidQuery) { + t.Fatalf("owner query %+v: %v", query, err) + } + } +} + +func TestAdminAuditCursorAndFilters(t *testing.T) { + pool, audit := openAudit(t) + p := createProject(t, pool) + for range 3 { + if err := record(t, pool, adminaudit.WithSource(t.Context(), adminSource(p.id)), func(ctx context.Context, q *sqlc.Queries) error { + return auditpg.RecordAdminMutation(ctx, q, p.tenant, "update", "agent", "agent-"+p.id) + }); err != nil { + t.Fatal(err) + } + } + first, err := audit.ListAdminAudit(t.Context(), adminaudit.Filter{ProjectID: p.id, Limit: 2}) + if err != nil || len(first.Data) != 2 || !first.HasMore || first.NextCursor == "" { + t.Fatalf("first %+v %v", first, err) + } + rest, err := audit.ListAdminAudit(t.Context(), adminaudit.Filter{ProjectID: p.id, Limit: 2, After: first.NextCursor}) + if err != nil || len(rest.Data) != 1 || rest.HasMore || rest.Data[0].ID == first.Data[0].ID || rest.Data[0].ID == first.Data[1].ID { + t.Fatalf("rest %+v %v", rest, err) + } + for _, filter := range []adminaudit.Filter{{ProjectID: p.id, Limit: 2, After: first.NextCursor, Action: "delete"}, {After: "malformed"}, {Limit: 101}, {ProjectID: strings.Repeat("x", 129)}} { + if _, err := audit.ListAdminAudit(t.Context(), filter); !errors.Is(err, adminaudit.ErrInvalidQuery) { + t.Fatalf("filter %+v: %v", filter, err) + } + } +} diff --git a/services/core/internal/persistence/postgres/auditpg/record.go b/services/core/internal/persistence/postgres/auditpg/record.go new file mode 100644 index 000000000..7543c214c --- /dev/null +++ b/services/core/internal/persistence/postgres/auditpg/record.go @@ -0,0 +1,101 @@ +// Package auditpg stores write and administrator audit records in PostgreSQL. +// Every other adapter records audit rows through it, inside its own business +// transaction; it is the one adapter that other adapters call directly. +package auditpg + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +// The recorders run in the caller's business transaction: q must belong to it, +// and they never begin or commit one. An error aborts the business write. No +// request body or secret reaches a row. + +// RecordWriteAudit records a public write in tenant and the resources the write +// genuinely created. Administrator provenance takes precedence. A write without +// provenance, such as internal lifecycle work, stays unattributed; malformed +// provenance fails closed with writeaudit.ErrInvalidSource. A request that +// already recorded its operation records nothing more. +func RecordWriteAudit(ctx context.Context, q *sqlc.Queries, tenant, action, resourceType, resourceID, parentID string, created ...writeaudit.Resource) error { + if _, ok := adminaudit.FromContext(ctx); ok { + return RecordAdminMutation(ctx, q, tenant, action, resourceType, resourceID) + } + source, ok := writeaudit.FromContext(ctx) + if !ok { + return nil + } + resources := append([]writeaudit.Resource{{Type: resourceType, ID: resourceID, ParentID: parentID}}, created...) + if err := writeaudit.ValidateRecord(source, tenant, action, resources); err != nil { + return err + } + tenantID, err := pgunit.ParseID(tenant) + if err != nil { + return writeaudit.ErrInvalidSource + } + id, err := q.InsertWriteAuditOperation(ctx, sqlc.InsertWriteAuditOperationParams{ + ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, TenantID: tenantID, + KeyID: source.KeyID, KeyName: source.Name, KeyPrefix: source.Prefix, KeyKind: source.Kind, + Action: action, ResourceType: resourceType, ResourceID: resourceID, ParentID: parentID, RequestID: source.RequestID, TraceID: source.TraceID, + }) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + for _, resource := range created { + if err := q.InsertWriteAuditOwner(ctx, sqlc.InsertWriteAuditOwnerParams{TenantID: tenantID, ResourceType: resource.Type, ResourceID: resource.ID, ParentID: resource.ParentID, OperationID: id}); err != nil { + return err + } + } + return nil +} + +// RecordAdminMutation records an administrator mutation in tenant, the Project +// that the administrator provenance names. Missing or malformed provenance +// fails with adminaudit.ErrInvalidSource. +func RecordAdminMutation(ctx context.Context, q *sqlc.Queries, tenant, action, resourceType, resourceID string) error { + source, ok := adminaudit.FromContext(ctx) + if !ok { + return adminaudit.ErrInvalidSource + } + if err := source.ValidateProjectMutation(action, resourceType, resourceID); err != nil { + return err + } + projectID, err := pgunit.ParseID(source.ProjectID) + if err != nil { + return adminaudit.ErrInvalidSource + } + tenantID, err := pgunit.ParseID(tenant) + if err != nil { + return adminaudit.ErrInvalidSource + } + // result_ids is retained for historical copy mappings; current writes record none. + _, err = q.InsertAdminAudit(ctx, sqlc.InsertAdminAuditParams{ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, TenantID: tenantID, AdminCredentialID: source.CredentialID, ActorLabel: source.ActorLabel, Action: action, ProjectID: projectID, ResourceType: resourceType, ResourceID: resourceID, ResultIds: []byte(`[]`), RequestID: source.RequestID, TraceID: source.TraceID}) + return err +} + +// RecordDeploymentMutation records a deployment-wide administrator mutation, +// which has no Project or tenant. Missing or malformed provenance fails with +// adminaudit.ErrInvalidSource. +func RecordDeploymentMutation(ctx context.Context, q *sqlc.Queries, action, resourceType, resourceID string) error { + source, ok := adminaudit.FromContext(ctx) + if !ok { + return adminaudit.ErrInvalidSource + } + if err := source.ValidateDeploymentMutation(action, resourceType, resourceID); err != nil { + return err + } + _, err := q.InsertAdminAudit(ctx, sqlc.InsertAdminAuditParams{ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, AdminCredentialID: source.CredentialID, ActorLabel: source.ActorLabel, Action: action, ResourceType: resourceType, ResourceID: resourceID, ResultIds: []byte(`[]`), RequestID: source.RequestID, TraceID: source.TraceID}) + return err +} diff --git a/services/core/internal/persistence/postgres/auditpg/store.go b/services/core/internal/persistence/postgres/auditpg/store.go new file mode 100644 index 000000000..e74ccd372 --- /dev/null +++ b/services/core/internal/persistence/postgres/auditpg/store.go @@ -0,0 +1,70 @@ +package auditpg + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "time" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +// Store reads the audit records and applies write audit retention. +type Store struct{ pool *pgunit.Pool } + +func New(pool *pgunit.Pool) *Store { return &Store{pool: pool} } + +var ( + _ writeaudit.Reader = (*Store)(nil) + _ adminaudit.Reader = (*Store)(nil) +) + +// A page cursor names the last row of its page and is bound to the digest of +// the query that produced it, so it cannot resume another tenant's or filter's +// list. +type cursor struct{ ID, Scope string } + +// cursorScope digests a query without its cursor and page size. +func cursorScope(query any) string { + encoded, _ := json.Marshal(query) + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func encodeCursor(id, scope string) string { + encoded, _ := json.Marshal(cursor{ID: id, Scope: scope}) + return base64.RawURLEncoding.EncodeToString(encoded) +} + +// decodeCursor returns the row a cursor names, or false when the cursor is +// malformed or belongs to another query. +func decodeCursor(value, scope string) (pgtype.UUID, bool) { + encoded, err := base64.RawURLEncoding.DecodeString(value) + var decoded cursor + if len(value) > 1024 || err != nil || json.Unmarshal(encoded, &decoded) != nil || decoded.Scope != scope { + return pgtype.UUID{}, false + } + id, err := pgunit.ParseID(decoded.ID) + return id, err == nil +} + +// utc normalizes a bound so that equal instants digest to the same scope. +func utc(value *time.Time) *time.Time { + if value == nil { + return nil + } + normalized := value.UTC() + return &normalized +} + +func timestamp(value *time.Time) pgtype.Timestamptz { + if value == nil { + return pgtype.Timestamptz{} + } + return pgtype.Timestamptz{Time: *value, Valid: true} +} diff --git a/services/core/internal/persistence/postgres/auditpg/write_operations.go b/services/core/internal/persistence/postgres/auditpg/write_operations.go new file mode 100644 index 000000000..806143272 --- /dev/null +++ b/services/core/internal/persistence/postgres/auditpg/write_operations.go @@ -0,0 +1,150 @@ +package auditpg + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" +) + +func apiKey(id, name, prefix, kind string, revoked pgtype.Timestamptz) writeaudit.APIKey { + key := writeaudit.APIKey{ID: id, Name: name, Prefix: prefix, Kind: kind} + if revoked.Valid { + value := revoked.Time + key.RevokedAt = &value + } + return key +} + +// GetResourceOwners returns each resource's recorded creator in request order. +// A resource created through a removed administrator copy reports that audit +// entry instead of a key. +func (s *Store) GetResourceOwners(ctx context.Context, tenantID, resourceType string, resourceIDs []string) ([]writeaudit.ResourceOwner, error) { + tenant, err := pgunit.ParseID(tenantID) + if err != nil { + return nil, writeaudit.ErrInvalidQuery + } + if err := writeaudit.ValidateOwnerQuery(resourceType, resourceIDs); err != nil { + return nil, err + } + keys := make(map[string]writeaudit.APIKey, len(resourceIDs)) + admins := make(map[string]string) + err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + rows, err := q.GetResourceOwners(ctx, sqlc.GetResourceOwnersParams{TenantID: tenant, ResourceType: resourceType, Column3: resourceIDs}) + if err != nil { + return err + } + for _, row := range rows { + keys[row.ResourceID] = apiKey(row.KeyID, row.KeyName, row.KeyPrefix, row.KeyKind, row.RevokedAt) + } + adminRows, err := q.GetAdminResourceOwners(ctx, sqlc.GetAdminResourceOwnersParams{TenantID: tenant, ResourceType: resourceType, Column3: resourceIDs}) + if err != nil { + return err + } + for _, row := range adminRows { + admins[row.ResourceID] = uuid.UUID(row.AuditID.Bytes).String() + } + return nil + }) + if err != nil { + return nil, err + } + result := make([]writeaudit.ResourceOwner, 0, len(resourceIDs)) + for _, id := range resourceIDs { + owner := writeaudit.ResourceOwner{ResourceID: id} + if key, ok := keys[id]; ok { + owner.APIKey = &key + source := "api_key" + owner.Source = &source + } + if auditID, ok := admins[id]; ok && owner.APIKey == nil { + source := "admin_copy" + owner.Source = &source + owner.AdminAuditID = &auditID + } + result = append(result, owner) + } + return result, nil +} + +// ListWriteOperations pages a tenant's committed writes, newest first, with +// the ID as the tie breaker. +func (s *Store) ListWriteOperations(ctx context.Context, tenantID string, filter writeaudit.Filter) (writeaudit.Page, error) { + tenant, err := pgunit.ParseID(tenantID) + if err != nil { + return writeaudit.Page{}, writeaudit.ErrInvalidQuery + } + filter, err = filter.Validate() + if err != nil { + return writeaudit.Page{}, err + } + query := filter + query.After, query.Limit = "", 0 + query.CreatedAfter, query.CreatedBefore = utc(query.CreatedAfter), utc(query.CreatedBefore) + scope := cursorScope(struct { + Tenant string + Filter writeaudit.Filter + }{uuid.UUID(tenant.Bytes).String(), query}) + params := sqlc.ListWriteOperationsParams{TenantID: tenant, KeyID: filter.KeyID, ResourceType: filter.ResourceType, ResourceID: filter.ResourceID, CreatedAfter: timestamp(filter.CreatedAfter), CreatedBefore: timestamp(filter.CreatedBefore), PageLimit: int32(filter.Limit + 1), AfterID: pgtype.UUID{Valid: true}} + var rows []sqlc.ListWriteOperationsRow + err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { + q := sqlc.New(tx) + if filter.After != "" { + id, ok := decodeCursor(filter.After, scope) + if !ok { + return writeaudit.ErrInvalidQuery + } + var err error + params.AfterTime, err = q.GetWriteAuditCursor(ctx, sqlc.GetWriteAuditCursorParams{TenantID: tenant, ID: id}) + if errors.Is(err, pgx.ErrNoRows) { + return writeaudit.ErrInvalidQuery + } + if err != nil { + return err + } + params.AfterID = id + } + var err error + rows, err = q.ListWriteOperations(ctx, params) + return err + }) + if err != nil { + return writeaudit.Page{}, err + } + page := writeaudit.Page{Data: make([]writeaudit.Operation, 0, min(len(rows), filter.Limit)), HasMore: len(rows) > filter.Limit} + if page.HasMore { + rows = rows[:filter.Limit] + } + for _, row := range rows { + page.Data = append(page.Data, writeaudit.Operation{ID: uuid.UUID(row.ID.Bytes).String(), Action: row.Action, ResourceType: row.ResourceType, ResourceID: row.ResourceID, ParentID: row.ParentID, RequestID: row.RequestID, TraceID: row.TraceID, APIKey: apiKey(row.KeyID, row.KeyName, row.KeyPrefix, row.KeyKind, row.RevokedAt), CreatedAt: row.CreatedAt.Time}) + } + if page.HasMore { + page.NextCursor = encodeCursor(page.Data[len(page.Data)-1].ID, scope) + } + return page, nil +} + +// DeleteExpiredWriteOperations deletes at most limit operations older than +// olderThan in one transaction. It never removes an operation referenced by a +// genuine creation anchor, even after the resource is deleted. +func (s *Store) DeleteExpiredWriteOperations(ctx context.Context, olderThan time.Time, limit int) (int64, error) { + if olderThan.IsZero() || limit < 1 || limit > 1000 { + return 0, fmt.Errorf("auditpg: invalid retention batch of %d before %v", limit, olderThan) + } + var deleted int64 + err := s.pool.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { + var err error + deleted, err = sqlc.New(tx).DeleteExpiredWriteOperations(ctx, sqlc.DeleteExpiredWriteOperationsParams{CreatedAt: pgtype.Timestamptz{Time: olderThan, Valid: true}, Limit: int32(limit)}) + return err + }) + return deleted, err +} diff --git a/services/core/internal/persistence/postgres/pgunit/ids.go b/services/core/internal/persistence/postgres/pgunit/ids.go new file mode 100644 index 000000000..af0d9feb5 --- /dev/null +++ b/services/core/internal/persistence/postgres/pgunit/ids.go @@ -0,0 +1,50 @@ +package pgunit + +import ( + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +// ErrInvalidID reports a value that is not a nonzero UUID. Adapters translate +// it into their domain's invalid-input error. +var ErrInvalidID = errors.New("nonzero UUID required") + +// unknownID never names a stored resource: Core assigns version 4 or 5 UUIDs, +// and the maximum UUID is neither. +var unknownID = uuid.Max + +// ParseID parses a resource identifier that must name a resource, such as a +// request-body reference. A value that is not a nonzero UUID is ErrInvalidID. +func ParseID(value string) (pgtype.UUID, error) { + id, err := uuid.Parse(value) + if err != nil || id == uuid.Nil { + return pgtype.UUID{}, ErrInvalidID + } + return pgtype.UUID{Bytes: id, Valid: true}, nil +} + +// PathID parses a caller-supplied resource path identifier. A value that cannot +// name a resource resolves to an identifier that never exists, so the request +// follows exactly the path of a well-formed missing identifier, including +// validation order. Request-body references use ParseID; cursors use +// LookupCursor. +func PathID(value string) pgtype.UUID { + id, err := ParseID(value) + if err != nil { + return pgtype.UUID{Bytes: unknownID, Valid: true} + } + return id +} + +// LookupCursor resolves the cursor of a list whose unresolved cursor is a 404 +// (Agents, Sessions, Turns, Templates, Vaults and Credentials) like a path +// identifier: a value that cannot name a resource becomes an identifier that +// never exists, so the list follows exactly the missing-cursor path. +func LookupCursor(value string) string { + if _, err := ParseID(value); err != nil { + return unknownID.String() + } + return value +} diff --git a/services/core/internal/persistence/postgres/pgunit/ids_test.go b/services/core/internal/persistence/postgres/pgunit/ids_test.go new file mode 100644 index 000000000..6b54debb1 --- /dev/null +++ b/services/core/internal/persistence/postgres/pgunit/ids_test.go @@ -0,0 +1,37 @@ +package pgunit + +import ( + "errors" + "strings" + "testing" + + "github.com/google/uuid" +) + +func TestIdentifierParsing(t *testing.T) { + valid := uuid.NewString() + unknown := uuid.Max.String() + for _, tc := range []struct { + value string + parsed bool + path string + cursor string + }{ + {valid, true, valid, valid}, + {strings.ToUpper(valid), true, valid, strings.ToUpper(valid)}, + {"", false, unknown, unknown}, + {"not-a-uuid", false, unknown, unknown}, + {uuid.Nil.String(), false, unknown, unknown}, + } { + id, err := ParseID(tc.value) + if tc.parsed != (err == nil) || !tc.parsed && !errors.Is(err, ErrInvalidID) || tc.parsed && uuid.UUID(id.Bytes).String() != tc.path { + t.Errorf("ParseID(%q) = %v, %v", tc.value, id, err) + } + if path := PathID(tc.value); !path.Valid || uuid.UUID(path.Bytes).String() != tc.path { + t.Errorf("PathID(%q) = %v", tc.value, path) + } + if cursor := LookupCursor(tc.value); cursor != tc.cursor { + t.Errorf("LookupCursor(%q) = %q", tc.value, cursor) + } + } +} diff --git a/services/core/internal/persistence/postgres/pgunit/pgunit.go b/services/core/internal/persistence/postgres/pgunit/pgunit.go index c33c84d0e..1bd064a11 100644 --- a/services/core/internal/persistence/postgres/pgunit/pgunit.go +++ b/services/core/internal/persistence/postgres/pgunit/pgunit.go @@ -8,6 +8,8 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" ) // Transactions state their isolation instead of inheriting the server default: @@ -22,6 +24,13 @@ type Pool struct{ pool *pgxpool.Pool } func NewPool(pool *pgxpool.Pool) *Pool { return &Pool{pool: pool} } +// Queries returns queries bound to the pool, outside any transaction. Use it +// only for a read that is a single statement and needs no transaction, such as +// a per-request key lookup, where Snapshot would add BEGIN and COMMIT round +// trips. A read of several statements uses Snapshot, and a write uses +// Transaction. +func (p *Pool) Queries() *sqlc.Queries { return sqlc.New(p.pool) } + // Transaction runs apply in a read committed transaction and commits only when // apply returns nil. func (p *Pool) Transaction(ctx context.Context, apply func(context.Context, pgx.Tx) error) error { diff --git a/services/core/internal/persistence/postgres/pgunit/pgunit_test.go b/services/core/internal/persistence/postgres/pgunit/pgunit_test.go new file mode 100644 index 000000000..83e9322fa --- /dev/null +++ b/services/core/internal/persistence/postgres/pgunit/pgunit_test.go @@ -0,0 +1,34 @@ +package pgunit + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" +) + +// Pool-bound queries read committed rows outside any transaction. +func TestPoolQueriesReadOutsideTransactions(t *testing.T) { + pool := NewPool(pgtest.Open(t)) + tenant := pgtype.UUID{Bytes: uuid.New(), Valid: true} + id := pgtype.UUID{Bytes: uuid.New(), Valid: true} + cursor := sqlc.GetWriteAuditCursorParams{TenantID: tenant, ID: id} + if _, err := pool.Queries().GetWriteAuditCursor(t.Context(), cursor); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("missing row: %v", err) + } + if err := pool.Transaction(t.Context(), func(ctx context.Context, tx pgx.Tx) error { + _, err := sqlc.New(tx).InsertWriteAuditOperation(ctx, sqlc.InsertWriteAuditOperationParams{ID: id, TenantID: tenant, KeyID: "key", KeyName: "key", KeyPrefix: "key", KeyKind: "issued", Action: "create", ResourceType: "agent", ResourceID: "agent", RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + return err + }); err != nil { + t.Fatal(err) + } + if created, err := pool.Queries().GetWriteAuditCursor(t.Context(), cursor); err != nil || !created.Valid { + t.Fatalf("committed row: %v %v", created, err) + } +} diff --git a/services/core/internal/persistence/postgres/pgunit/text.go b/services/core/internal/persistence/postgres/pgunit/text.go new file mode 100644 index 000000000..c1c6fad17 --- /dev/null +++ b/services/core/internal/persistence/postgres/pgunit/text.go @@ -0,0 +1,17 @@ +package pgunit + +import ( + "errors" + + "github.com/jackc/pgx/v5/pgconn" +) + +// IsUnstorableText reports PostgreSQL rejecting client text it cannot +// represent: U+0000 or invalid UTF-8 in a text parameter (22021), or a jsonb +// \u0000 escape (22P05). The rejected statement stores nothing, and writes +// sharing its transaction roll back. Adapters return textvalue.ErrUnstorable in +// its place and never keep the database error in the chain. +func IsUnstorableText(err error) bool { + var databaseError *pgconn.PgError + return errors.As(err, &databaseError) && (databaseError.Code == "22021" || databaseError.Code == "22P05") +} diff --git a/services/core/internal/persistence/postgres/pgunit/text_test.go b/services/core/internal/persistence/postgres/pgunit/text_test.go new file mode 100644 index 000000000..227f40856 --- /dev/null +++ b/services/core/internal/persistence/postgres/pgunit/text_test.go @@ -0,0 +1,26 @@ +package pgunit + +import ( + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestIsUnstorableText(t *testing.T) { + for _, tc := range []struct { + err error + want bool + }{ + {&pgconn.PgError{Code: "22021"}, true}, + {fmt.Errorf("insert: %w", &pgconn.PgError{Code: "22P05"}), true}, + {&pgconn.PgError{Code: "23505"}, false}, + {errors.New("22021"), false}, + {nil, false}, + } { + if got := IsUnstorableText(tc.err); got != tc.want { + t.Errorf("IsUnstorableText(%v) = %v", tc.err, got) + } + } +} diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index d0825d2a7..3ba60af6e 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -105,8 +105,9 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { h, err := api.NewHandler(api.Dependencies{ Engine: "codex", CoreKeys: auth, InstallationBindings: s, Projects: s, Vaults: s, ModelProviders: s, Files: s, Skills: s, EnvironmentTemplates: s, Agents: s, Sessions: s, SessionEvents: s, SessionHistory: s, Subagents: s, Artifacts: s, - SessionAdmin: s, Environments: s, Admin: s, WriteAudit: s, ExecutorConnections: struct{ api.ExecutorConnections }{}, - Metrics: struct{ api.Metrics }{}, RuntimeObservations: struct{ api.RuntimeObservations }{}, RuntimeHistory: struct{ api.RuntimeHistory }{}, + SessionAdmin: s, Environments: s, Admin: s, AdminAudit: struct{ api.AdminAudit }{}, WriteAudit: struct{ api.WriteAudit }{}, + ExecutorConnections: struct{ api.ExecutorConnections }{}, + Metrics: struct{ api.Metrics }{}, RuntimeObservations: struct{ api.RuntimeObservations }{}, RuntimeHistory: struct{ api.RuntimeHistory }{}, Execution: &api.Execution{ExecutorURL: "wss://core.example/api/v1/agent-daemon/ws", Admission: s, SessionArchive: s, Workspaces: struct{ api.EnvironmentWorkspaces }{}}, Sandboxes: &api.Sandboxes{Deployment: s, DeploymentChanges: leaseSetup{t: t, store: w, installation: installation}, ConfigurationDiscovery: struct{ api.ConfigurationDiscovery }{}}, }) diff --git a/services/core/internal/store/admin_audit.go b/services/core/internal/store/admin_audit.go deleted file mode 100644 index bc4b08acb..000000000 --- a/services/core/internal/store/admin_audit.go +++ /dev/null @@ -1,41 +0,0 @@ -package store - -import ( - "context" - "fmt" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" -) - -// recordAdminMutation runs in the business transaction and never records request bodies or secrets. -func recordAdminMutation(ctx context.Context, q *sqlc.Queries, tenant, action, resourceType, resourceID string) error { - source, ok := adminaudit.FromContext(ctx) - if !ok || !auditText(source.CredentialID, 64, true) || !auditText(source.ActorLabel, 128, false) || !auditText(source.RequestID, 128, true) || !auditText(source.TraceID, 128, true) || !auditText(source.ProjectID, 128, true) || !auditText(action, 64, true) || !auditText(resourceType, 64, true) || !auditText(resourceID, 256, true) { - return fmt.Errorf("%w: invalid administrator audit source", ErrInvalidInput) - } - projectID, err := parseID(source.ProjectID) - if err != nil { - return err - } - tenantID, err := parseID(tenant) - if err != nil { - return err - } - // result_ids is retained for historical copy mappings; current writes record none. - _, err = q.InsertAdminAudit(ctx, sqlc.InsertAdminAuditParams{ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, TenantID: tenantID, AdminCredentialID: source.CredentialID, ActorLabel: source.ActorLabel, Action: action, ProjectID: projectID, ResourceType: resourceType, ResourceID: resourceID, ResultIds: []byte(`[]`), RequestID: source.RequestID, TraceID: source.TraceID}) - return err -} - -// recordDeploymentMutation audits a deployment-wide write, which has no Project -// or tenant, in the business transaction. -func recordDeploymentMutation(ctx context.Context, q *sqlc.Queries, action, resourceType, resourceID string) error { - source, ok := adminaudit.FromContext(ctx) - if !ok || source.ProjectID != "" || !auditText(source.CredentialID, 64, true) || !auditText(source.ActorLabel, 128, false) || !auditText(source.RequestID, 128, true) || !auditText(source.TraceID, 128, true) || !auditText(action, 64, true) || !auditText(resourceType, 64, true) || !auditText(resourceID, 256, true) { - return fmt.Errorf("%w: invalid administrator audit source", ErrInvalidInput) - } - _, err := q.InsertAdminAudit(ctx, sqlc.InsertAdminAuditParams{ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, AdminCredentialID: source.CredentialID, ActorLabel: source.ActorLabel, Action: action, ResourceType: resourceType, ResourceID: resourceID, ResultIds: []byte(`[]`), RequestID: source.RequestID, TraceID: source.TraceID}) - return err -} diff --git a/services/core/internal/store/admin_history.go b/services/core/internal/store/admin_history.go deleted file mode 100644 index e98f0ca2e..000000000 --- a/services/core/internal/store/admin_history.go +++ /dev/null @@ -1,110 +0,0 @@ -package store - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -type AdminAuditFilter struct { - ProjectID, ResourceType, ResourceID, Action, After string - CreatedAfter, CreatedBefore *time.Time - Limit int -} - -// AdminAuditOperation is one administrator write. ProjectID is null for -// deployment-wide writes, such as deployment default model providers. -type AdminAuditOperation struct { - ID string `json:"id"` - CreatedAt time.Time `json:"created_at"` - AdminCredentialID string `json:"admin_credential_id"` - ActorLabel string `json:"actor_label"` - Action string `json:"action"` - ProjectID *string `json:"project_id" extensions:"x-nullable"` - ResourceType string `json:"resource_type"` - ResourceID string `json:"resource_id"` - ResultIDs json.RawMessage `json:"result_ids" swaggertype:"array,object"` - RequestID string `json:"request_id"` - TraceID string `json:"trace_id"` -} -type AdminAuditPage struct { - Data []AdminAuditOperation `json:"data"` - HasMore bool `json:"has_more"` - NextCursor string `json:"next_cursor"` -} - -func (s *Store) ListAdminAudit(ctx context.Context, filter AdminAuditFilter) (AdminAuditPage, error) { - page := AdminAuditPage{Data: []AdminAuditOperation{}} - if filter.Limit == 0 { - filter.Limit = 50 - } - if filter.Limit < 1 || filter.Limit > 100 || !auditText(filter.ProjectID, 128, false) || !auditText(filter.ResourceType, 64, false) || !auditText(filter.ResourceID, 256, false) || !auditText(filter.Action, 64, false) || filter.CreatedAfter != nil && filter.CreatedBefore != nil && !filter.CreatedAfter.Before(*filter.CreatedBefore) { - return page, ErrInvalidInput - } - normalized := filter - normalized.After = "" - normalized.Limit = 0 - if normalized.CreatedAfter != nil { - v := normalized.CreatedAfter.UTC() - normalized.CreatedAfter = &v - } - if normalized.CreatedBefore != nil { - v := normalized.CreatedBefore.UTC() - normalized.CreatedBefore = &v - } - raw, _ := json.Marshal(normalized) - digest := sha256.Sum256(raw) - scope := hex.EncodeToString(digest[:]) - params := sqlc.ListAdminAuditLogParams{ProjectID: filter.ProjectID, ResourceType: filter.ResourceType, ResourceID: filter.ResourceID, Action: filter.Action, CreatedAfter: auditTimestamp(filter.CreatedAfter), CreatedBefore: auditTimestamp(filter.CreatedBefore), AfterID: pgtype.UUID{Valid: true}, PageLimit: int32(filter.Limit + 1)} - if filter.After != "" { - raw, err := base64.RawURLEncoding.DecodeString(filter.After) - var cursor writeAuditCursor - if len(filter.After) > 1024 || err != nil || json.Unmarshal(raw, &cursor) != nil || cursor.Scope != scope { - return page, ErrInvalidInput - } - params.AfterID, err = parseID(cursor.ID) - if err != nil { - return page, ErrInvalidInput - } - params.AfterTime, err = s.queries.AdminAuditCursor(ctx, params.AfterID) - if errors.Is(err, pgx.ErrNoRows) { - return page, ErrInvalidInput - } - if err != nil { - return page, err - } - } - rows, err := s.queries.ListAdminAuditLog(ctx, params) - if err != nil { - return page, err - } - page.HasMore = len(rows) > filter.Limit - if page.HasMore { - rows = rows[:filter.Limit] - } - for _, row := range rows { - page.Data = append(page.Data, AdminAuditOperation{ID: uuid.UUID(row.ID.Bytes).String(), CreatedAt: row.CreatedAt.Time, AdminCredentialID: row.AdminCredentialID, ActorLabel: row.ActorLabel, Action: row.Action, ProjectID: auditProjectID(row.ProjectID), ResourceType: row.ResourceType, ResourceID: row.ResourceID, ResultIDs: row.ResultIds, RequestID: row.RequestID, TraceID: row.TraceID}) - } - if page.HasMore { - raw, _ := json.Marshal(writeAuditCursor{ID: page.Data[len(page.Data)-1].ID, Scope: scope}) - page.NextCursor = base64.RawURLEncoding.EncodeToString(raw) - } - return page, nil -} - -func auditProjectID(id pgtype.UUID) *string { - if !id.Valid { - return nil - } - value := uuid.UUID(id.Bytes).String() - return &value -} diff --git a/services/core/internal/store/admin_session_archive.go b/services/core/internal/store/admin_session_archive.go index e323ec9d6..08695dceb 100644 --- a/services/core/internal/store/admin_session_archive.go +++ b/services/core/internal/store/admin_session_archive.go @@ -3,9 +3,12 @@ package store import ( "context" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "time" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -127,7 +130,7 @@ func (s *Store) archiveManagedSession(ctx context.Context, tenantID, sessionID s return err } } - if err := recordAdminMutation(ctx, q, tenantID, "archive", "session", runtimeUUID(session)); err != nil { + if err := auditpg.RecordAdminMutation(ctx, q, tenantID, "archive", "session", runtimeUUID(session)); err != nil { return err } result, err = getManagedSessionArchive(ctx, q, tenant, session) @@ -142,7 +145,7 @@ func (s *Store) GetManagedSessionArchive(ctx context.Context, tenantID, sessionI if err != nil { return ManagedSessionArchive{}, err } - return getManagedSessionArchive(ctx, s.queries, tenant, parsePathID(sessionID)) + return getManagedSessionArchive(ctx, s.queries, tenant, pgunit.PathID(sessionID)) } func getManagedSessionArchive(ctx context.Context, q *sqlc.Queries, tenant, session pgtype.UUID) (ManagedSessionArchive, error) { diff --git a/services/core/internal/store/admin_summary.go b/services/core/internal/store/admin_summary.go index 2ed0ff00a..5f24af4a7 100644 --- a/services/core/internal/store/admin_summary.go +++ b/services/core/internal/store/admin_summary.go @@ -40,7 +40,7 @@ func (s *Store) ReadAdminSummary(ctx context.Context, tenantID string, filter Ad return err } counts = AdminAssetCounts{Agents: raw.Agents, Skills: raw.Skills, EnvironmentTemplates: raw.EnvironmentTemplates, Files: raw.Files, Vaults: raw.Vaults, Credentials: raw.Credentials} - params := sqlc.AdminSummarySessionsParams{TenantID: tenant, CreatedAfter: auditTimestamp(filter.CreatedAfter), CreatedBefore: auditTimestamp(filter.CreatedBefore), AfterID: pgtype.UUID{Valid: true}} + params := sqlc.AdminSummarySessionsParams{TenantID: tenant, CreatedAfter: summaryTimestamp(filter.CreatedAfter), CreatedBefore: summaryTimestamp(filter.CreatedBefore), AfterID: pgtype.UUID{Valid: true}} for { rows, err := q.AdminSummarySessions(ctx, params) if err != nil { @@ -120,3 +120,10 @@ func (s *Store) ListAdminRuntimeTargets(ctx context.Context, tenantIDs []string, } return page, nil } + +func summaryTimestamp(value *time.Time) pgtype.Timestamptz { + if value == nil { + return pgtype.Timestamptz{} + } + return pgtype.Timestamptz{Time: *value, Valid: true} +} diff --git a/services/core/internal/store/agent_model_execution.go b/services/core/internal/store/agent_model_execution.go index 371b679b9..5a6cb0567 100644 --- a/services/core/internal/store/agent_model_execution.go +++ b/services/core/internal/store/agent_model_execution.go @@ -7,6 +7,7 @@ import ( "fmt" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -23,7 +24,7 @@ func (s *Store) saveAgentModelExecution(ctx context.Context, q *sqlc.Queries, te } encrypted, err := s.credentialCipher.SealAgentModelExecution(raw, tenant, uuid.UUID(agent.Bytes).String()) if err != nil { - return ErrCredentialStorageUnavailable + return credentialcrypto.ErrUnavailable } return q.SaveAgentModelExecution(ctx, sqlc.SaveAgentModelExecutionParams{AgentID: agent, EncryptedConfig: encrypted}) } @@ -61,11 +62,11 @@ func (s *Store) GetAgentForSession(ctx context.Context, tenantID, agentID string } raw, err := s.credentialCipher.OpenAgentModelExecution(row.EncryptedConfig, uuid.UUID(tenant.Bytes).String(), uuid.UUID(id.Bytes).String()) if err != nil { - return SavedAgent{}, nil, ErrCredentialStorageUnavailable + return SavedAgent{}, nil, credentialcrypto.ErrUnavailable } var provider v1.ModelProviderInput if json.Unmarshal(raw, &provider) != nil || provider.Validate() != nil { - return SavedAgent{}, nil, ErrCredentialStorageUnavailable + return SavedAgent{}, nil, credentialcrypto.ErrUnavailable } return agent, &provider, nil } diff --git a/services/core/internal/store/agent_model_execution_test.go b/services/core/internal/store/agent_model_execution_test.go index 7fee1280b..a6d0dfcf0 100644 --- a/services/core/internal/store/agent_model_execution_test.go +++ b/services/core/internal/store/agent_model_execution_test.go @@ -72,17 +72,17 @@ func TestAgentModelExecutionAtomicEncryptedSnapshot(t *testing.T) { if _, _, err := withoutKey.GetAgentForSession(ctx, tenant, agent.ID, false); err != nil { t.Fatal("explicit override required Agent decryption", err) } - if _, _, err := withoutKey.GetAgentForSession(ctx, tenant, agent.ID, true); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, _, err := withoutKey.GetAgentForSession(ctx, tenant, agent.ID, true); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("missing cipher accepted", err) } - if _, err := withoutKey.CreateAgent(ctx, tenant, input); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := withoutKey.CreateAgent(ctx, tenant, input); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("unencrypted Agent create accepted", err) } replacement := agentProviderFixture(1) if _, err := s.UpdateAgent(ctx, uuid.NewString(), agent.ID, UpdateAgentInput{Configuration: agentProviderConfiguration(t, replacement, "codex"), ModelProvider: replacement, ModelProviderSet: true}); !errors.Is(err, ErrNotFound) { t.Fatal("foreign tenant replacement accepted", err) } - if _, err := withoutKey.UpdateAgent(ctx, tenant, agent.ID, UpdateAgentInput{Configuration: agentProviderConfiguration(t, replacement, "codex"), ModelProvider: replacement, ModelProviderSet: true}); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := withoutKey.UpdateAgent(ctx, tenant, agent.ID, UpdateAgentInput{Configuration: agentProviderConfiguration(t, replacement, "codex"), ModelProvider: replacement, ModelProviderSet: true}); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("unencrypted replacement accepted", err) } current, inherited, err := s.GetAgentForSession(ctx, tenant, agent.ID, true) diff --git a/services/core/internal/store/agents.go b/services/core/internal/store/agents.go index 9713cc98b..46419c728 100644 --- a/services/core/internal/store/agents.go +++ b/services/core/internal/store/agents.go @@ -15,6 +15,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/metadata" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) // SavedAgent is reusable configuration owned by an execution tenant. It has no @@ -73,7 +75,7 @@ func (s *Store) CreateAgent(ctx context.Context, tenantID string, input CreateAg if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "create", "agent", created.ID, "", AuditResource{Type: "agent", ID: created.ID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "agent", created.ID, "", writeaudit.Resource{Type: "agent", ID: created.ID}) }) if err != nil { return SavedAgent{}, fmt.Errorf("create agent: %w", err) diff --git a/services/core/internal/store/agents_delete.go b/services/core/internal/store/agents_delete.go index 081df8359..1d0cb6050 100644 --- a/services/core/internal/store/agents_delete.go +++ b/services/core/internal/store/agents_delete.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -28,7 +29,7 @@ func (s *Store) DeleteAgent(ctx context.Context, tenantID, agentID string) (stri return err } deletedID = uuid.UUID(deleted.Bytes).String() - return recordWriteAudit(ctx, q, tenantID, "delete", "agent", deletedID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "agent", deletedID, "") }) if errors.Is(err, pgx.ErrNoRows) { return "", ErrNotFound diff --git a/services/core/internal/store/agents_list.go b/services/core/internal/store/agents_list.go index 70c443a77..f8c59e252 100644 --- a/services/core/internal/store/agents_list.go +++ b/services/core/internal/store/agents_list.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -24,7 +25,7 @@ func (s *Store) ListAgents(ctx context.Context, tenantID, cursor string, limit i } params := sqlc.ListAgentsParams{TenantID: tenant, PageLimit: int32(limit + 1), AfterID: pgtype.UUID{Valid: true}, Ascending: ascending} if cursor != "" { - after, err := s.GetAgent(ctx, tenantID, lookupCursor(cursor)) + after, err := s.GetAgent(ctx, tenantID, pgunit.LookupCursor(cursor)) if err != nil { return AgentPage{}, err } diff --git a/services/core/internal/store/agents_update.go b/services/core/internal/store/agents_update.go index f748d9601..da49eb654 100644 --- a/services/core/internal/store/agents_update.go +++ b/services/core/internal/store/agents_update.go @@ -10,6 +10,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/metadata" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -30,10 +32,7 @@ func (s *Store) UpdateAgent(ctx context.Context, tenantID, agentID string, input if err != nil { return SavedAgent{}, err } - id, err := parseID(agentID) - if err != nil { - return SavedAgent{}, err - } + id := pgunit.PathID(agentID) raw := input.Configuration if len(raw) == 0 { raw = json.RawMessage(`{}`) @@ -103,7 +102,7 @@ func (s *Store) UpdateAgent(ctx context.Context, tenantID, agentID string, input if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "update", "agent", updated.ID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "update", "agent", updated.ID, "") }) if err != nil { return SavedAgent{}, fmt.Errorf("update agent: %w", err) diff --git a/services/core/internal/store/credential_oauth_secret.go b/services/core/internal/store/credential_oauth_secret.go index aaa1f0c3c..943b14eb5 100644 --- a/services/core/internal/store/credential_oauth_secret.go +++ b/services/core/internal/store/credential_oauth_secret.go @@ -45,7 +45,7 @@ func oauthBinding(tenantID string, credential Credential) credentialcrypto.Bindi func (s *Store) sealOAuth(tenantID string, credential Credential, secret oauthSecret) ([]byte, []byte, error) { if s.credentialCipher == nil { - return nil, nil, ErrCredentialStorageUnavailable + return nil, nil, credentialcrypto.ErrUnavailable } if !validOAuthMetadata(secret.Metadata) { return nil, nil, ErrInvalidInput @@ -67,7 +67,7 @@ func (s *Store) sealOAuth(tenantID string, credential Credential, secret oauthSe func (s *Store) openOAuth(tenantID string, credential Credential, ciphertext []byte) (oauthSecret, error) { if s.credentialCipher == nil { - return oauthSecret{}, ErrCredentialStorageUnavailable + return oauthSecret{}, credentialcrypto.ErrUnavailable } plaintext, err := s.credentialCipher.Open(ciphertext, oauthBinding(tenantID, credential)) if err != nil { diff --git a/services/core/internal/store/deployment_model_providers.go b/services/core/internal/store/deployment_model_providers.go index da8bae0e5..32f38518a 100644 --- a/services/core/internal/store/deployment_model_providers.go +++ b/services/core/internal/store/deployment_model_providers.go @@ -8,7 +8,9 @@ import ( "time" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -67,7 +69,7 @@ func (s *Store) SetDeploymentModelProvider(ctx context.Context, harness string, } encrypted, err := s.credentialCipher.SealDeploymentModelProvider(raw, harness) if err != nil { - return DeploymentModelProvider{}, ErrCredentialStorageUnavailable + return DeploymentModelProvider{}, credentialcrypto.ErrUnavailable } var result DeploymentModelProvider err = s.pooled.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { @@ -80,7 +82,7 @@ func (s *Store) SetDeploymentModelProvider(ctx context.Context, harness string, return err } result = deploymentModelProvider(row.Harness, row.Protocol, row.BaseUrl, row.ContextWindow, row.MaxOutputTokens, row.Model, row.HarnessConfig, row.UpdatedAt.Time, row.LastUsedAt, row.LastErrorAt, row.LastErrorCode) - return recordDeploymentMutation(ctx, q, "set", "deployment_model_provider", harness) + return auditpg.RecordDeploymentMutation(ctx, q, "set", "deployment_model_provider", harness) }) return result, err } @@ -93,7 +95,7 @@ func (s *Store) DeleteDeploymentModelProvider(ctx context.Context, harness strin if _, err := q.DeleteDeploymentModelProvider(ctx, harness); err != nil { return err } - return recordDeploymentMutation(ctx, q, "delete", "deployment_model_provider", harness) + return auditpg.RecordDeploymentMutation(ctx, q, "delete", "deployment_model_provider", harness) }) } @@ -110,11 +112,11 @@ func (s *Store) DeploymentModelProvider(ctx context.Context, harness string) (*D } raw, err := s.credentialCipher.OpenDeploymentModelProvider(snapshot.EncryptedConfig, harness) if err != nil { - return nil, ErrCredentialStorageUnavailable + return nil, credentialcrypto.ErrUnavailable } var configuration v1.ModelConfigurationInput if json.Unmarshal(raw, &configuration) != nil { - return nil, ErrCredentialStorageUnavailable + return nil, credentialcrypto.ErrUnavailable } // A decrypted but unsupported configuration is not a credential failure. // Keep its stored snapshot intact so the operator can inspect and replace it. diff --git a/services/core/internal/store/deployment_model_providers_http_test.go b/services/core/internal/store/deployment_model_providers_http_test.go index 182325d90..993508dc6 100644 --- a/services/core/internal/store/deployment_model_providers_http_test.go +++ b/services/core/internal/store/deployment_model_providers_http_test.go @@ -15,6 +15,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" "github.com/google/uuid" @@ -218,7 +220,7 @@ func TestDeploymentModelProvidersHTTP(t *testing.T) { if providerOf(hostedID) != "deployment-canary" { t.Fatal("removing the default changed an existing Session") } - page, err := st.ListAdminAudit(t.Context(), store.AdminAuditFilter{ResourceType: "deployment_model_provider", ResourceID: "codex"}) + page, err := auditpg.New(pgunit.NewPool(pool)).ListAdminAudit(t.Context(), adminaudit.Filter{ResourceType: "deployment_model_provider", ResourceID: "codex"}) if err != nil || len(page.Data) < 4 || page.Data[0].Action != "delete" || page.Data[0].ProjectID != nil { t.Fatal("deployment writes not audited", page, err) } diff --git a/services/core/internal/store/deployment_model_providers_test.go b/services/core/internal/store/deployment_model_providers_test.go index 62542d505..3251c7eb1 100644 --- a/services/core/internal/store/deployment_model_providers_test.go +++ b/services/core/internal/store/deployment_model_providers_test.go @@ -10,6 +10,8 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" ) @@ -30,7 +32,7 @@ func TestDeploymentModelProviderEncryptedAuditedAndReplaced(t *testing.T) { if _, err := s.SetDeploymentModelProvider(ctx, "codex", v1.ModelConfigurationInput{ModelProvider: v1.ModelProviderInput{Protocol: "unknown", BaseURL: provider.BaseURL, APIKey: "k"}, Model: "fixture"}); !errors.Is(err, ErrInvalidInput) { t.Fatal("unknown upstream protocol accepted", err) } - if _, err := New(pool).SetDeploymentModelProvider(ctx, "codex", v1.ModelConfigurationInput{ModelProvider: provider, Model: "fixture"}); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := New(pool).SetDeploymentModelProvider(ctx, "codex", v1.ModelConfigurationInput{ModelProvider: provider, Model: "fixture"}); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("key stored without encryption", err) } saved, err := s.SetDeploymentModelProvider(ctx, "codex", v1.ModelConfigurationInput{ModelProvider: provider, Model: "fixture"}) @@ -52,7 +54,7 @@ func TestDeploymentModelProviderEncryptedAuditedAndReplaced(t *testing.T) { t.Fatal("unset harness returned a default", err) } other, _ := credentialcrypto.New(bytes.Repeat([]byte{48}, 32)) - if _, err := NewWithCredentialCipher(pool, other).DeploymentModelProvider(ctx, "codex"); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := NewWithCredentialCipher(pool, other).DeploymentModelProvider(ctx, "codex"); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("wrong encryption key did not fail closed", err) } replacement := v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://replacement.example/v1", APIKey: "replacement-key"} @@ -71,7 +73,7 @@ func TestDeploymentModelProviderEncryptedAuditedAndReplaced(t *testing.T) { if got, err := s.DeploymentModelProvider(ctx, "codex"); err != nil || got != nil { t.Fatal("deleted default remained", err) } - page, err := s.ListAdminAudit(ctx, AdminAuditFilter{ResourceType: "deployment_model_provider", CreatedAfter: &started}) + page, err := auditpg.New(pgunit.NewPool(pool)).ListAdminAudit(ctx, adminaudit.Filter{ResourceType: "deployment_model_provider", CreatedAfter: &started}) if err != nil || len(page.Data) != 4 { t.Fatal("deployment writes not audited", len(page.Data), err) } @@ -80,10 +82,10 @@ func TestDeploymentModelProviderEncryptedAuditedAndReplaced(t *testing.T) { t.Fatal("unexpected deployment audit entry", entry) } } - if scoped, err := s.ListAdminAudit(ctx, AdminAuditFilter{ProjectID: "00000000-0000-4000-8000-000000000001"}); err != nil || len(scoped.Data) != 0 { + if scoped, err := auditpg.New(pgunit.NewPool(pool)).ListAdminAudit(ctx, adminaudit.Filter{ProjectID: "00000000-0000-4000-8000-000000000001"}); err != nil || len(scoped.Data) != 0 { t.Fatal("Project filter returned deployment entries", err) } - if _, err := s.SetDeploymentModelProvider(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "abcd1234", RequestID: "r", TraceID: "t", ProjectID: "00000000-0000-4000-8000-000000000001"}), "codex", v1.ModelConfigurationInput{ModelProvider: provider, Model: "fixture"}); !errors.Is(err, ErrInvalidInput) { + if _, err := s.SetDeploymentModelProvider(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "abcd1234", RequestID: "r", TraceID: "t", ProjectID: "00000000-0000-4000-8000-000000000001"}), "codex", v1.ModelConfigurationInput{ModelProvider: provider, Model: "fixture"}); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("Project-scoped audit source accepted for a deployment write", err) } if got, _ := s.DeploymentModelProvider(ctx, "codex"); got != nil { diff --git a/services/core/internal/store/environment_executor_management.go b/services/core/internal/store/environment_executor_management.go index 278135247..e2dfc399e 100644 --- a/services/core/internal/store/environment_executor_management.go +++ b/services/core/internal/store/environment_executor_management.go @@ -8,6 +8,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -58,7 +60,7 @@ func (s *Store) ProjectExecutorCredentialState(ctx context.Context, project iden if err != nil { return ExecutorCredentialState{}, err } - environmentID := parsePathID(environment) + environmentID := pgunit.PathID(environment) result := ExecutorCredentialState{Credentials: []ExecutorCredential{}} err = s.pooled.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { q := s.queries.WithTx(tx) @@ -159,7 +161,7 @@ func (s *Store) RevokeProjectExecutorCredential(ctx context.Context, project ide func executorCredentialAudit(project identity.Principal, action, keyID string) func(context.Context, *sqlc.Queries) error { return func(ctx context.Context, q *sqlc.Queries) error { - return recordAdminMutation(ctx, q, project.TenantID, action, "executor_credential", keyID) + return auditpg.RecordAdminMutation(ctx, q, project.TenantID, action, "executor_credential", keyID) } } @@ -217,7 +219,7 @@ func (s *Store) exactExecutorRestriction(ctx context.Context, principal identity if err != nil { return err } - want := parsePathID(environment) + want := pgunit.PathID(environment) actual, err := s.executorCredentialRestriction(ctx, principal, tenant, id) if err != nil { return err diff --git a/services/core/internal/store/environment_executor_management_test.go b/services/core/internal/store/environment_executor_management_test.go index 645df6d90..07d54d68d 100644 --- a/services/core/internal/store/environment_executor_management_test.go +++ b/services/core/internal/store/environment_executor_management_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/google/uuid" ) @@ -46,7 +47,7 @@ func TestProjectEnvironmentExecutorManagement(t *testing.T) { keyID := uuid.NewString() // The audit entry commits with the write: without an audit source nothing is issued. - if _, err := s.IssueProjectExecutorCredential(ctx, p, one.ID, keyID, false); !errors.Is(err, ErrInvalidInput) { + if _, err := s.IssueProjectExecutorCredential(ctx, p, one.ID, keyID, false); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("unaudited issue", err) } issued, err := s.IssueProjectExecutorCredential(admin(), p, one.ID, keyID, false) diff --git a/services/core/internal/store/environment_file_writes.go b/services/core/internal/store/environment_file_writes.go index 4a389219f..ded255fa0 100644 --- a/services/core/internal/store/environment_file_writes.go +++ b/services/core/internal/store/environment_file_writes.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) @@ -61,7 +62,7 @@ func (s *Store) ReserveEnvironmentFileWrite(ctx context.Context, tenant, environ } var origin []byte if source, ok := writeaudit.FromContext(ctx); ok { - if err := validateWriteAuditSource(source, tenant); err != nil { + if err := source.Validate(tenant); err != nil { return EnvironmentFileWrite{}, err } origin, err = json.Marshal(source) @@ -186,7 +187,7 @@ func (s *Store) SettleEnvironmentFileWrite(ctx context.Context, tenant, environm return err } auditCtx := writeaudit.WithSource(ctx, source) - return recordWriteAudit(auditCtx, q, tenant, "upload_file", "environment", environment, uuid.UUID(session.Bytes).String()) + return auditpg.RecordWriteAudit(auditCtx, q, tenant, "upload_file", "environment", environment, uuid.UUID(session.Bytes).String()) } return nil }) diff --git a/services/core/internal/store/environment_inputs.go b/services/core/internal/store/environment_inputs.go index 7d2edd454..1d4e3625d 100644 --- a/services/core/internal/store/environment_inputs.go +++ b/services/core/internal/store/environment_inputs.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" ) const ( @@ -61,7 +62,7 @@ func (s *Store) ReserveEnvironmentInput(ctx context.Context, tenantID, sessionID var result EnvironmentInputReservation err = s.withEnvironmentInputSession(ctx, tenantID, sessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { audit := func() error { - return recordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") } previous, err := q.FindEnvironmentInputReservation(ctx, sqlc.FindEnvironmentInputReservationParams{ SessionID: session, IdempotencyKey: key, Batch: encoded, diff --git a/services/core/internal/store/environment_templates.go b/services/core/internal/store/environment_templates.go index 68f74aac7..530ec74ef 100644 --- a/services/core/internal/store/environment_templates.go +++ b/services/core/internal/store/environment_templates.go @@ -4,15 +4,19 @@ import ( "context" "encoding/json" "errors" - "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" - "github.com/MiniMax-AI/OpenAgentCore/internal/agentplugin" "time" "unicode/utf8" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentplugin" + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -113,7 +117,7 @@ func (s *Store) CreateEnvironmentTemplate(ctx context.Context, tenantID string, if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "create", "environment_template", result.ID, "", AuditResource{Type: "environment_template", ID: result.ID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "environment_template", result.ID, "", writeaudit.Resource{Type: "environment_template", ID: result.ID}) }) return result, err } @@ -141,7 +145,7 @@ func (s *Store) UpdateEnvironmentTemplate(ctx context.Context, tenantID, templat return EnvironmentTemplate{}, err } // Sealing runs before the update lookup, so a malformed ID must take the same path. - id := parsePathID(templateID) + id := pgunit.PathID(templateID) var name pgtype.Text if in.Name != nil { name = pgtype.Text{String: *in.Name, Valid: true} @@ -170,7 +174,7 @@ func (s *Store) UpdateEnvironmentTemplate(ctx context.Context, tenantID, templat if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "update", "environment_template", result.ID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "update", "environment_template", result.ID, "") }) return result, err } @@ -192,7 +196,7 @@ func (s *Store) DeleteEnvironmentTemplate(ctx context.Context, tenantID, templat return err } deletedID = uuid.UUID(result.Bytes).String() - return recordWriteAudit(ctx, q, tenantID, "delete", "environment_template", deletedID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "environment_template", deletedID, "") }) if errors.Is(err, pgx.ErrNoRows) { return "", ErrNotFound @@ -218,7 +222,7 @@ func (s *Store) ListEnvironmentTemplates(ctx context.Context, tenantID, cursor s } params := sqlc.ListEnvironmentTemplatesParams{TenantID: tenant, PageLimit: int32(limit + 1), AfterID: pgtype.UUID{Valid: true}, Ascending: ascending} if cursor != "" { - after, err := s.GetEnvironmentTemplate(ctx, tenantID, lookupCursor(cursor)) + after, err := s.GetEnvironmentTemplate(ctx, tenantID, pgunit.LookupCursor(cursor)) if err != nil { return EnvironmentTemplatePage{}, err } diff --git a/services/core/internal/store/environments.go b/services/core/internal/store/environments.go index d9b9f9bb0..68bb57f34 100644 --- a/services/core/internal/store/environments.go +++ b/services/core/internal/store/environments.go @@ -13,6 +13,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" ) // Environment retains execution ownership; its configuration is an internal snapshot, not a public response. @@ -53,7 +54,7 @@ func (s *Store) GetEnvironment(ctx context.Context, tenantID, environmentID stri if err != nil { return Environment{}, err } - id := parsePathID(environmentID) + id := pgunit.PathID(environmentID) row, err := s.queries.GetEnvironment(ctx, sqlc.GetEnvironmentParams{TenantID: tenant, ID: id}) return environmentFromRow(row.Environment, row.TenantID, row.Configuration, err) } diff --git a/services/core/internal/store/item_reads.go b/services/core/internal/store/item_reads.go index 2eee5cf67..d0c3f37e3 100644 --- a/services/core/internal/store/item_reads.go +++ b/services/core/internal/store/item_reads.go @@ -7,6 +7,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) @@ -26,7 +27,7 @@ func (s *Store) ListItems(ctx context.Context, tenantID, sessionID, cursor strin if cursor != "" { // Any cursor that is not an Item of this Session, including a // malformed one, is an invalid cursor rather than a missing resource. - row, err := q.GetSessionItem(ctx, sqlc.GetSessionItemParams{SessionID: session, ID: parsePathID(cursor)}) + row, err := q.GetSessionItem(ctx, sqlc.GetSessionItemParams{SessionID: session, ID: pgunit.PathID(cursor)}) if errors.Is(err, pgx.ErrNoRows) { return errItemCursor } diff --git a/services/core/internal/store/list_cursors.go b/services/core/internal/store/list_cursors.go index 6aad72f19..f3821fe34 100644 --- a/services/core/internal/store/list_cursors.go +++ b/services/core/internal/store/list_cursors.go @@ -47,14 +47,3 @@ func unresolvedCursor(err, cursor error) error { } return err } - -// lookupCursor resolves a cursor of a list whose unresolved cursor is a 404 -// (Agents, Sessions, Turns, Templates, Vaults and Credentials) like a path -// identifier: a value that cannot name a resource becomes UnknownResourceID, -// so the list follows exactly the missing-cursor path. -func lookupCursor(value string) string { - if _, err := parseID(value); err != nil { - return UnknownResourceID - } - return value -} diff --git a/services/core/internal/store/mcp_credential_selection_public_test.go b/services/core/internal/store/mcp_credential_selection_public_test.go index 39352c2c6..3ec19653a 100644 --- a/services/core/internal/store/mcp_credential_selection_public_test.go +++ b/services/core/internal/store/mcp_credential_selection_public_test.go @@ -202,7 +202,7 @@ func TestMCPCredentialSelectionPublicPostgres(t *testing.T) { {"M4 foreign tenant", tokenA, inline(tool("records", url, reference(credentialB)), vaults(attachedA)), 400, notAttached(credentialB)}, {"M4 unattached", tokenA, inline(tool("records", url, reference(unattachedA)), vaults(attachedA)), 400, notAttached(unattachedA)}, {"M4 unattached B", tokenB, inline(tool("records", url, reference(credentialB)), vaults(otherVaultB)), 400, notAttached(credentialB)}, - {"M4 missing", tokenA, inline(tool("records", url, reference(store.UnknownResourceID)), vaults(attachedA)), 400, notAttached(store.UnknownResourceID)}, + {"M4 missing", tokenA, inline(tool("records", url, reference(uuid.Max.String())), vaults(attachedA)), 400, notAttached(uuid.Max.String())}, {"M4 malformed", tokenA, inline(tool("records", url, reference("not-a-credential")), vaults(attachedA)), 400, notAttached("not-a-credential")}, {"M4 unbounded", tokenA, inline(tool("records", url, reference(long)), vaults(attachedA)), 400, invalid("MCP credential_id was not found in an attached vault")}, {"M4 unprintable", tokenA, inline(tool("records", url, reference("bad\x01id")), vaults(attachedA)), 400, invalid("MCP credential_id was not found in an attached vault")}, diff --git a/services/core/internal/store/mcp_credentials.go b/services/core/internal/store/mcp_credentials.go index f2209e14b..3e635f8a7 100644 --- a/services/core/internal/store/mcp_credentials.go +++ b/services/core/internal/store/mcp_credentials.go @@ -183,7 +183,7 @@ func (s *Store) MCPBearerToken(ctx context.Context, tenantID string, vaultIDs [] return "", errors.New("cannot read MCP credential") } if s.credentialCipher == nil { - return "", ErrCredentialStorageUnavailable + return "", credentialcrypto.ErrUnavailable } plaintext, err := s.credentialCipher.Open(ciphertext, credentialcrypto.Binding{ TenantID: uuid.UUID(tenant.Bytes).String(), VaultID: uuid.UUID(vault.Bytes).String(), CredentialID: uuid.UUID(id.Bytes).String(), diff --git a/services/core/internal/store/mcp_credentials_test.go b/services/core/internal/store/mcp_credentials_test.go index f7779139b..cb76b56e5 100644 --- a/services/core/internal/store/mcp_credentials_test.go +++ b/services/core/internal/store/mcp_credentials_test.go @@ -93,7 +93,7 @@ func TestMCPCredentialSelectionAndScopedDecryption(t *testing.T) { if err != nil || got != token { t.Fatal("frozen selection or opaque bytes changed across restart", err) } - if got, err := public.MCPBearerToken(t.Context(), tenant, attached, bindings[0]); !errors.Is(err, ErrCredentialStorageUnavailable) || got != "" { + if got, err := public.MCPBearerToken(t.Context(), tenant, attached, bindings[0]); !errors.Is(err, credentialcrypto.ErrUnavailable) || got != "" { t.Fatal("missing key did not fail execution closed") } key[0] ^= 1 diff --git a/services/core/internal/store/project_api_keys.go b/services/core/internal/store/project_api_keys.go index c4224b526..1d5dc0a0a 100644 --- a/services/core/internal/store/project_api_keys.go +++ b/services/core/internal/store/project_api_keys.go @@ -11,6 +11,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -38,10 +40,6 @@ type ProjectAPIKeyPage struct { HasMore bool `json:"has_more"` } -func validProjectKeyDigest(digest string) bool { - decoded, err := hex.DecodeString(digest) - return err == nil && len(decoded) == sha256.Size && hex.EncodeToString(decoded) == digest -} func projectKeyMetadata(row sqlc.ProjectApiKey) ProjectAPIKey { key := ProjectAPIKey{ID: uuid.UUID(row.ID.Bytes).String(), ProjectID: uuid.UUID(row.ProjectID.Bytes).String(), Name: row.Name, Prefix: row.Prefix, CreatedAt: row.CreatedAt.Time} if row.RevokedAt.Valid { @@ -97,7 +95,7 @@ func (s *Store) CreateProjectAPIKey(ctx context.Context, project, id, name strin return err } result = IssuedProjectAPIKey{ProjectAPIKey: projectKeyMetadata(row), Key: token} - return recordAdminMutation(ctx, q, uuid.UUID(p.TenantID.Bytes).String(), "create", "api_key", id) + return auditpg.RecordAdminMutation(ctx, q, uuid.UUID(p.TenantID.Bytes).String(), "create", "api_key", id) }) if err != nil { return IssuedProjectAPIKey{}, err @@ -163,11 +161,11 @@ func (s *Store) RevokeProjectAPIKey(ctx context.Context, project, id string) err if err != nil { return err } - return recordAdminMutation(ctx, q, uuid.UUID(p.TenantID.Bytes).String(), "revoke", "api_key", id) + return auditpg.RecordAdminMutation(ctx, q, uuid.UUID(p.TenantID.Bytes).String(), "revoke", "api_key", id) }) } func (s *Store) ResolveProjectAPIKey(ctx context.Context, digest string) (ProjectAPIKeyBinding, error) { - if !validProjectKeyDigest(digest) { + if !writeaudit.ValidKeyDigest(digest) { return ProjectAPIKeyBinding{}, ErrNotFound } row, err := s.queries.ResolveProjectAPIKey(ctx, digest) diff --git a/services/core/internal/store/project_api_keys_test.go b/services/core/internal/store/project_api_keys_test.go index 5ebe2967d..67e059d4e 100644 --- a/services/core/internal/store/project_api_keys_test.go +++ b/services/core/internal/store/project_api_keys_test.go @@ -115,7 +115,7 @@ func TestProjectKeysShareIdentityAndArchiveRetainsAssets(t *testing.T) { func TestProjectManagementRequiresAtomicAudit(t *testing.T) { s, _ := testStore(t) id := uuid.NewString() - if _, err := s.CreateProject(t.Context(), id, "unaudited"); !errors.Is(err, ErrInvalidInput) { + if _, err := s.CreateProject(t.Context(), id, "unaudited"); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("unaudited Project accepted") } if _, err := s.GetProject(t.Context(), id); !errors.Is(err, ErrNotFound) { @@ -123,17 +123,17 @@ func TestProjectManagementRequiresAtomicAudit(t *testing.T) { } p := createTestProject(t, s) keyID := uuid.NewString() - if _, err := s.CreateProjectAPIKey(t.Context(), p.ID, keyID, "unaudited"); !errors.Is(err, ErrInvalidInput) { + if _, err := s.CreateProjectAPIKey(t.Context(), p.ID, keyID, "unaudited"); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("unaudited key accepted") } page, err := s.ListProjectAPIKeys(t.Context(), p.ID, "", 20, true) if err != nil || len(page.Data) != 0 { t.Fatal("unaudited key persisted") } - if _, err := s.RenameProject(t.Context(), p.ID, "bad"); !errors.Is(err, ErrInvalidInput) { + if _, err := s.RenameProject(t.Context(), p.ID, "bad"); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("unaudited rename accepted") } - if _, err := s.ArchiveProject(t.Context(), p.ID); !errors.Is(err, ErrInvalidInput) { + if _, err := s.ArchiveProject(t.Context(), p.ID); !errors.Is(err, adminaudit.ErrInvalidSource) { t.Fatal("unaudited archive accepted") } current, err := s.GetProject(t.Context(), p.ID) diff --git a/services/core/internal/store/projects.go b/services/core/internal/store/projects.go index 37a7041a6..d6d00f811 100644 --- a/services/core/internal/store/projects.go +++ b/services/core/internal/store/projects.go @@ -11,6 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -84,7 +85,7 @@ func (s *Store) CreateProject(ctx context.Context, id, name string) (Project, er return err } result = projectBinding(row).Project - return recordAdminMutation(ctx, q, result.TenantID, "create", "project", id) + return auditpg.RecordAdminMutation(ctx, q, result.TenantID, "create", "project", id) }) if err != nil { return Project{}, err @@ -172,7 +173,7 @@ func (s *Store) mutateProject(ctx context.Context, id, name string, archive bool return err } result = projectBinding(row).Project - return recordAdminMutation(ctx, q, result.TenantID, action, "project", id) + return auditpg.RecordAdminMutation(ctx, q, result.TenantID, action, "project", id) }) if err != nil { return Project{}, err diff --git a/services/core/internal/store/public_handler_fixture_test.go b/services/core/internal/store/public_handler_fixture_test.go index e196be009..4d96803ab 100644 --- a/services/core/internal/store/public_handler_fixture_test.go +++ b/services/core/internal/store/public_handler_fixture_test.go @@ -10,6 +10,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimehistory" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" @@ -22,10 +24,11 @@ import ( const testExecutorURL = "wss://core.example/api/v1/agent-daemon/ws" // publicHandler serves s through api.NewHandler. s backs every area the Store -// implements, and db is the database and credential key that built s. keys -// authenticate as Project keys and "admin" as the Core key. Metrics, Runtime -// observation and history, and executor connections are strict stand-ins. -// Execution and Sandboxes stay disabled unless configure sets them. +// implements, and db is the database and credential key that built s; the +// audit reads come from db. keys authenticate as Project keys and "admin" as +// the Core key. Metrics, Runtime observation and history, and executor +// connections are strict stand-ins. Execution and Sandboxes stay disabled +// unless configure sets them. func publicHandler(t testing.TB, s *store.Store, db fixtureDB, keys fixtureKeyResolver, engine string, configure ...func(*api.Dependencies)) (http.Handler, error) { t.Helper() admin, err := api.NewDeploymentAuthenticator([]string{runtimedevice.HashCredential("admin")}) @@ -33,11 +36,12 @@ func publicHandler(t testing.TB, s *store.Store, db fixtureDB, keys fixtureKeyRe return nil, err } strict := strictStandIn{t} + audit := auditpg.New(pgunit.NewPool(db.pool)) deps := api.Dependencies{ Engine: engine, CoreKeys: admin, InstallationBindings: s, Projects: fixtureProjects{Store: s, keys: keys}, Vaults: s, ModelProviders: s, Files: s, Skills: s, EnvironmentTemplates: s, Agents: s, Sessions: s, SessionEvents: s, SessionHistory: s, Subagents: s, - Artifacts: s, SessionAdmin: s, Environments: s, Admin: s, WriteAudit: s, + Artifacts: s, SessionAdmin: s, Environments: s, Admin: s, AdminAudit: audit, WriteAudit: audit, ExecutorConnections: strict, Metrics: strict, RuntimeObservations: strict, RuntimeHistory: strict, } for _, c := range configure { diff --git a/services/core/internal/store/sandbox_deployment_mutations.go b/services/core/internal/store/sandbox_deployment_mutations.go index c6a4eac92..93e4a4c54 100644 --- a/services/core/internal/store/sandbox_deployment_mutations.go +++ b/services/core/internal/store/sandbox_deployment_mutations.go @@ -7,6 +7,7 @@ import ( "math" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" "github.com/jackc/pgx/v5" ) @@ -213,7 +214,7 @@ func (s *Store) UpdateSandboxDeployment(ctx context.Context, installation string if input.ReplacesCredential() { action = "replace_credential" } - if err := recordDeploymentMutation(ctx, q, action, "sandbox_deployment", installation); err != nil { + if err := auditpg.RecordDeploymentMutation(ctx, q, action, "sandbox_deployment", installation); err != nil { return err } } diff --git a/services/core/internal/store/sandbox_reset.go b/services/core/internal/store/sandbox_reset.go index 010567c38..1346d9080 100644 --- a/services/core/internal/store/sandbox_reset.go +++ b/services/core/internal/store/sandbox_reset.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) @@ -95,7 +96,7 @@ func (s *Store) StartSandboxReset(ctx context.Context, installation string, inpu if err := q.ForceSandboxReset(ctx); err != nil { return err } - if err := recordDeploymentMutation(ctx, q, "reset_force", "sandbox_deployment", installation); err != nil { + if err := auditpg.RecordDeploymentMutation(ctx, q, "reset_force", "sandbox_deployment", installation); err != nil { return err } } @@ -113,7 +114,7 @@ func (s *Store) StartSandboxReset(ctx context.Context, installation string, inpu if err := q.StartSandboxReset(ctx, sqlc.StartSandboxResetParams{Clear: pgtype.Text{String: input.Clear, Valid: true}, DeadlineSeconds: deadline, Audit: audit}); err != nil { return err } - if err := recordDeploymentMutation(ctx, q, "reset_start", "sandbox_deployment", installation); err != nil { + if err := auditpg.RecordDeploymentMutation(ctx, q, "reset_start", "sandbox_deployment", installation); err != nil { return err } } @@ -134,7 +135,7 @@ func (s *Store) CancelSandboxReset(ctx context.Context, installation string, gen if err := q.CancelSandboxReset(ctx); err != nil { return err } - if err := recordDeploymentMutation(ctx, q, "reset_cancel", "sandbox_deployment", installation); err != nil { + if err := auditpg.RecordDeploymentMutation(ctx, q, "reset_cancel", "sandbox_deployment", installation); err != nil { return err } } @@ -159,7 +160,7 @@ func (s *Store) AdvanceSandboxResetDeadline(ctx context.Context) error { if err := q.ForceSandboxReset(ctx); err != nil { return err } - return recordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_deadline", "sandbox_deployment", runtimeUUID(d.InstallationID)) + return auditpg.RecordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_deadline", "sandbox_deployment", runtimeUUID(d.InstallationID)) }) } @@ -208,7 +209,7 @@ func (s *Store) CompleteSandboxReset(ctx context.Context, installation string, g if err := q.ClearSandboxGenerations(ctx); err != nil { return err } - if err := recordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation); err != nil { + if err := auditpg.RecordDeploymentMutation(adminaudit.WithSource(ctx, source), q, "reset_complete", "sandbox_deployment", installation); err != nil { return err } result, err = s.deploymentView(ctx, q) diff --git a/services/core/internal/store/session_artifacts.go b/services/core/internal/store/session_artifacts.go index a7b28c86f..888b83bac 100644 --- a/services/core/internal/store/session_artifacts.go +++ b/services/core/internal/store/session_artifacts.go @@ -7,6 +7,8 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -51,7 +53,7 @@ func (s *Store) ListSessionArtifacts(ctx context.Context, tenantID, sessionID, e params := sqlc.ListSessionArtifactsParams{TenantID: tenant, SessionID: session, PageLimit: int32(limit + 1), Ascending: ascending, AfterID: pgtype.UUID{Valid: true}} if environmentID != "" { // A malformed filter matches nothing, like another Environment's ID (HE-56). - params.EnvironmentID = parsePathID(environmentID) + params.EnvironmentID = pgunit.PathID(environmentID) } if cursor != "" { // Any cursor that is not an Artifact of this Session, including a @@ -142,7 +144,7 @@ func (s *Store) DeleteSessionArtifact(ctx context.Context, tenantID, sessionID, if err := objects.Unlink(ctx, oid.Uint32); err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "delete", "artifact", uuid.UUID(lookup.ID.Bytes).String(), uuid.UUID(lookup.SessionID.Bytes).String()) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "artifact", uuid.UUID(lookup.ID.Bytes).String(), uuid.UUID(lookup.SessionID.Bytes).String()) }) } diff --git a/services/core/internal/store/session_creation_identity.go b/services/core/internal/store/session_creation_identity.go index 31efd4330..2a0080410 100644 --- a/services/core/internal/store/session_creation_identity.go +++ b/services/core/internal/store/session_creation_identity.go @@ -11,6 +11,7 @@ import ( "strings" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" @@ -30,7 +31,7 @@ func (s *Store) fingerprintedProvider(provider *v1.ModelProviderInput) (*v1.Mode } fingerprint, err := s.credentialCipher.Fingerprint(modelProviderKeyPurpose, provider.APIKey) if err != nil { - return nil, ErrCredentialStorageUnavailable + return nil, credentialcrypto.ErrUnavailable } copy := *provider copy.APIKey = "fingerprint:" + fingerprint @@ -111,7 +112,7 @@ func (s *Store) FindSessionCreation(ctx context.Context, tenantID, key string, r return SessionCreation{}, ErrInvalidInput } hash, err := s.creationRequestHash(request) - if errors.Is(err, ErrCredentialStorageUnavailable) { + if errors.Is(err, credentialcrypto.ErrUnavailable) { // Without the credential key no Session with a provider bundle can have // been committed or can be created; creation reports the missing key // after request validation. diff --git a/services/core/internal/store/session_deletion.go b/services/core/internal/store/session_deletion.go index 3d69804be..44ba6772b 100644 --- a/services/core/internal/store/session_deletion.go +++ b/services/core/internal/store/session_deletion.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -23,7 +24,7 @@ var ErrSessionNotIdle = errors.New("session must be durably idle or failed witho func (s *Store) DeleteSession(ctx context.Context, tenantID, sessionID string) error { return s.withLockedSession(ctx, tenantID, sessionID, true, func(ctx context.Context, q *sqlc.Queries, session sqlc.LockSessionRow) error { audit := func() error { - return recordWriteAudit(ctx, q, tenantID, "delete", "session", uuid.UUID(session.ID.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "session", uuid.UUID(session.ID.Bytes).String(), "") } if session.DeletedAt.Valid { return audit() diff --git a/services/core/internal/store/session_deletion_lifecycle_public_test.go b/services/core/internal/store/session_deletion_lifecycle_public_test.go index 9e2d1c1f8..6bcc32244 100644 --- a/services/core/internal/store/session_deletion_lifecycle_public_test.go +++ b/services/core/internal/store/session_deletion_lifecycle_public_test.go @@ -7,8 +7,11 @@ import ( "reflect" "testing" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/store" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" ) @@ -21,6 +24,7 @@ const deletionAgent = `"agent":{"id":"agent_deletion","model":"fixture","tools": func TestSessionDeletionLifecyclePostgres(t *testing.T) { // An isolated database keeps the no-write digest independent of other tests. s, db := newManagedTestStoreDB(t) + audit := auditpg.New(pgunit.NewPool(db.pool)) ctx := t.Context() tenant, owner, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() auth := newTestAuthenticator(t, []testAPIKey{ @@ -199,8 +203,8 @@ func TestSessionDeletionLifecyclePostgres(t *testing.T) { t.Fatalf("settled Session deletion: %d %s", status, first) } digest := databaseDigest(t, db.pool) - filter := store.WriteOperationFilter{ResourceType: "session", ResourceID: id, Limit: 100} - beforeAudit, err := s.ListWriteOperations(ctx, tenant, filter) + filter := writeaudit.Filter{ResourceType: "session", ResourceID: id, Limit: 100} + beforeAudit, err := audit.ListWriteOperations(ctx, tenant, filter) if err != nil { t.Fatal(err) } @@ -209,7 +213,7 @@ func TestSessionDeletionLifecyclePostgres(t *testing.T) { t.Fatalf("repeated deletion: %d %s", status, again) } } - afterAudit, err := s.ListWriteOperations(ctx, tenant, filter) + afterAudit, err := audit.ListWriteOperations(ctx, tenant, filter) if err != nil || len(afterAudit.Data) != len(beforeAudit.Data)+2 || !reflect.DeepEqual(afterAudit.Data[2:], beforeAudit.Data) { t.Fatal("repeated deletion must append exactly two operation records", err) } diff --git a/services/core/internal/store/session_diagnostics.go b/services/core/internal/store/session_diagnostics.go index b956fc9f5..c8f75b280 100644 --- a/services/core/internal/store/session_diagnostics.go +++ b/services/core/internal/store/session_diagnostics.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -35,7 +36,7 @@ func (s *Store) GetSessionDiagnosticsSnapshot(ctx context.Context, tenantID, ses var session Session err = s.pooled.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { q := s.queries.WithTx(tx) - row, err := q.GetSession(ctx, sqlc.GetSessionParams{TenantID: tenant, ID: parsePathID(sessionID)}) + row, err := q.GetSession(ctx, sqlc.GetSessionParams{TenantID: tenant, ID: pgunit.PathID(sessionID)}) if err != nil { return err } diff --git a/services/core/internal/store/session_events.go b/services/core/internal/store/session_events.go index a4c361afd..14104cc75 100644 --- a/services/core/internal/store/session_events.go +++ b/services/core/internal/store/session_events.go @@ -9,6 +9,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -96,7 +97,7 @@ func (s *Store) SessionEventCursor(ctx context.Context, tenantID, sessionID stri if err != nil { return 0, err } - id := parsePathID(sessionID) + id := pgunit.PathID(sessionID) cursor, err := s.queries.SessionEventCursor(ctx, sqlc.SessionEventCursorParams{TenantID: tenant, ID: id}) if errors.Is(err, pgx.ErrNoRows) { return 0, ErrNotFound diff --git a/services/core/internal/store/session_execution_configuration.go b/services/core/internal/store/session_execution_configuration.go index 7d0b21567..dd74455b3 100644 --- a/services/core/internal/store/session_execution_configuration.go +++ b/services/core/internal/store/session_execution_configuration.go @@ -8,6 +8,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -67,7 +68,7 @@ func (s *Store) GetSessionExecutionConfiguration(ctx context.Context, tenantID, if err != nil { return v1.SessionExecutionConfiguration{}, err } - row, err := s.queries.GetSessionExecutionConfiguration(ctx, sqlc.GetSessionExecutionConfigurationParams{TenantID: tenant, SessionID: parsePathID(sessionID)}) + row, err := s.queries.GetSessionExecutionConfiguration(ctx, sqlc.GetSessionExecutionConfigurationParams{TenantID: tenant, SessionID: pgunit.PathID(sessionID)}) if errors.Is(err, pgx.ErrNoRows) { return v1.SessionExecutionConfiguration{}, ErrNotFound } diff --git a/services/core/internal/store/session_initial_input.go b/services/core/internal/store/session_initial_input.go index e79a8c2ca..680db13ec 100644 --- a/services/core/internal/store/session_initial_input.go +++ b/services/core/internal/store/session_initial_input.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/json" + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/google/uuid" @@ -11,6 +12,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) func validateInitialInputs(inputs []Input) ([]Input, json.RawMessage, error) { @@ -107,14 +110,14 @@ func (s *Store) createSessionResources(ctx context.Context, tenant string, param return err } audit := func() error { - var created []AuditResource + var created []writeaudit.Resource if row.ID == params.ID { - created = append(created, AuditResource{Type: "session", ID: uuid.UUID(row.ID.Bytes).String()}) + created = append(created, writeaudit.Resource{Type: "session", ID: uuid.UUID(row.ID.Bytes).String()}) if environment != nil { - created = append(created, AuditResource{Type: "environment", ID: environment.ID, ParentID: uuid.UUID(row.ID.Bytes).String()}) + created = append(created, writeaudit.Resource{Type: "environment", ID: environment.ID, ParentID: uuid.UUID(row.ID.Bytes).String()}) } } - return recordWriteAudit(ctx, q, tenant, "create", "session", uuid.UUID(row.ID.Bytes).String(), "", created...) + return auditpg.RecordWriteAudit(ctx, q, tenant, "create", "session", uuid.UUID(row.ID.Bytes).String(), "", created...) } if row.ID != params.ID || len(inputs) == 0 { return audit() diff --git a/services/core/internal/store/session_metadata.go b/services/core/internal/store/session_metadata.go index 94fa01d5b..08e8c63d4 100644 --- a/services/core/internal/store/session_metadata.go +++ b/services/core/internal/store/session_metadata.go @@ -7,6 +7,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/metadata" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -16,7 +18,7 @@ func (s *Store) UpdateSessionMetadata(ctx context.Context, tenantID, sessionID s if err != nil { return Session{}, err } - id := parsePathID(sessionID) + id := pgunit.PathID(sessionID) encoded, err := metadata.Encode(values) if err != nil { return Session{}, fmt.Errorf("%w: %w", ErrInvalidInput, err) @@ -29,7 +31,7 @@ func (s *Store) UpdateSessionMetadata(ctx context.Context, tenantID, sessionID s if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "update", "session", uuid.UUID(row.ID.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "update", "session", uuid.UUID(row.ID.Bytes).String(), "") }) if errors.Is(err, pgx.ErrNoRows) { return Session{}, ErrNotFound diff --git a/services/core/internal/store/session_model_execution.go b/services/core/internal/store/session_model_execution.go index 8332bc824..ad6565c49 100644 --- a/services/core/internal/store/session_model_execution.go +++ b/services/core/internal/store/session_model_execution.go @@ -8,6 +8,7 @@ import ( "io" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -23,7 +24,7 @@ func (s *Store) saveSessionModelExecution(ctx context.Context, q *sqlc.Queries, } encrypted, err := s.credentialCipher.SealModelExecution(raw, tenant, uuid.UUID(session.Bytes).String()) if err != nil { - return ErrCredentialStorageUnavailable + return credentialcrypto.ErrUnavailable } return q.SaveSessionModelExecution(ctx, sqlc.SaveSessionModelExecutionParams{SessionID: session, EncryptedConfig: encrypted}) } diff --git a/services/core/internal/store/session_model_execution_test.go b/services/core/internal/store/session_model_execution_test.go index e22cff186..5d26e43fb 100644 --- a/services/core/internal/store/session_model_execution_test.go +++ b/services/core/internal/store/session_model_execution_test.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/json" "errors" + "testing" + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/google/uuid" - "testing" ) func TestSessionModelExecutionEncryptedAndBound(t *testing.T) { @@ -54,7 +55,7 @@ func TestSessionModelExecutionEncryptedAndBound(t *testing.T) { t.Fatal("missing cipher succeeded") } input.IdempotencyKey = uuid.NewString() - if _, err := New(pool).CreateSession(ctx, tenant, input); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := New(pool).CreateSession(ctx, tenant, input); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("unencrypted create", err) } var count int diff --git a/services/core/internal/store/session_transaction.go b/services/core/internal/store/session_transaction.go index 8145d04fd..8e68fbd60 100644 --- a/services/core/internal/store/session_transaction.go +++ b/services/core/internal/store/session_transaction.go @@ -8,6 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" ) // All Turn admission and lifecycle writes lock the tenant-scoped Session first. @@ -37,7 +38,7 @@ func (s *Store) withLockedSession(ctx context.Context, tenantID, sessionID strin return err } // Public paths resolve malformed IDs as missing; internal callers keep parseID. - id := parsePathID(sessionID) + id := pgunit.PathID(sessionID) if !public { if id, err = parseID(sessionID); err != nil { return err diff --git a/services/core/internal/store/session_write_audit.go b/services/core/internal/store/session_write_audit.go index bc96bd6f2..cf9dde78a 100644 --- a/services/core/internal/store/session_write_audit.go +++ b/services/core/internal/store/session_write_audit.go @@ -4,6 +4,7 @@ import ( "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -15,6 +16,6 @@ func (s *Store) AuditSessionOperation(ctx context.Context, tenantID, sessionID, return ErrInvalidInput } return s.withPublicSession(ctx, tenantID, sessionID, func(ctx context.Context, q *sqlc.Queries, session pgtype.UUID) error { - return recordWriteAudit(ctx, q, tenantID, action, "session", uuid.UUID(session.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, action, "session", uuid.UUID(session.Bytes).String(), "") }) } diff --git a/services/core/internal/store/sessions.go b/services/core/internal/store/sessions.go index 47f32f831..0dfdfb9a5 100644 --- a/services/core/internal/store/sessions.go +++ b/services/core/internal/store/sessions.go @@ -227,7 +227,7 @@ func (s *Store) GetSession(ctx context.Context, tenantID, sessionID string) (Ses if err != nil { return Session{}, err } - id := parsePathID(sessionID) + id := pgunit.PathID(sessionID) row, err := s.queries.GetSession(ctx, sqlc.GetSessionParams{TenantID: tenant, ID: id}) if errors.Is(err, pgx.ErrNoRows) { return Session{}, ErrNotFound @@ -254,7 +254,7 @@ func (s *Store) ListSessions(ctx context.Context, tenantID, cursor string, limit params.AgentID = pgtype.Text{String: *agentID, Valid: true} } if cursor != "" { - after, err := s.GetSession(ctx, tenantID, lookupCursor(cursor)) + after, err := s.GetSession(ctx, tenantID, pgunit.LookupCursor(cursor)) if err != nil { return SessionPage{}, err } @@ -281,30 +281,21 @@ func (s *Store) ListSessions(ctx context.Context, tenantID, cursor string, limit return page, nil } +// parseID translates pgunit's identifier rule into store's invalid-input +// error. Path identifiers use pgunit.PathID and lookup cursors +// pgunit.LookupCursor. func parseID(value string) (pgtype.UUID, error) { - id, err := uuid.Parse(value) - if err != nil || id == uuid.Nil { - return pgtype.UUID{}, fmt.Errorf("%w: nonzero UUID required", ErrInvalidInput) - } - return pgtype.UUID{Bytes: id, Valid: true}, nil -} - -// UnknownResourceID never names a stored resource: Core assigns version 4 or 5 -// UUIDs, and the maximum UUID is neither. -var UnknownResourceID = uuid.Max.String() - -// parsePathID parses a caller-supplied resource path identifier. A value that -// cannot name a resource resolves to UnknownResourceID, so the request follows -// exactly the path of a well-formed missing identifier, including validation -// order. Request-body references keep parseID; see lookupCursor for cursors. -func parsePathID(value string) pgtype.UUID { - id, err := parseID(value) + id, err := pgunit.ParseID(value) if err != nil { - return pgtype.UUID{Bytes: uuid.Max, Valid: true} + return pgtype.UUID{}, fmt.Errorf("%w: %w", ErrInvalidInput, err) } - return id + return id, nil } +// pathID resolves a caller-supplied path identifier with pgunit.PathID for an +// operation that passes it on as a string. +func pathID(value string) string { return uuid.UUID(pgunit.PathID(value).Bytes).String() } + func sessionFromRow(row sqlc.Session) (Session, error) { session := Session{ID: uuid.UUID(row.ID.Bytes).String(), TenantID: uuid.UUID(row.TenantID.Bytes).String(), Engine: row.Engine, CreatedAt: row.CreatedAt.Time, RequiredActions: []v1.FunctionCallAction{}} creator, err := sessionCreator(row.CreatorKind, row.CreatorID) diff --git a/services/core/internal/store/skill_versions.go b/services/core/internal/store/skill_versions.go index bc7f314ee..a8a4c3ebc 100644 --- a/services/core/internal/store/skill_versions.go +++ b/services/core/internal/store/skill_versions.go @@ -7,6 +7,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentskill" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/jackc/pgx/v5" ) @@ -39,8 +41,8 @@ func (s *Store) CreateSkillVersion(ctx context.Context, tenantID, skillID string if err := q.AdvanceSkillVersion(ctx, sqlc.AdvanceSkillVersionParams{TenantID: tenant, ID: id, MakeDefault: makeDefault, Name: metadata.Name, Description: metadata.Description}); err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "upload_version", "skill_version", result.ID, result.SkillID, - AuditResource{Type: "skill_version", ID: result.ID, ParentID: result.SkillID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "upload_version", "skill_version", result.ID, result.SkillID, + writeaudit.Resource{Type: "skill_version", ID: result.ID, ParentID: result.SkillID}) }) if errors.Is(err, pgx.ErrNoRows) { err = ErrNotFound @@ -119,7 +121,7 @@ func (s *Store) DeleteSkillVersion(ctx context.Context, tenantID, skillID, versi if _, err = q.DeleteSkill(ctx, sqlc.DeleteSkillParams{TenantID: tenant, ID: id}); err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "delete", "skill_version", result.ID, result.SkillID) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "skill_version", result.ID, result.SkillID) } row, err := q.DeleteSkillVersion(ctx, sqlc.DeleteSkillVersionParams{TenantID: tenant, SkillID: id, Version: number}) if err != nil { @@ -131,7 +133,7 @@ func (s *Store) DeleteSkillVersion(ctx context.Context, tenantID, skillID, versi return err } } - return recordWriteAudit(ctx, q, tenantID, "delete", "skill_version", result.ID, result.SkillID) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "skill_version", result.ID, result.SkillID) }) if errors.Is(err, pgx.ErrNoRows) { err = ErrNotFound diff --git a/services/core/internal/store/skills.go b/services/core/internal/store/skills.go index cc33d9a8d..35ba7de4c 100644 --- a/services/core/internal/store/skills.go +++ b/services/core/internal/store/skills.go @@ -10,7 +10,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentskill" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -56,9 +58,9 @@ func (s *Store) CreateSkill(ctx context.Context, tenantID string, archive []byte return err } result = skillFromRow(row) - return recordWriteAudit(ctx, q, tenantID, "create", "skill", result.ID, "", - AuditResource{Type: "skill", ID: result.ID}, - AuditResource{Type: "skill_version", ID: initial.ID, ParentID: result.ID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "skill", result.ID, "", + writeaudit.Resource{Type: "skill", ID: result.ID}, + writeaudit.Resource{Type: "skill_version", ID: initial.ID, ParentID: result.ID}) }) return result, err } @@ -99,7 +101,7 @@ func (s *Store) UpdateSkillDefault(ctx context.Context, tenantID, skillID, versi return err } result = skillFromRow(row) - return recordWriteAudit(ctx, q, tenantID, "update_default_version", "skill", result.ID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "update_default_version", "skill", result.ID, "") }) if errors.Is(err, pgx.ErrNoRows) { err = ErrNotFound @@ -117,7 +119,7 @@ func (s *Store) DeleteSkill(ctx context.Context, tenantID, skillID string) error if _, err := q.DeleteSkill(ctx, sqlc.DeleteSkillParams{TenantID: tenant, ID: id}); err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "delete", "skill", skillID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "skill", skillID, "") }) if errors.Is(err, pgx.ErrNoRows) { return ErrNotFound @@ -156,7 +158,7 @@ func skillResourceID(value, prefix string) (pgtype.UUID, error) { return pgtype.UUID{Bytes: id, Valid: true}, nil } -// skillPathIDs resolves a Skill path identifier. Like parsePathID, a malformed +// skillPathIDs resolves a Skill path identifier. Like pgunit.PathID, a malformed // value resolves to an identifier that never exists, so the request follows the // missing-Skill path. Request-body references keep skillIDs. func skillPathIDs(tenantID, skillID string) (pgtype.UUID, pgtype.UUID, error) { diff --git a/services/core/internal/store/source_files.go b/services/core/internal/store/source_files.go index f77a4ec1b..517eb86b7 100644 --- a/services/core/internal/store/source_files.go +++ b/services/core/internal/store/source_files.go @@ -15,6 +15,8 @@ import ( "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) const MaxSourceFileBytes int64 = 512 << 20 @@ -79,7 +81,7 @@ func (s *Store) CreateSourceFile(ctx context.Context, tenantID string, upload fu return fmt.Errorf("create source file: %w", err) } resource := sourceFileFromRow(row) - if err := recordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "create", "file", resource.ID, "", AuditResource{Type: "file", ID: resource.ID}); err != nil { + if err := auditpg.RecordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "create", "file", resource.ID, "", writeaudit.Resource{Type: "file", ID: resource.ID}); err != nil { return err } created = resource @@ -197,7 +199,7 @@ func (s *Store) DeleteSourceFile(ctx context.Context, tenantID, fileID string) e if err := objects.Unlink(ctx, oid.Uint32); err != nil { return err } - return recordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "delete", "file", fileID, "") + return auditpg.RecordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "delete", "file", fileID, "") }) } diff --git a/services/core/internal/store/subagent_item_reads.go b/services/core/internal/store/subagent_item_reads.go index a309e134f..80542544f 100644 --- a/services/core/internal/store/subagent_item_reads.go +++ b/services/core/internal/store/subagent_item_reads.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "errors" + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) @@ -40,7 +42,7 @@ func (s *Store) listChildItems(ctx context.Context, tenant, session, child, turn if after != "" { // Any cursor outside this child (and Turn) scope, including a // malformed one, uses the Session Item cursor error. - row, err := q.GetChildItem(ctx, sqlc.GetChildItemParams{SessionID: sid, SubagentID: childID, ID: parsePathID(after)}) + row, err := q.GetChildItem(ctx, sqlc.GetChildItemParams{SessionID: sid, SubagentID: childID, ID: pgunit.PathID(after)}) if errors.Is(err, pgx.ErrNoRows) || (err == nil && p.TurnID.Valid && p.TurnID != row.TurnID) { return errItemCursor } diff --git a/services/core/internal/store/subagent_reads.go b/services/core/internal/store/subagent_reads.go index c993afea3..4934b28a5 100644 --- a/services/core/internal/store/subagent_reads.go +++ b/services/core/internal/store/subagent_reads.go @@ -7,13 +7,14 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" ) func publicSubagent(ctx context.Context, q *sqlc.Queries, session pgtype.UUID, id string) (v1.Subagent, error) { - row, err := q.GetPublicSubagent(ctx, sqlc.GetPublicSubagentParams{SessionID: session, ID: parsePathID(id)}) + row, err := q.GetPublicSubagent(ctx, sqlc.GetPublicSubagentParams{SessionID: session, ID: pgunit.PathID(id)}) if errors.Is(err, pgx.ErrNoRows) { return v1.Subagent{}, ErrNotFound } @@ -82,7 +83,7 @@ func (s *Store) ListSubagents(ctx context.Context, tenant, session, after string } func childTurn(ctx context.Context, q *sqlc.Queries, session pgtype.UUID, child, id string) (sqlc.SubagentTurn, error) { - row, err := q.GetChildTurn(ctx, sqlc.GetChildTurnParams{SessionID: session, ID: parsePathID(id)}) + row, err := q.GetChildTurn(ctx, sqlc.GetChildTurnParams{SessionID: session, ID: pgunit.PathID(id)}) if errors.Is(err, pgx.ErrNoRows) || (err == nil && uuid.UUID(row.SubagentID.Bytes).String() != child) { return row, ErrNotFound } diff --git a/services/core/internal/store/turn_inputs.go b/services/core/internal/store/turn_inputs.go index 9a794d9b2..c9628dc0d 100644 --- a/services/core/internal/store/turn_inputs.go +++ b/services/core/internal/store/turn_inputs.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" ) type InputReceipt struct { @@ -73,7 +74,7 @@ func (s *Store) SubmitInputs(ctx context.Context, tenantID, sessionID, key strin } if len(previous) > 0 { receipts = previous - return recordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") } if slices.ContainsFunc(batch, func(input Input) bool { return input.Kind == "message" }) { if err := checkEnvironmentFileWriteGate(ctx, q, session); err != nil { @@ -90,7 +91,7 @@ func (s *Store) SubmitInputs(ctx context.Context, tenantID, sessionID, key strin } receipts = append(receipts, receipt) } - return recordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "send_events", "session", uuid.UUID(session.Bytes).String(), "") }) if err != nil { return nil, fmt.Errorf("submit turn inputs: %w", err) diff --git a/services/core/internal/store/turn_reads.go b/services/core/internal/store/turn_reads.go index 60c27496c..ae3162767 100644 --- a/services/core/internal/store/turn_reads.go +++ b/services/core/internal/store/turn_reads.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -28,7 +29,7 @@ func (s *Store) ListTurns(ctx context.Context, tenantID, sessionID, cursor strin params := sqlc.ListRootTurnsParams{TenantID: tenant, SessionID: session, PageLimit: int32(limit + 1), AfterID: pgtype.UUID{Valid: true}, Ascending: ascending} if cursor != "" { // A child Turn is not a Session Turn, so its ID is a missing cursor here. - after, err := s.GetTurn(ctx, tenantID, sessionID, lookupCursor(cursor)) + after, err := s.GetTurn(ctx, tenantID, sessionID, pgunit.LookupCursor(cursor)) if err != nil { return TurnPage{}, err } diff --git a/services/core/internal/store/turns.go b/services/core/internal/store/turns.go index 6a046a711..1196deb04 100644 --- a/services/core/internal/store/turns.go +++ b/services/core/internal/store/turns.go @@ -13,6 +13,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" ) var ErrTurnConflict = errors.New("turn state changed or cancellation was requested") @@ -164,7 +165,7 @@ func turnLookup(tenantID, sessionID, turnID string) (sqlc.GetTurnParams, error) // Turn-scoped resource. Unparsable values are indistinguishable from missing ones. func publicTurnLookup(tenantID, sessionID, turnID string) (sqlc.GetTurnParams, error) { tenant, err := parseID(tenantID) - return sqlc.GetTurnParams{TenantID: tenant, SessionID: parsePathID(sessionID), ID: parsePathID(turnID)}, err + return sqlc.GetTurnParams{TenantID: tenant, SessionID: pgunit.PathID(sessionID), ID: pgunit.PathID(turnID)}, err } func turnFromRow(row sqlc.Turn) Turn { diff --git a/services/core/internal/store/unstorable_text.go b/services/core/internal/store/unstorable_text.go index 33155cb3d..8c730eb95 100644 --- a/services/core/internal/store/unstorable_text.go +++ b/services/core/internal/store/unstorable_text.go @@ -1,16 +1,8 @@ package store -import ( - "errors" +import "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/jackc/pgx/v5/pgconn" -) - -// UnstorableText reports PostgreSQL rejecting client text it cannot represent: -// U+0000 or invalid UTF-8 in a text parameter (22021), or a jsonb \u0000 escape -// (22P05). The rejected statement stores nothing, and writes sharing its -// transaction roll back. -func UnstorableText(err error) bool { - var databaseError *pgconn.PgError - return errors.As(err, &databaseError) && (databaseError.Code == "22021" || databaseError.Code == "22P05") -} +// UnstorableText reports PostgreSQL rejecting text it cannot store; see +// pgunit.IsUnstorableText. Store returns database errors untranslated, so api +// checks them with this until store is deleted. +func UnstorableText(err error) bool { return pgunit.IsUnstorableText(err) } diff --git a/services/core/internal/store/vault_credentials.go b/services/core/internal/store/vault_credentials.go index 63e583d43..870085af1 100644 --- a/services/core/internal/store/vault_credentials.go +++ b/services/core/internal/store/vault_credentials.go @@ -10,14 +10,15 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/oauthrefresh" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" ) -var ErrCredentialStorageUnavailable = errors.New("credential encryption is not configured") - // Credential contains only public metadata. Secret ciphertext is never selected // by resource reads; decryption belongs to scoped execution lookup only. type Credential struct { @@ -42,15 +43,12 @@ func (s *Store) CreateStaticCredential(ctx context.Context, tenantID, vaultID st if err != nil { return Credential{}, err } - vault, err := parseID(vaultID) - if err != nil { - return Credential{}, err - } + vault := pgunit.PathID(vaultID) if !validVaultName(input.Name) || input.MCPServerURL == "" { return Credential{}, ErrInvalidInput } if s.credentialCipher == nil { - return Credential{}, ErrCredentialStorageUnavailable + return Credential{}, credentialcrypto.ErrUnavailable } id := uuid.New() binding := credentialcrypto.Binding{TenantID: uuid.UUID(tenant.Bytes).String(), VaultID: uuid.UUID(vault.Bytes).String(), CredentialID: id.String(), AuthType: "static_bearer", Destination: input.MCPServerURL} @@ -72,7 +70,7 @@ func (s *Store) CreateStaticCredential(ctx context.Context, tenantID, vaultID st if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "create", "credential", created.ID, created.VaultID, AuditResource{Type: "credential", ID: created.ID, ParentID: created.VaultID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "credential", created.ID, created.VaultID, writeaudit.Resource{Type: "credential", ID: created.ID, ParentID: created.VaultID}) }) if errors.Is(err, pgx.ErrNoRows) { return Credential{}, ErrNotFound diff --git a/services/core/internal/store/vault_credentials_delete.go b/services/core/internal/store/vault_credentials_delete.go index 432330906..8fb28bb8c 100644 --- a/services/core/internal/store/vault_credentials_delete.go +++ b/services/core/internal/store/vault_credentials_delete.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -31,7 +32,7 @@ func (s *Store) DeleteCredential(ctx context.Context, tenantID, vaultID, credent return err } deletedID = uuid.UUID(deleted.Bytes).String() - return recordWriteAudit(ctx, q, tenantID, "delete", "credential", deletedID, uuid.UUID(vault.Bytes).String()) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "credential", deletedID, uuid.UUID(vault.Bytes).String()) }) if errors.Is(err, pgx.ErrNoRows) { return "", ErrNotFound diff --git a/services/core/internal/store/vault_credentials_list.go b/services/core/internal/store/vault_credentials_list.go index 778182f27..082dfa5aa 100644 --- a/services/core/internal/store/vault_credentials_list.go +++ b/services/core/internal/store/vault_credentials_list.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -15,6 +16,7 @@ type CredentialPage struct { } func (s *Store) ListCredentials(ctx context.Context, tenantID, vaultID, cursor string, limit int, ascending bool, statuses []string) (CredentialPage, error) { + vaultID = pathID(vaultID) // An inaccessible parent is not an authorized empty collection. vault, err := s.GetVault(ctx, tenantID, vaultID) if err != nil { @@ -35,7 +37,7 @@ func (s *Store) ListCredentials(ctx context.Context, tenantID, vaultID, cursor s parent, _ := parseID(vault.ID) params := sqlc.ListCredentialsParams{TenantID: tenant, VaultID: parent, PageLimit: int32(limit + 1), AfterID: pgtype.UUID{Valid: true}, Ascending: ascending, Statuses: statuses} if cursor != "" { - after, err := s.GetCredential(ctx, tenantID, vaultID, lookupCursor(cursor)) + after, err := s.GetCredential(ctx, tenantID, vaultID, pgunit.LookupCursor(cursor)) if err != nil { return CredentialPage{}, err } diff --git a/services/core/internal/store/vault_credentials_list_test.go b/services/core/internal/store/vault_credentials_list_test.go index 40cbb4871..57bd8d9c4 100644 --- a/services/core/internal/store/vault_credentials_list_test.go +++ b/services/core/internal/store/vault_credentials_list_test.go @@ -133,11 +133,15 @@ func TestCredentialListFilteringOwnershipAndKeylessReconnect(t *testing.T) { owner, vault, cursor string limit int statuses []string - }{{"invalid", vaults[0].ID, "", 20, nil}, {tenant, "invalid", "", 20, nil}, {tenant, vaults[0].ID, "", 0, nil}, {tenant, vaults[0].ID, "", 101, nil}, {tenant, vaults[0].ID, "", 20, []string{"deleted"}}} { + }{{"invalid", vaults[0].ID, "", 20, nil}, {tenant, vaults[0].ID, "", 0, nil}, {tenant, vaults[0].ID, "", 101, nil}, {tenant, vaults[0].ID, "", 20, []string{"deleted"}}} { if _, err := reader.ListCredentials(ctx, tc.owner, tc.vault, tc.cursor, tc.limit, false, tc.statuses); !errors.Is(err, ErrInvalidInput) { t.Fatal("invalid internal query accepted", err) } } + // A malformed Vault path identifier follows the missing-Vault path. + if _, err := reader.ListCredentials(ctx, tenant, "invalid", "", 20, false, nil); !errors.Is(err, ErrNotFound) { + t.Fatal("malformed Vault was not missing", err) + } pool.Close() reopened, _ := testStore(t) if got := read(reopened, true, nil, 20); !reflect.DeepEqual(got, all) || snapshot(reopened) != before { diff --git a/services/core/internal/store/vault_credentials_oauth.go b/services/core/internal/store/vault_credentials_oauth.go index d1a720607..c2016132f 100644 --- a/services/core/internal/store/vault_credentials_oauth.go +++ b/services/core/internal/store/vault_credentials_oauth.go @@ -5,6 +5,9 @@ import ( "errors" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" @@ -15,10 +18,7 @@ func (s *Store) CreateOAuthCredential(ctx context.Context, tenantID, vaultID str if err != nil { return Credential{}, ErrNotFound } - vault, err := parseID(vaultID) - if err != nil { - return Credential{}, ErrNotFound - } + vault := pgunit.PathID(vaultID) if !validVaultName(input.Name) || input.MCPServerURL == "" || !validOAuthMetadata(input.OAuth) { return Credential{}, ErrInvalidInput } @@ -48,7 +48,7 @@ func (s *Store) CreateOAuthCredential(ctx context.Context, tenantID, vaultID str if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "create", "credential", created.ID, created.VaultID, AuditResource{Type: "credential", ID: created.ID, ParentID: created.VaultID}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "credential", created.ID, created.VaultID, writeaudit.Resource{Type: "credential", ID: created.ID, ParentID: created.VaultID}) }) if errors.Is(err, pgx.ErrNoRows) { return Credential{}, ErrNotFound @@ -60,6 +60,7 @@ func (s *Store) CreateOAuthCredential(ctx context.Context, tenantID, vaultID str } func (s *Store) UpdateOAuthCredential(ctx context.Context, tenantID, vaultID, credentialID string, input UpdateOAuthCredentialInput) (Credential, error) { + vaultID, credentialID = pathID(vaultID), pathID(credentialID) current, err := s.GetCredential(ctx, tenantID, vaultID, credentialID) if err != nil { return Credential{}, err @@ -102,7 +103,7 @@ func (s *Store) UpdateOAuthCredential(ctx context.Context, tenantID, vaultID, cr if err != nil { return err } - if err := recordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "update", "credential", updated.ID, updated.VaultID); err != nil { + if err := auditpg.RecordWriteAudit(ctx, s.queries.WithTx(tx), tenantID, "update", "credential", updated.ID, updated.VaultID); err != nil { return errors.New("credential update failed") } return nil diff --git a/services/core/internal/store/vault_credentials_oauth_test.go b/services/core/internal/store/vault_credentials_oauth_test.go index 3a9454732..f41c9744e 100644 --- a/services/core/internal/store/vault_credentials_oauth_test.go +++ b/services/core/internal/store/vault_credentials_oauth_test.go @@ -128,10 +128,10 @@ func TestOAuthCredentialMetadataEncryptionAndScope(t *testing.T) { if _, err := s.CreateOAuthCredential(t.Context(), uuid.NewString(), vault.ID, input); !errors.Is(err, ErrNotFound) { t.Fatal("foreign creation admitted") } - if _, err := New(pool).CreateOAuthCredential(t.Context(), tenant, vault.ID, input); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := New(pool).CreateOAuthCredential(t.Context(), tenant, vault.ID, input); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("keyless creation admitted") } - if token, err := New(pool).MCPBearerToken(t.Context(), tenant, []string{vault.ID}, binding); !errors.Is(err, ErrCredentialStorageUnavailable) || token != "" { + if token, err := New(pool).MCPBearerToken(t.Context(), tenant, []string{vault.ID}, binding); !errors.Is(err, credentialcrypto.ErrUnavailable) || token != "" { t.Fatal("keyless execution admitted") } wrongCipher, _ := credentialcrypto.New(bytes.Repeat([]byte{18}, 32)) diff --git a/services/core/internal/store/vault_credentials_test.go b/services/core/internal/store/vault_credentials_test.go index 4be312fc9..3faafd097 100644 --- a/services/core/internal/store/vault_credentials_test.go +++ b/services/core/internal/store/vault_credentials_test.go @@ -66,7 +66,7 @@ func TestStaticCredentialsPersistEncryptedAndRemainScoped(t *testing.T) { t.Fatal("separate creates reused a credential identity") } valid := CreateStaticCredentialInput{Name: "Rejected", MCPServerURL: "https://mcp.example/tools", Token: opaque} - if _, err := withoutKey.CreateStaticCredential(ctx, tenant, vaults[0].ID, valid); !errors.Is(err, ErrCredentialStorageUnavailable) { + if _, err := withoutKey.CreateStaticCredential(ctx, tenant, vaults[0].ID, valid); !errors.Is(err, credentialcrypto.ErrUnavailable) { t.Fatal("missing encryption key did not fail writes closed") } for _, target := range []struct{ tenant, vault string }{{tenant, vaults[2].ID}, {foreignTenant, vaults[0].ID}, {tenant, uuid.NewString()}} { diff --git a/services/core/internal/store/vault_credentials_update.go b/services/core/internal/store/vault_credentials_update.go index c86bcd265..e96929470 100644 --- a/services/core/internal/store/vault_credentials_update.go +++ b/services/core/internal/store/vault_credentials_update.go @@ -6,6 +6,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -18,6 +19,7 @@ type UpdateStaticCredentialInput struct { // supplies immutable AAD; the mutation independently checks that same scope. // A subsequent dispatch reads the replacement through the existing frozen binding. func (s *Store) UpdateStaticCredential(ctx context.Context, tenantID, vaultID, credentialID string, input UpdateStaticCredentialInput) (Credential, error) { + vaultID, credentialID = pathID(vaultID), pathID(credentialID) tenant, err := parseID(tenantID) if err != nil { return Credential{}, ErrNotFound @@ -38,7 +40,7 @@ func (s *Store) UpdateStaticCredential(ctx context.Context, tenantID, vaultID, c return Credential{}, ErrInvalidInput } if s.credentialCipher == nil { - return Credential{}, ErrCredentialStorageUnavailable + return Credential{}, credentialcrypto.ErrUnavailable } ciphertext, err := s.credentialCipher.Seal([]byte(input.Token), credentialcrypto.Binding{ TenantID: uuid.UUID(tenant.Bytes).String(), VaultID: current.VaultID, @@ -60,7 +62,7 @@ func (s *Store) UpdateStaticCredential(ctx context.Context, tenantID, vaultID, c if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "update", "credential", updated.ID, updated.VaultID) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "update", "credential", updated.ID, updated.VaultID) }) if errors.Is(err, pgx.ErrNoRows) { return Credential{}, ErrNotFound diff --git a/services/core/internal/store/vaults.go b/services/core/internal/store/vaults.go index 05eb8ca0a..4f66c594c 100644 --- a/services/core/internal/store/vaults.go +++ b/services/core/internal/store/vaults.go @@ -14,6 +14,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/metadata" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" ) // Vault is a tenant-owned resource, independent of Sessions and engine execution. @@ -62,7 +64,7 @@ func (s *Store) CreateVault(ctx context.Context, tenantID string, input CreateVa if err != nil { return err } - return recordWriteAudit(ctx, q, tenantID, "create", "vault", created.ID, "", AuditResource{Type: "vault", ID: created.ID, ParentID: ""}) + return auditpg.RecordWriteAudit(ctx, q, tenantID, "create", "vault", created.ID, "", writeaudit.Resource{Type: "vault", ID: created.ID, ParentID: ""}) }) if err != nil { return Vault{}, fmt.Errorf("create vault: %w", err) diff --git a/services/core/internal/store/vaults_delete.go b/services/core/internal/store/vaults_delete.go index 451bd8b13..811240a21 100644 --- a/services/core/internal/store/vaults_delete.go +++ b/services/core/internal/store/vaults_delete.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -27,7 +28,7 @@ func (s *Store) DeleteVault(ctx context.Context, tenantID, vaultID string) (stri return err } deletedID = uuid.UUID(deleted.Bytes).String() - return recordWriteAudit(ctx, q, tenantID, "delete", "vault", deletedID, "") + return auditpg.RecordWriteAudit(ctx, q, tenantID, "delete", "vault", deletedID, "") }) if errors.Is(err, pgx.ErrNoRows) { return "", ErrNotFound diff --git a/services/core/internal/store/vaults_list.go b/services/core/internal/store/vaults_list.go index 0fff6d29e..3015a8c60 100644 --- a/services/core/internal/store/vaults_list.go +++ b/services/core/internal/store/vaults_list.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -32,7 +33,7 @@ func (s *Store) ListVaults(ctx context.Context, tenantID, cursor string, limit i } params := sqlc.ListVaultsParams{TenantID: tenant, PageLimit: int32(limit + 1), AfterID: pgtype.UUID{Valid: true}, Ascending: ascending, Statuses: statuses} if cursor != "" { - after, err := s.GetVault(ctx, tenantID, lookupCursor(cursor)) + after, err := s.GetVault(ctx, tenantID, pgunit.LookupCursor(cursor)) if err != nil { return VaultPage{}, err } diff --git a/services/core/internal/store/write_audit.go b/services/core/internal/store/write_audit.go deleted file mode 100644 index cffee8cf8..000000000 --- a/services/core/internal/store/write_audit.go +++ /dev/null @@ -1,101 +0,0 @@ -package store - -import ( - "context" - "errors" - "fmt" - "strings" - "unicode" - "unicode/utf8" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -// AuditResource identifies a resource genuinely created by the current transaction. -type AuditResource struct{ Type, ID, ParentID string } - -// ValidAuditResourceType is shared by the write recorder and administrator queries. -func ValidAuditResourceType(value string) bool { - switch value { - case "agent", "session", "environment", "environment_template", "skill", "skill_version", "file", "vault", "credential", "artifact": - return true - } - return false -} - -func auditText(value string, max int, required bool) bool { - return (!required || value != "") && utf8.ValidString(value) && utf8.RuneCountInString(value) <= max && !strings.ContainsFunc(value, unicode.IsControl) -} - -func validateWriteAuditSource(source writeaudit.Source, tenant string) error { - actual, err := parseID(source.TenantID) - expected, expectedErr := parseID(tenant) - valid := err == nil && expectedErr == nil && actual == expected && - auditText(source.Name, 80, false) && auditText(source.RequestID, 128, true) && auditText(source.TraceID, 128, true) - switch source.Kind { - case "static", "console": - digest := strings.TrimPrefix(source.KeyID, "static:") - valid = valid && strings.HasPrefix(source.KeyID, "static:") && validProjectKeyDigest(digest) && source.Prefix == digest[:min(len(digest), 8)] - case "issued": - _, idErr := parseID(source.KeyID) - valid = valid && idErr == nil && len(source.Prefix) == 11 && strings.HasPrefix(source.Prefix, "pc_") - for _, c := range strings.TrimPrefix(source.Prefix, "pc_") { - valid = valid && (c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_' || c == '-') - } - default: - valid = false - } - if !valid { - return fmt.Errorf("%w: invalid write audit source", ErrInvalidInput) - } - return nil -} - -// recordWriteAudit must use the caller's business transaction. Internal callers -// without a source stay unattributed; a malformed supplied source fails closed. -func recordWriteAudit(ctx context.Context, q *sqlc.Queries, tenant, action, resourceType, resourceID, parentID string, created ...AuditResource) error { - if _, ok := adminaudit.FromContext(ctx); ok { - return recordAdminMutation(ctx, q, tenant, action, resourceType, resourceID) - } - source, ok := writeaudit.FromContext(ctx) - if !ok { - return nil - } - if err := validateWriteAuditSource(source, tenant); err != nil { - return err - } - switch action { - case "create", "update", "delete", "send_events", "upload_file", "upload_version", "update_default_version": - default: - return fmt.Errorf("%w: invalid write audit action", ErrInvalidInput) - } - resources := append([]AuditResource{{Type: resourceType, ID: resourceID, ParentID: parentID}}, created...) - for _, resource := range resources { - if !ValidAuditResourceType(resource.Type) || !auditText(resource.ID, 256, true) || !auditText(resource.ParentID, 256, false) { - return fmt.Errorf("%w: invalid write audit resource", ErrInvalidInput) - } - } - tenantID, _ := parseID(tenant) - id, err := q.InsertWriteAuditOperation(ctx, sqlc.InsertWriteAuditOperationParams{ - ID: pgtype.UUID{Bytes: uuid.New(), Valid: true}, TenantID: tenantID, - KeyID: source.KeyID, KeyName: source.Name, KeyPrefix: source.Prefix, KeyKind: source.Kind, - Action: action, ResourceType: resourceType, ResourceID: resourceID, ParentID: parentID, RequestID: source.RequestID, TraceID: source.TraceID, - }) - if errors.Is(err, pgx.ErrNoRows) { - return nil - } - if err != nil { - return err - } - for _, resource := range created { - if err := q.InsertWriteAuditOwner(ctx, sqlc.InsertWriteAuditOwnerParams{TenantID: tenantID, ResourceType: resource.Type, ResourceID: resource.ID, ParentID: resource.ParentID, OperationID: id}); err != nil { - return err - } - } - return nil -} diff --git a/services/core/internal/store/write_audit_queries.go b/services/core/internal/store/write_audit_queries.go deleted file mode 100644 index 198a17558..000000000 --- a/services/core/internal/store/write_audit_queries.go +++ /dev/null @@ -1,200 +0,0 @@ -package store - -import ( - "context" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -type AuditAPIKey struct { - ID string `json:"id"` - Name string `json:"name"` - Prefix string `json:"prefix"` - Kind string `json:"kind"` - RevokedAt *time.Time `json:"revoked_at"` -} - -type ResourceOwner struct { - ResourceID string `json:"resource_id"` - APIKey *AuditAPIKey `json:"api_key"` - Source *string `json:"source"` - AdminAuditID *string `json:"admin_audit_id"` -} - -type WriteOperation struct { - ID string `json:"id"` - Action string `json:"action"` - ResourceType string `json:"resource_type"` - ResourceID string `json:"resource_id"` - ParentID string `json:"parent_id"` - RequestID string `json:"request_id"` - TraceID string `json:"trace_id"` - APIKey AuditAPIKey `json:"api_key"` - CreatedAt time.Time `json:"created_at"` -} - -type WriteOperationFilter struct { - KeyID, ResourceType, ResourceID, After string - CreatedAfter, CreatedBefore *time.Time - Limit int -} - -type WriteOperationPage struct { - Data []WriteOperation `json:"data"` - HasMore bool `json:"has_more"` - NextCursor string `json:"next_cursor"` -} - -func auditAPIKey(id, name, prefix, kind string, revoked pgtype.Timestamptz) AuditAPIKey { - key := AuditAPIKey{ID: id, Name: name, Prefix: prefix, Kind: kind} - if revoked.Valid { - value := revoked.Time - key.RevokedAt = &value - } - return key -} - -func (s *Store) GetResourceOwners(ctx context.Context, tenantID, resourceType string, resourceIDs []string) ([]ResourceOwner, error) { - tenant, err := parseID(tenantID) - if err != nil { - return nil, err - } - if !ValidAuditResourceType(resourceType) || len(resourceIDs) < 1 || len(resourceIDs) > 100 { - return nil, fmt.Errorf("%w: invalid owner lookup", ErrInvalidInput) - } - for _, id := range resourceIDs { - if !auditText(id, 256, true) { - return nil, fmt.Errorf("%w: invalid resource ID", ErrInvalidInput) - } - } - rows, err := s.queries.GetResourceOwners(ctx, sqlc.GetResourceOwnersParams{TenantID: tenant, ResourceType: resourceType, Column3: resourceIDs}) - if err != nil { - return nil, err - } - keys := make(map[string]AuditAPIKey, len(rows)) - for _, row := range rows { - keys[row.ResourceID] = auditAPIKey(row.KeyID, row.KeyName, row.KeyPrefix, row.KeyKind, row.RevokedAt) - } - adminRows, err := s.queries.GetAdminResourceOwners(ctx, sqlc.GetAdminResourceOwnersParams{TenantID: tenant, ResourceType: resourceType, Column3: resourceIDs}) - if err != nil { - return nil, err - } - admins := make(map[string]string, len(adminRows)) - for _, row := range adminRows { - admins[row.ResourceID] = uuid.UUID(row.AuditID.Bytes).String() - } - result := make([]ResourceOwner, 0, len(resourceIDs)) - for _, id := range resourceIDs { - owner := ResourceOwner{ResourceID: id} - if key, ok := keys[id]; ok { - owner.APIKey = &key - source := "api_key" - owner.Source = &source - } - if auditID, ok := admins[id]; ok && owner.APIKey == nil { - source := "admin_copy" - owner.Source = &source - owner.AdminAuditID = &auditID - } - result = append(result, owner) - } - return result, nil -} - -type writeAuditCursor struct{ ID, Scope string } - -func auditCursorScope(tenant string, filter WriteOperationFilter) string { - filter.After = "" - filter.Limit = 0 - if filter.CreatedAfter != nil { - value := filter.CreatedAfter.UTC() - filter.CreatedAfter = &value - } - if filter.CreatedBefore != nil { - value := filter.CreatedBefore.UTC() - filter.CreatedBefore = &value - } - encoded, _ := json.Marshal(struct { - Tenant string - Filter WriteOperationFilter - }{tenant, filter}) - hash := sha256.Sum256(encoded) - return hex.EncodeToString(hash[:]) -} - -func auditTimestamp(value *time.Time) pgtype.Timestamptz { - if value == nil { - return pgtype.Timestamptz{} - } - return pgtype.Timestamptz{Time: *value, Valid: true} -} - -func (s *Store) ListWriteOperations(ctx context.Context, tenantID string, filter WriteOperationFilter) (WriteOperationPage, error) { - empty := WriteOperationPage{} - tenant, err := parseID(tenantID) - if err != nil { - return empty, err - } - if filter.Limit == 0 { - filter.Limit = 20 - } - if filter.Limit < 1 || filter.Limit > 100 || filter.ResourceType != "" && !ValidAuditResourceType(filter.ResourceType) || !auditText(filter.KeyID, 128, false) || !auditText(filter.ResourceID, 256, false) || filter.CreatedAfter != nil && filter.CreatedBefore != nil && !filter.CreatedAfter.Before(*filter.CreatedBefore) { - return empty, fmt.Errorf("%w: invalid write operation filter", ErrInvalidInput) - } - scope := auditCursorScope(uuid.UUID(tenant.Bytes).String(), filter) - params := sqlc.ListWriteOperationsParams{TenantID: tenant, KeyID: filter.KeyID, ResourceType: filter.ResourceType, ResourceID: filter.ResourceID, CreatedAfter: auditTimestamp(filter.CreatedAfter), CreatedBefore: auditTimestamp(filter.CreatedBefore), PageLimit: int32(filter.Limit + 1), AfterID: pgtype.UUID{Valid: true}} - if filter.After != "" { - encoded, decodeErr := base64.RawURLEncoding.DecodeString(filter.After) - var cursor writeAuditCursor - if len(filter.After) > 1024 || decodeErr != nil || json.Unmarshal(encoded, &cursor) != nil || cursor.Scope != scope { - return empty, fmt.Errorf("%w: invalid write operation cursor", ErrInvalidInput) - } - id, parseErr := parseID(cursor.ID) - if parseErr != nil { - return empty, fmt.Errorf("%w: invalid write operation cursor", ErrInvalidInput) - } - params.AfterTime, err = s.queries.GetWriteAuditCursor(ctx, sqlc.GetWriteAuditCursorParams{TenantID: tenant, ID: id}) - if errors.Is(err, pgx.ErrNoRows) { - return empty, fmt.Errorf("%w: invalid write operation cursor", ErrInvalidInput) - } - if err != nil { - return empty, err - } - params.AfterID = id - } - rows, err := s.queries.ListWriteOperations(ctx, params) - if err != nil { - return empty, err - } - page := WriteOperationPage{Data: make([]WriteOperation, 0, min(len(rows), filter.Limit)), HasMore: len(rows) > filter.Limit} - if page.HasMore { - rows = rows[:filter.Limit] - } - for _, row := range rows { - page.Data = append(page.Data, WriteOperation{ID: uuid.UUID(row.ID.Bytes).String(), Action: row.Action, ResourceType: row.ResourceType, ResourceID: row.ResourceID, ParentID: row.ParentID, RequestID: row.RequestID, TraceID: row.TraceID, APIKey: auditAPIKey(row.KeyID, row.KeyName, row.KeyPrefix, row.KeyKind, row.RevokedAt), CreatedAt: row.CreatedAt.Time}) - } - if page.HasMore { - encoded, _ := json.Marshal(writeAuditCursor{ID: page.Data[len(page.Data)-1].ID, Scope: scope}) - page.NextCursor = base64.RawURLEncoding.EncodeToString(encoded) - } - return page, nil -} - -// DeleteExpiredWriteOperations bounds each retention transaction and never removes -// an operation referenced by a genuine creation anchor, even after resource deletion. -func (s *Store) DeleteExpiredWriteOperations(ctx context.Context, olderThan time.Time, limit int) (int64, error) { - if olderThan.IsZero() || limit < 1 || limit > 1000 { - return 0, fmt.Errorf("%w: invalid audit retention batch", ErrInvalidInput) - } - return s.queries.DeleteExpiredWriteOperations(ctx, sqlc.DeleteExpiredWriteOperationsParams{CreatedAt: pgtype.Timestamptz{Time: olderThan, Valid: true}, Limit: int32(limit)}) -} diff --git a/services/core/internal/store/write_audit_test.go b/services/core/internal/store/write_audit_test.go deleted file mode 100644 index 03dbb740c..000000000 --- a/services/core/internal/store/write_audit_test.go +++ /dev/null @@ -1,280 +0,0 @@ -package store - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -func auditTestSource(tenant string) writeaudit.Source { - digest := projectKeyDigest(uuid.NewString()) - return writeaudit.Source{KeyID: "static:" + digest, Prefix: digest[:8], Name: "test key", Kind: "static", TenantID: tenant, RequestID: uuid.NewString(), TraceID: uuid.NewString()} -} - -func auditTestRecord(t *testing.T, s *Store, source writeaudit.Source, action, kind, id string, created ...AuditResource) { - t.Helper() - ctx := writeaudit.WithSource(t.Context(), source) - if err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { - return recordWriteAudit(ctx, s.queries.WithTx(tx), source.TenantID, action, kind, id, "", created...) - }); err != nil { - t.Fatal(err) - } -} - -func TestWriteAuditAtomicCommitAndRollback(t *testing.T) { - s, pool := testStore(t) - tenant := uuid.NewString() - source := auditTestSource(tenant) - for _, failure := range []string{"", "after_audit", "invalid_source", "database_audit_failure"} { - t.Run(failure, func(t *testing.T) { - id := uuid.NewString() - source.RequestID = uuid.NewString() - if failure == "invalid_source" { - source.TraceID = "" - } else { - source.TraceID = uuid.NewString() - } - ctx := writeaudit.WithSource(t.Context(), source) - err := pgx.BeginFunc(ctx, pool, func(tx pgx.Tx) error { - q := s.queries.WithTx(tx) - tenantUUID, _ := parseID(tenant) - _, err := q.CreateAgent(ctx, sqlc.CreateAgentParams{ID: pgtype.UUID{Bytes: uuid.MustParse(id), Valid: true}, TenantID: tenantUUID, Metadata: []byte("{}"), Configuration: []byte("{}")}) - if err != nil { - return err - } - if failure == "database_audit_failure" { - // This transaction-local constraint deliberately rejects the audit insert. - if _, err := tx.Exec(ctx, "ALTER TABLE write_audit_operations ADD CONSTRAINT audit_test_failure CHECK (false) NOT VALID"); err != nil { - return err - } - } - if err := recordWriteAudit(ctx, q, tenant, "create", "agent", id, "", AuditResource{Type: "agent", ID: id}); err != nil { - return err - } - if failure == "after_audit" { - return errors.New("business failure after audit") - } - return nil - }) - if (err != nil) != (failure != "") { - t.Fatalf("commit result: %v", err) - } - _, agentErr := s.GetAgent(t.Context(), tenant, id) - if failure == "" && agentErr != nil || failure != "" && !errors.Is(agentErr, ErrNotFound) { - t.Fatalf("business rollback: %v", agentErr) - } - page, err := s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{ResourceID: id}) - if err != nil { - t.Fatal(err) - } - want := 0 - if failure == "" { - want = 1 - } - if len(page.Data) != want { - t.Fatalf("audit count %d want %d", len(page.Data), want) - } - owners, err := s.GetResourceOwners(t.Context(), tenant, "agent", []string{id}) - if err != nil || (owners[0].APIKey != nil) != (failure == "") { - t.Fatalf("ownership rollback: %+v %v", owners, err) - } - }) - } - // A context without authenticated provenance is an intentional internal write. - if err := recordWriteAudit(context.Background(), nil, tenant, "create", "agent", uuid.NewString(), ""); err != nil { - t.Fatal(err) - } -} - -func TestWriteAuditOwnersIdentityReplayAndRevocation(t *testing.T) { - s, _ := testStore(t) - principal := projectKeyPrincipal() - tenant := principal.TenantID - a := auditTestSource(tenant) - id, implicit := uuid.NewString(), uuid.NewString() - auditTestRecord(t, s, a, "create", "session", id, AuditResource{Type: "session", ID: id}, AuditResource{Type: "environment", ID: implicit, ParentID: id}) - // Same request may reach a commit receipt twice but cannot create another owner. - replayID := uuid.NewString() - auditTestRecord(t, s, a, "create", "session", id, AuditResource{Type: "session", ID: replayID}) - b := auditTestSource(tenant) - b.Kind = "console" - auditTestRecord(t, s, b, "update", "session", id) - owners, err := s.GetResourceOwners(t.Context(), tenant, "session", []string{replayID, id, id, "historical"}) - if err != nil || len(owners) != 4 || owners[0].APIKey != nil || owners[1].APIKey.ID != a.KeyID || owners[2].APIKey.ID != a.KeyID || owners[3].APIKey != nil { - t.Fatalf("owners %+v: %v", owners, err) - } - implicitOwners, err := s.GetResourceOwners(t.Context(), tenant, "environment", []string{implicit}) - if err != nil || implicitOwners[0].APIKey.ID != a.KeyID { - t.Fatalf("implicit owner: %+v %v", implicitOwners, err) - } - foreign, err := s.GetResourceOwners(t.Context(), uuid.NewString(), "session", []string{id}) - if err != nil || foreign[0].APIKey != nil { - t.Fatalf("foreign owner: %+v %v", foreign, err) - } - page, err := s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{ResourceID: id}) - if err != nil || len(page.Data) != 2 || page.Data[0].APIKey.Kind != "console" || page.Data[1].APIKey.ID != a.KeyID { - t.Fatalf("request dedup or key identity: %+v %v", page, err) - } - project := createTestProject(t, s) - issued, err := s.CreateProjectAPIKey(keyAdminContext(t.Context(), project.ID), project.ID, uuid.NewString(), "issued key") - if err != nil { - t.Fatal(err) - } - tenant = project.TenantID - c := auditTestSource(tenant) - c.KeyID, c.Name, c.Prefix, c.Kind = issued.ID, issued.Name, issued.Prefix, "issued" - fileID := "file_" + uuid.NewString() - auditTestRecord(t, s, c, "create", "file", fileID, AuditResource{Type: "file", ID: fileID}) - if err := s.RevokeProjectAPIKey(keyAdminContext(t.Context(), project.ID), project.ID, issued.ID); err != nil { - t.Fatal(err) - } - revoked, err := s.GetResourceOwners(t.Context(), tenant, "file", []string{fileID}) - if err != nil || revoked[0].APIKey.RevokedAt == nil || revoked[0].APIKey.Name != issued.Name { - t.Fatalf("revocation metadata: %+v %v", revoked, err) - } - page, err = s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{KeyID: c.KeyID}) - if err != nil || len(page.Data) != 1 || page.Data[0].APIKey.RevokedAt == nil { - t.Fatalf("history revocation: %+v %v", page, err) - } -} - -// The copy operation was removed; its committed provenance must stay readable. -func TestHistoricalAdminCopyProvenance(t *testing.T) { - s, pool := testStore(t) - project := createTestProject(t, s) - auditID, agentID := uuid.NewString(), uuid.NewString() - if _, err := pool.Exec(t.Context(), `INSERT INTO admin_audit_log(id,tenant_id,project_id,admin_credential_id,actor_label,action,resource_type,resource_id,result_ids,request_id,trace_id) - VALUES($1,$2,$3,'digest','admin','copy','agent','source-agent',$4::jsonb,'request','trace')`, auditID, project.TenantID, project.ID, `[{"type":"agent","source_id":"source-agent","target_id":"`+agentID+`"}]`); err != nil { - t.Fatal(err) - } - if _, err := pool.Exec(t.Context(), "INSERT INTO admin_resource_owners(tenant_id,resource_type,resource_id,audit_id) VALUES($1,'agent',$2,$3)", project.TenantID, agentID, auditID); err != nil { - t.Fatal(err) - } - owners, err := s.GetResourceOwners(t.Context(), project.TenantID, "agent", []string{agentID}) - if err != nil || len(owners) != 1 || owners[0].APIKey != nil || owners[0].Source == nil || *owners[0].Source != "admin_copy" || owners[0].AdminAuditID == nil || *owners[0].AdminAuditID != auditID { - t.Fatalf("historical copy owner: %+v %v", owners, err) - } - page, err := s.ListAdminAudit(t.Context(), AdminAuditFilter{ProjectID: project.ID, Action: "copy"}) - if err != nil || len(page.Data) != 1 || page.Data[0].ID != auditID || !strings.Contains(string(page.Data[0].ResultIDs), agentID) { - t.Fatalf("historical copy audit: %+v %v", page, err) - } -} - -func TestWriteAuditCursorFiltersAndRetention(t *testing.T) { - s, pool := testStore(t) - tenant := uuid.NewString() - source := auditTestSource(tenant) - agent, err := s.CreateAgent(t.Context(), tenant, CreateAgentInput{Configuration: []byte("{}")}) - if err != nil { - t.Fatal(err) - } - id := agent.ID - auditTestRecord(t, s, source, "create", "agent", id, AuditResource{Type: "agent", ID: id}) - for i := 0; i < 4; i++ { - source.RequestID = uuid.NewString() - auditTestRecord(t, s, source, "update", "agent", id) - } - // Equal timestamps exercise the ID tie breaker, independent of insertion order. - stamp := time.Date(1800, 1, 1, 0, 0, 0, 0, time.UTC) - if _, err := pool.Exec(t.Context(), "UPDATE write_audit_operations SET created_at=$1 WHERE tenant_id=$2", stamp, tenant); err != nil { - t.Fatal(err) - } - first, err := s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{Limit: 2}) - if err != nil || len(first.Data) != 2 || !first.HasMore || first.NextCursor == "" { - t.Fatalf("first %+v %v", first, err) - } - seen := map[string]bool{} - page := first - for { - for _, row := range page.Data { - if seen[row.ID] { - t.Fatal("duplicate cursor item") - } - seen[row.ID] = true - } - if !page.HasMore { - break - } - page, err = s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{Limit: 2, After: page.NextCursor}) - if err != nil { - t.Fatal(err) - } - } - if len(seen) != 5 { - t.Fatalf("lost rows %d", len(seen)) - } - for _, filter := range []WriteOperationFilter{{Limit: 2, After: first.NextCursor, KeyID: "different"}, {After: "malformed"}} { - if _, err := s.ListWriteOperations(t.Context(), tenant, filter); !errors.Is(err, ErrInvalidInput) { - t.Fatalf("cursor filter: %v", err) - } - } - if _, err := s.ListWriteOperations(t.Context(), uuid.NewString(), WriteOperationFilter{After: first.NextCursor}); !errors.Is(err, ErrInvalidInput) { - t.Fatalf("cross tenant cursor: %v", err) - } - for _, filter := range []WriteOperationFilter{{CreatedBefore: &stamp}, {KeyID: "missing"}, {ResourceType: "file"}, {ResourceID: "missing"}} { - page, err := s.ListWriteOperations(t.Context(), tenant, filter) - if err != nil || len(page.Data) != 0 { - t.Fatalf("filter %+v: %+v %v", filter, page, err) - } - } - inclusive, err := s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{CreatedAfter: &stamp}) - if err != nil || len(inclusive.Data) != 5 { - t.Fatalf("inclusive lower bound: %+v %v", inclusive, err) - } - cutoff := stamp.Add(time.Hour) - n, err := s.DeleteExpiredWriteOperations(t.Context(), cutoff, 2) - if err != nil || n != 2 { - t.Fatalf("bounded retention %d %v", n, err) - } - n, err = s.DeleteExpiredWriteOperations(t.Context(), cutoff, 1000) - if err != nil || n != 2 { - t.Fatalf("remaining retention %d %v", n, err) - } - page, err = s.ListWriteOperations(t.Context(), tenant, WriteOperationFilter{}) - if err != nil || len(page.Data) != 1 || page.Data[0].Action != "create" { - t.Fatalf("creator retention %+v %v", page, err) - } - // Deleting the business resource cannot cascade through provenance. - if _, err := pool.Exec(t.Context(), "DELETE FROM agents WHERE tenant_id=$1 AND id=$2", tenant, id); err != nil { - t.Fatal(err) - } - owners, err := s.GetResourceOwners(t.Context(), tenant, "agent", []string{id}) - if err != nil || owners[0].APIKey == nil { - t.Fatalf("deleted creator %+v %v", owners, err) - } -} - -func TestWriteAuditSourceValidation(t *testing.T) { - valid := auditTestSource(uuid.NewString()) - for _, field := range []string{"tenant", "key", "prefix", "kind", "request", "trace", "name"} { - source := valid - switch field { - case "tenant": - source.TenantID = uuid.NewString() - case "key": - source.KeyID = "static:abcd" - case "prefix": - source.Prefix = "bad" - case "kind": - source.Kind = "unknown" - case "request": - source.RequestID = "" - case "trace": - source.TraceID = "" - case "name": - source.Name = strings.Repeat("x", 81) - } - ctx := writeaudit.WithSource(t.Context(), source) - if err := recordWriteAudit(ctx, nil, valid.TenantID, "create", "agent", "resource", ""); !errors.Is(err, ErrInvalidInput) { - t.Fatalf("%s accepted: %v", field, err) - } - } -} diff --git a/services/core/internal/textvalue/textvalue.go b/services/core/internal/textvalue/textvalue.go new file mode 100644 index 000000000..5eb9b5c4e --- /dev/null +++ b/services/core/internal/textvalue/textvalue.go @@ -0,0 +1,11 @@ +// Package textvalue holds the one error shared by every domain for request text +// that Core cannot store. +package textvalue + +import "errors" + +// ErrUnstorable reports text that PostgreSQL cannot represent: U+0000 or +// invalid UTF-8 in a text value, or a \u0000 escape in a jsonb value. Nothing +// was stored, and the writes sharing the rejected statement's transaction +// rolled back. +var ErrUnstorable = errors.New("text contains characters that cannot be stored") diff --git a/services/core/internal/writeaudit/context.go b/services/core/internal/writeaudit/context.go index 58d1fa648..5ec84f1c8 100644 --- a/services/core/internal/writeaudit/context.go +++ b/services/core/internal/writeaudit/context.go @@ -1,5 +1,7 @@ // Package writeaudit carries authenticated, non-secret request provenance to -// business transactions. It does not authorize requests or replace principals. +// business transactions, and owns the rules for recording it and its read +// models. It does not authorize requests or replace principals. +// persistence/postgres/auditpg stores it. package writeaudit import "context" diff --git a/services/core/internal/writeaudit/reader.go b/services/core/internal/writeaudit/reader.go new file mode 100644 index 000000000..ed9964e72 --- /dev/null +++ b/services/core/internal/writeaudit/reader.go @@ -0,0 +1,90 @@ +package writeaudit + +import ( + "context" + "errors" + "time" +) + +// ErrInvalidQuery reports a provenance query with an invalid tenant, filter, +// page size or cursor. +var ErrInvalidQuery = errors.New("invalid write audit query") + +// Reader reads write provenance as safe read models, never request bodies or +// credentials. +type Reader interface { + GetResourceOwners(ctx context.Context, tenantID, resourceType string, resourceIDs []string) ([]ResourceOwner, error) + ListWriteOperations(ctx context.Context, tenantID string, filter Filter) (Page, error) +} + +// APIKey is the recorded identity of the key that made a write, with its +// current revocation time. +type APIKey struct { + ID string `json:"id"` + Name string `json:"name"` + Prefix string `json:"prefix"` + Kind string `json:"kind"` + RevokedAt *time.Time `json:"revoked_at"` +} + +// ResourceOwner is the recorded creator of one resource. Both creator fields +// are null for a resource without recorded creation provenance. +type ResourceOwner struct { + ResourceID string `json:"resource_id"` + APIKey *APIKey `json:"api_key"` + Source *string `json:"source"` + AdminAuditID *string `json:"admin_audit_id"` +} + +// Operation is one committed write. +type Operation struct { + ID string `json:"id"` + Action string `json:"action"` + ResourceType string `json:"resource_type"` + ResourceID string `json:"resource_id"` + ParentID string `json:"parent_id"` + RequestID string `json:"request_id"` + TraceID string `json:"trace_id"` + APIKey APIKey `json:"api_key"` + CreatedAt time.Time `json:"created_at"` +} + +// Filter selects committed writes, newest first. A zero Limit is the default +// page size. +type Filter struct { + KeyID, ResourceType, ResourceID, After string + CreatedAfter, CreatedBefore *time.Time + Limit int +} + +type Page struct { + Data []Operation `json:"data"` + HasMore bool `json:"has_more"` + NextCursor string `json:"next_cursor"` +} + +// Validate checks f and returns it with the default page size applied. The +// cursor is checked where it is resolved. +func (f Filter) Validate() (Filter, error) { + if f.Limit == 0 { + f.Limit = 20 + } + if f.Limit < 1 || f.Limit > 100 || f.ResourceType != "" && !ValidResourceType(f.ResourceType) || !ValidText(f.KeyID, 128, false) || !ValidText(f.ResourceID, 256, false) || f.CreatedAfter != nil && f.CreatedBefore != nil && !f.CreatedAfter.Before(*f.CreatedBefore) { + return Filter{}, ErrInvalidQuery + } + return f, nil +} + +// ValidateOwnerQuery checks a batch creator lookup of one to 100 resources of +// one audited type. +func ValidateOwnerQuery(resourceType string, resourceIDs []string) error { + if !ValidResourceType(resourceType) || len(resourceIDs) < 1 || len(resourceIDs) > 100 { + return ErrInvalidQuery + } + for _, id := range resourceIDs { + if !ValidText(id, 256, true) { + return ErrInvalidQuery + } + } + return nil +} diff --git a/services/core/internal/writeaudit/record.go b/services/core/internal/writeaudit/record.go new file mode 100644 index 000000000..6256aa14b --- /dev/null +++ b/services/core/internal/writeaudit/record.go @@ -0,0 +1,101 @@ +package writeaudit + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" +) + +// ErrInvalidSource reports a write audit record that cannot be recorded: +// malformed provenance, or an action or resource outside the audit +// vocabulary. The write it belongs to fails closed. +var ErrInvalidSource = errors.New("invalid write audit source") + +// Resource identifies a resource genuinely created by the current transaction. +type Resource struct{ Type, ID, ParentID string } + +// ValidResourceType reports whether value is an audited resource type. The list +// is closed; recording and owner queries share it. +func ValidResourceType(value string) bool { + switch value { + case "agent", "session", "environment", "environment_template", "skill", "skill_version", "file", "vault", "credential", "artifact": + return true + } + return false +} + +// ValidText reports whether value is valid UTF-8 of at most max runes without +// control characters, and nonempty when required. Every recorded or queried +// audit string passes it. +func ValidText(value string, max int, required bool) bool { + return (!required || value != "") && utf8.ValidString(value) && utf8.RuneCountInString(value) <= max && !strings.ContainsFunc(value, unicode.IsControl) +} + +// ValidKeyDigest reports whether digest is the lowercase hexadecimal SHA-256 +// digest that identifies a Project key. +func ValidKeyDigest(digest string) bool { + decoded, err := hex.DecodeString(digest) + return err == nil && len(decoded) == sha256.Size && hex.EncodeToString(decoded) == digest +} + +// Validate checks that s is well-formed provenance of a write in tenant. +func (s Source) Validate(tenant string) error { + if !s.valid(tenant) { + return fmt.Errorf("%w: invalid provenance", ErrInvalidSource) + } + return nil +} + +// ValidateRecord checks a write record in tenant: source must be well-formed +// provenance from tenant, action an audited write action, and every resource +// an audited resource. +func ValidateRecord(source Source, tenant, action string, resources []Resource) error { + if err := source.Validate(tenant); err != nil { + return err + } + switch action { + case "create", "update", "delete", "send_events", "upload_file", "upload_version", "update_default_version": + default: + return fmt.Errorf("%w: invalid action", ErrInvalidSource) + } + for _, resource := range resources { + if !ValidResourceType(resource.Type) || !ValidText(resource.ID, 256, true) || !ValidText(resource.ParentID, 256, false) { + return fmt.Errorf("%w: invalid resource", ErrInvalidSource) + } + } + return nil +} + +func (s Source) valid(tenant string) bool { + actual, err := parseID(s.TenantID) + expected, expectedErr := parseID(tenant) + valid := err == nil && expectedErr == nil && actual == expected && + ValidText(s.Name, 80, false) && ValidText(s.RequestID, 128, true) && ValidText(s.TraceID, 128, true) + switch s.Kind { + case "static", "console": + digest := strings.TrimPrefix(s.KeyID, "static:") + return valid && strings.HasPrefix(s.KeyID, "static:") && ValidKeyDigest(digest) && s.Prefix == digest[:min(len(digest), 8)] + case "issued": + _, idErr := parseID(s.KeyID) + valid = valid && idErr == nil && len(s.Prefix) == 11 && strings.HasPrefix(s.Prefix, "pc_") + for _, c := range strings.TrimPrefix(s.Prefix, "pc_") { + valid = valid && (c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_' || c == '-') + } + return valid + } + return false +} + +func parseID(value string) (uuid.UUID, error) { + id, err := uuid.Parse(value) + if err == nil && id == uuid.Nil { + err = errors.New("nil UUID") + } + return id, err +} diff --git a/services/core/internal/writeaudit/record_test.go b/services/core/internal/writeaudit/record_test.go new file mode 100644 index 000000000..5b0bc232b --- /dev/null +++ b/services/core/internal/writeaudit/record_test.go @@ -0,0 +1,77 @@ +package writeaudit + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func staticSource(tenant string) Source { + sum := sha256.Sum256([]byte("key")) + digest := hex.EncodeToString(sum[:]) + return Source{KeyID: "static:" + digest, Prefix: digest[:8], Name: "key", Kind: "static", TenantID: tenant, RequestID: "request", TraceID: "trace"} +} + +func TestValidateRecord(t *testing.T) { + tenant := uuid.NewString() + agent := []Resource{{Type: "agent", ID: "agent"}} + issued := staticSource(tenant) + issued.KeyID, issued.Prefix, issued.Kind = uuid.NewString(), "pc_Ab3_-xyz", "issued" + if err := ValidateRecord(staticSource(tenant), tenant, "create", agent); err != nil { + t.Fatal(err) + } + if err := ValidateRecord(issued, tenant, "update_default_version", []Resource{{Type: "skill_version", ID: "1", ParentID: "skill"}}); err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(*Source, *string, *[]Resource){ + "other tenant": func(s *Source, _ *string, _ *[]Resource) { s.TenantID = uuid.NewString() }, + "nil tenant": func(s *Source, _ *string, _ *[]Resource) { s.TenantID = uuid.Nil.String() }, + "short digest": func(s *Source, _ *string, _ *[]Resource) { s.KeyID = "static:abcd" }, + "prefix": func(s *Source, _ *string, _ *[]Resource) { s.Prefix = "bad" }, + "kind": func(s *Source, _ *string, _ *[]Resource) { s.Kind = "unknown" }, + "request": func(s *Source, _ *string, _ *[]Resource) { s.RequestID = "" }, + "trace": func(s *Source, _ *string, _ *[]Resource) { s.TraceID = "bad\x01" }, + "name": func(s *Source, _ *string, _ *[]Resource) { s.Name = strings.Repeat("x", 81) }, + "issued key ID": func(s *Source, _ *string, _ *[]Resource) { s.Kind = "issued" }, + "action": func(_ *Source, a *string, _ *[]Resource) { *a = "copy" }, + "resource type": func(_ *Source, _ *string, r *[]Resource) { *r = []Resource{{Type: "project", ID: "p"}} }, + "resource ID": func(_ *Source, _ *string, r *[]Resource) { *r = []Resource{{Type: "agent"}} }, + "resource parent": func(_ *Source, _ *string, r *[]Resource) { + *r = []Resource{{Type: "agent", ID: "a", ParentID: strings.Repeat("x", 257)}} + }, + } { + source, action, resources := staticSource(tenant), "create", agent + change(&source, &action, &resources) + if err := ValidateRecord(source, tenant, action, resources); !errors.Is(err, ErrInvalidSource) { + t.Errorf("%s accepted: %v", name, err) + } + } +} + +func TestReadQueries(t *testing.T) { + if f, err := (Filter{}).Validate(); err != nil || f.Limit != 20 { + t.Fatal(f, err) + } + now := time.Now() + for _, f := range []Filter{{Limit: -1}, {Limit: 101}, {ResourceType: "project"}, {KeyID: strings.Repeat("x", 129)}, {ResourceID: "bad\x00"}, {CreatedAfter: &now, CreatedBefore: &now}} { + if _, err := f.Validate(); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("filter %+v accepted", f) + } + } + if err := ValidateOwnerQuery("agent", []string{"a", "b"}); err != nil { + t.Fatal(err) + } + for _, q := range []struct { + kind string + ids []string + }{{"project", []string{"a"}}, {"agent", nil}, {"agent", make([]string, 101)}, {"agent", []string{""}}} { + if err := ValidateOwnerQuery(q.kind, q.ids); !errors.Is(err, ErrInvalidQuery) { + t.Errorf("owner query %s %d accepted", q.kind, len(q.ids)) + } + } +}