diff --git a/.changeset/consistent-tool-catalog.md b/.changeset/consistent-tool-catalog.md new file mode 100644 index 0000000000..b635cbaccc --- /dev/null +++ b/.changeset/consistent-tool-catalog.md @@ -0,0 +1,8 @@ +--- +"@executor-js/sdk": minor +"@executor-js/plugin-openapi": patch +"@executor-js/plugin-graphql": patch +"@executor-js/plugin-mcp": patch +--- + +Add bulk discovery of visible tools with self-contained input schemas, effective policies, and read-only metadata. Track catalog generations and rebuild ownership so concurrent rebuilds cannot publish a mixed catalog. Reject incomplete generations and rebuild affected connections on the next read. Keep toolkit visibility and policy decisions on one prepared snapshot. diff --git a/apps/cloud/drizzle/0018_tool_generation.sql b/apps/cloud/drizzle/0018_tool_generation.sql new file mode 100644 index 0000000000..7ff8756f81 --- /dev/null +++ b/apps/cloud/drizzle/0018_tool_generation.sql @@ -0,0 +1,2 @@ +ALTER TABLE "definition" ADD COLUMN "generation" text;--> statement-breakpoint +ALTER TABLE "tool" ADD COLUMN "generation" text; \ No newline at end of file diff --git a/apps/cloud/drizzle/0019_connection_tools_manifest.sql b/apps/cloud/drizzle/0019_connection_tools_manifest.sql new file mode 100644 index 0000000000..ac5b347cb7 --- /dev/null +++ b/apps/cloud/drizzle/0019_connection_tools_manifest.sql @@ -0,0 +1 @@ +ALTER TABLE "connection" ADD COLUMN "tools_manifest" json; \ No newline at end of file diff --git a/apps/cloud/drizzle/0020_connection_tools_rebuild.sql b/apps/cloud/drizzle/0020_connection_tools_rebuild.sql new file mode 100644 index 0000000000..f24b275afc --- /dev/null +++ b/apps/cloud/drizzle/0020_connection_tools_rebuild.sql @@ -0,0 +1 @@ +ALTER TABLE "connection" ADD COLUMN "tools_rebuild" text; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0018_snapshot.json b/apps/cloud/drizzle/meta/0018_snapshot.json new file mode 100644 index 0000000000..544c655bca --- /dev/null +++ b/apps/cloud/drizzle/meta/0018_snapshot.json @@ -0,0 +1,1516 @@ +{ + "id": "72cab41c-f919-4f73-812f-77f72e87cdaf", + "prevId": "42251aa3-ae24-4010-ac65-9f41e26cdc20", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/0019_snapshot.json b/apps/cloud/drizzle/meta/0019_snapshot.json new file mode 100644 index 0000000000..0358929a2b --- /dev/null +++ b/apps/cloud/drizzle/meta/0019_snapshot.json @@ -0,0 +1,1522 @@ +{ + "id": "54aeebc1-2d84-4b48-be95-cef046b1004b", + "prevId": "72cab41c-f919-4f73-812f-77f72e87cdaf", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "tools_manifest": { + "name": "tools_manifest", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/0020_snapshot.json b/apps/cloud/drizzle/meta/0020_snapshot.json new file mode 100644 index 0000000000..ce29f0cd5c --- /dev/null +++ b/apps/cloud/drizzle/meta/0020_snapshot.json @@ -0,0 +1,1528 @@ +{ + "id": "2f2214a9-8d41-43ca-9447-1525d35d9de8", + "prevId": "54aeebc1-2d84-4b48-be95-cef046b1004b", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "tools_manifest": { + "name": "tools_manifest", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_rebuild": { + "name": "tools_rebuild", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index 375397ceca..38dbd78453 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -127,6 +127,27 @@ "when": 1788287088210, "tag": "0017_lush_thunderbolts", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1788505381630, + "tag": "0018_tool_generation", + "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1788506387005, + "tag": "0019_connection_tools_manifest", + "breakpoints": true + }, + { + "idx": 20, + "version": "7", + "when": 1788507464393, + "tag": "0020_connection_tools_rebuild", + "breakpoints": true } ] } diff --git a/apps/cloud/src/db/executor-schema.ts b/apps/cloud/src/db/executor-schema.ts index 0db709b884..0137559578 100644 --- a/apps/cloud/src/db/executor-schema.ts +++ b/apps/cloud/src/db/executor-schema.ts @@ -62,6 +62,8 @@ export const connection = pgTable( description: text("description"), last_health: json("last_health"), tools_synced_at: bigint("tools_synced_at", { mode: "bigint" }), + tools_manifest: json("tools_manifest"), + tools_rebuild: text("tools_rebuild"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), @@ -156,6 +158,7 @@ export const tool = pgTable( input_schema: json("input_schema"), output_schema: json("output_schema"), annotations: json("annotations"), + generation: text("generation"), created_at: timestamp("created_at").notNull(), updated_at: timestamp("updated_at").notNull(), row_id: varchar("row_id", { length: 255 }) @@ -186,6 +189,7 @@ export const definition = pgTable( plugin_id: text("plugin_id").notNull(), name: text("name").notNull(), schema: json("schema").notNull(), + generation: text("generation"), created_at: timestamp("created_at").notNull(), row_id: varchar("row_id", { length: 255 }) .primaryKey() diff --git a/apps/local/src/db/executor-schema.ts b/apps/local/src/db/executor-schema.ts index a7f6ceb48e..81afacb562 100644 --- a/apps/local/src/db/executor-schema.ts +++ b/apps/local/src/db/executor-schema.ts @@ -39,6 +39,8 @@ export const connection = sqliteTable( item_ids: text("item_ids").notNull(), credential_write: text("credential_write"), identity_label: text("identity_label"), + tools_manifest: text("tools_manifest"), + tools_rebuild: text("tools_rebuild"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), @@ -124,6 +126,7 @@ export const tool = sqliteTable( input_schema: text("input_schema"), output_schema: text("output_schema"), annotations: text("annotations"), + generation: text("generation"), created_at: integer("created_at").notNull(), updated_at: integer("updated_at").notNull(), row_id: text("row_id").primaryKey().notNull(), @@ -151,6 +154,7 @@ export const definition = sqliteTable( plugin_id: text("plugin_id").notNull(), name: text("name").notNull(), schema: text("schema").notNull(), + generation: text("generation"), created_at: integer("created_at").notNull(), row_id: text("row_id").primaryKey().notNull(), tenant: text("tenant").notNull(), diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 8014584695..77c2522553 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -227,6 +227,32 @@ export const coreTables = defineTables({ // Epoch ms of the last tool (re)production for this connection. Stale // vs the integration's `config_revised_at` → re-produced on next read. tools_synced_at: nullableBigintColumn("tools_synced_at"), + // The catalog MANIFEST: which build is active for this connection and + // how many tool + definition rows it wrote, as JSON + // `{ generation, tools, definitions }`. Written by every rebuild in the + // same transaction as its rows where the engine has one; on D1 it may + // commit BEFORE the row batch. Either way a reader joining tools to + // definitions (`tools.describeAll`) accepts the rows only when every + // one carries this generation and the counts match exactly — that + // agreement, not commit order, is what proves the catalog is whole on + // a backend that commits each statement on its own + // (D1) and across isolates the per-executor write lock cannot see. + // Null for connections built before the manifest existed. A null + // manifest means "no proven-whole catalog": `tools.describeAll` refuses + // the connection, and the stale-catalog scan treats it as needing a + // rebuild, which stamps it. Rows written by a rebuild that died before + // its stamp are refused the same way, and `tools_synced_at` is cleared + // FIRST on every rebuild so such a death is also rescanned. + tools_manifest: nullableJsonColumn("tools_manifest"), + // The rebuild OWNERSHIP token: the id of the build currently replacing + // this connection's catalog, or null when no build is in flight. A + // rebuild claims it as its first statement (together with clearing + // `tools_synced_at`) and its final stamp is conditioned on still holding + // it. Two builds in different isolates cannot both finish: whichever + // claims last owns the token, and the other's stamp finds it changed + // and writes nothing — so a manifest can never describe rows another + // build has since replaced. Cleared by the stamp that wins. + tools_rebuild: nullableTextColumn("tools_rebuild"), oauth_client: nullableTextColumn("oauth_client"), // The OWNER of `oauth_client` (a Personal connection may be minted through // a shared Workspace app), set together with `oauth_client`; null for @@ -336,6 +362,14 @@ export const coreTables = defineTables({ input_schema: nullableJsonColumn("input_schema"), output_schema: nullableJsonColumn("output_schema"), annotations: nullableJsonColumn("annotations"), + // One opaque id per catalog (re)build, shared by every tool AND + // definition row that build wrote for the connection. A reader that + // must join the two tables (`tools.describeAll`) compares it to prove + // both halves came from the same build — a wall-clock stamp cannot + // (SQLite stores it at second resolution; two builds can share it). + // Nullable only for rows written before the column existed; the next + // rebuild stamps them. + generation: nullableTextColumn("generation"), created_at: dateColumn("created_at"), updated_at: dateColumn("updated_at"), }, @@ -356,6 +390,8 @@ export const coreTables = defineTables({ // rows (22001) — that drift broke cloud migration 0013 once already. name: textColumn("name"), schema: jsonColumn("schema"), + /** Same value as the `tool` rows written by the same build; see there. */ + generation: nullableTextColumn("generation"), created_at: dateColumn("created_at"), }, ["tenant", "owner", "subject", "integration", "connection", "name"], @@ -449,10 +485,12 @@ export type OAuthClientRow = FumaRow; export type OAuthSessionRow = FumaRow; export type ToolRow = FumaRow; /** The tool-row projection the invoke/list hot paths load: everything except - * the heavy `input_schema`/`output_schema` JSON, which only `tools.schema` - * (describe) needs. Plugin `invokeTool` receives this shape — operation - * details ride in plugin storage or `annotations`, not the row schemas. */ -export type ToolInvocationRow = Omit; + * the heavy `input_schema`/`output_schema` JSON (which only `tools.schema` + * needs) and the build `generation` marker (which only the + * tools-to-definitions join in `tools.describeAll` reads). Plugin + * `invokeTool` receives this shape — operation details ride in plugin + * storage or `annotations`, not the row schemas. */ +export type ToolInvocationRow = Omit; /** The columns backing {@link ToolInvocationRow}, for `select` projections. */ export const TOOL_INVOCATION_COLUMNS = [ "tenant", diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index fb2835a529..4df22cb892 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -4,6 +4,7 @@ import { Data, Effect, Inspectable, Logger, Predicate, Result, Scheduler } from import { ElicitationResponse, type ElicitationHandler } from "./elicitation"; import { ToolNotFoundError } from "./errors"; import { createExecutor } from "./executor"; +import { matchPattern } from "./policies"; import { StorageError, type FumaDb } from "./fuma-runtime"; import { AuthTemplateSlug, @@ -54,6 +55,10 @@ const addr = (tool: string): ToolAddress => ToolAddress.make(`tools.${INTEG}.org // resolveTools (with shared $defs), and supports ctx.transaction rollback. // --------------------------------------------------------------------------- +/** Toggled by a test so the demo plugin's next discovery differs from what + * is persisted. Module-level because the plugin closure is created once. */ +const demoDiscoversExtra = { value: false }; + const demoPlugin = definePlugin(() => ({ id: "demo" as const, credentialProviders: [memoryProvider()], @@ -68,6 +73,11 @@ const demoPlugin = definePlugin(() => ({ resolveTools: () => Effect.succeed({ tools: [ + // A test may flip this to make one discovery differ from the last + // persisted catalog (see the lost-claim rebuild test). + ...(demoDiscoversExtra.value + ? [{ name: ToolName.make("extra"), description: "extra" }] + : []), { name: ToolName.make("inspect"), description: "inspect", @@ -775,6 +785,594 @@ describe("createExecutor", () => { }), ); + it.effect("tools.describeAll inlines only the reachable input definitions per tool", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { + provider: ProviderKey.make("memory"), + id: ProviderItemId.make("v"), + }, + }); + + const all = yield* executor.tools.describeAll(); + const inspect = all.find((tool) => tool.name === "inspect"); + const run = all.find((tool) => tool.name === "run"); + expect(inspect).toBeDefined(); + expect(run).toBeDefined(); + // The INPUT schema's transitive `$ref` closure rides along under `$defs`, + // so the schema is self-contained on the wire. `Owner` is only reachable + // from the output schema and `Unused` from nothing, so neither appears. + const inlined = inspect?.inputSchema as { $defs?: Record }; + expect(Object.keys(inlined.$defs ?? {}).sort()).toEqual(["Cat", "Collar", "Dog", "Pet"]); + // A tool with no declared input carries no schema at all. + expect(run?.inputSchema).toBeUndefined(); + }), + ); + + // The tools-to-definitions join in `describeAll` must never serve one + // build's schemas with another build's `$defs`. On a backend with no + // interactive transactions (D1) a rebuild is visible statement by statement, + // so the read is proven consistent by the `generation` stamp each build + // writes on every row, not by a transaction. Modelled here by a storage + // proxy that lets a rebuild commit BETWEEN the tool read and the definition + // read, exactly the interleaving a snapshot would have hidden. + it.effect("tools.describeAll never joins tool rows to definitions from another build", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + // Fires once: after the tool read of a describeAll, before its + // definition read, run the armed rebuild to completion. + const race: { rebuild: (() => Promise) | null } = { rebuild: null }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "findMany") { + return async (table: unknown, query: unknown) => { + const result = await ( + target.findMany as (t: unknown, q: unknown) => Promise + )(table, query); + if (table === "tool" && race.rebuild) { + const rebuild = race.rebuild; + race.rebuild = null; + await rebuild(); + } + return result; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + const before = yield* executor.tools.describeAll(); + const beforeInspect = before.find((tool) => tool.name === "inspect"); + expect(beforeInspect, "the seeded build is served whole").toBeDefined(); + + // Arm: a full rebuild of the same connection lands between the two reads. + race.rebuild = () => + Effect.runPromise( + executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }), + ).then(() => undefined); + const after = yield* executor.tools.describeAll(); + expect(race.rebuild, "the rebuild ran mid-read").toBeNull(); + + // Whatever was served is ONE build: the same reachable `$defs` as a + // clean read, never an old schema against new (or missing) definitions. + const inspect = after.find((tool) => tool.name === "inspect"); + expect(inspect, "the tool is still served").toBeDefined(); + const inlined = inspect?.inputSchema as { $defs?: Record }; + expect(Object.keys(inlined.$defs ?? {}).sort()).toEqual(["Cat", "Collar", "Dog", "Pet"]); + }), + ); + + // On D1 the manifest and the row batch are separate commits, so a reader + // can land with a manifest whose rows are not (all) there. The manifest's + // exact generation + row counts are what let `describeAll` tell that apart + // from a finished build. Modelled here by deleting the definitions out from + // under a stamped + // catalog: the row counts no longer match the manifest. + it.effect("tools.describeAll refuses a catalog whose rows do not match its manifest", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + const whole = yield* executor.tools.describeAll(); + expect(whole.length).toBeGreaterThan(0); + + // The half-written window: definitions gone, manifest still says N. + yield* Effect.promise(() => + config.db.deleteMany("definition", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + const outcome = yield* Effect.result(executor.tools.describeAll()); + expect(Result.isFailure(outcome), "a partial catalog is refused, not served").toBe(true); + // The refusal is not the end: it stale-marks exactly this connection, + // and the very next read's stale scan rebuilds it and serves it whole. + const [marked] = yield* Effect.promise(() => + config.db.findMany("connection", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + expect(marked?.tools_synced_at, "the torn connection is stale-marked").toBeNull(); + const recovered = yield* executor.tools.describeAll(); + expect( + recovered.map((tool) => tool.name).sort(), + "the next read rebuilds and serves", + ).toEqual(["inspect", "run"]); + const inlined = recovered.find((tool) => tool.name === "inspect")?.inputSchema as { + $defs?: Record; + }; + expect(Object.keys(inlined.$defs ?? {}).sort(), "definitions are back").toEqual([ + "Cat", + "Collar", + "Dog", + "Pet", + ]); + }), + ); + + // A catalog written before the manifest column existed has rows and no + // manifest — exactly what a rebuild that died before its stamp leaves too. + // Nothing proves it whole, so it is not served; and because a null manifest + // marks the connection stale, the read that trips on it is the read that + // rebuilds and stamps it. Modelled by wiping the manifest under a live + // catalog. + it.effect( + "tools.describeAll refuses a catalog with no manifest and the next read rebuilds it", + () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + // Pre-upgrade shape: rows present, manifest absent, stamp present. + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + set: { tools_manifest: null }, + }), + ); + // `describeAll` runs the stale scan first, which sees the null manifest + // and rebuilds — so the served catalog is the freshly stamped one, and + // the manifest is back. + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]); + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(row?.tools_manifest, "the rebuild stamped a manifest").not.toBeNull(); + }), + ); + + // Two builds of one connection in two isolates. A claims the rebuild + // token; B claims it after A (so B owns the connection); A's fenced + // replacement then runs. Its guard is "tools_rebuild is still A", which + // no longer holds, so the WHOLE unit — deletes, inserts, stamp — must be + // discarded, leaving B's claim and a null sync stamp for the stale scan to + // settle. Modelled with a storage proxy that lets "B" re-claim between A's + // claim and A's `replaceMany`, then simulates B dying by never stamping. + it.effect( + "a rebuild that lost its claim to a competing build writes nothing, and the row stays stale", + () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean; applied: boolean | undefined } = { + armed: false, + applied: undefined, + }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "replaceMany") { + return async (plan: unknown) => { + // A has claimed; before A's fenced unit runs, B claims. + if (raceState.armed) { + raceState.armed = false; + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { tools_synced_at: null, tools_rebuild: "build-B" }, + }, + ); + } + const out = (await ( + target.replaceMany as (p: unknown) => Promise<{ applied: boolean }> + )(plan)) as { applied: boolean }; + raceState.applied = out.applied; + return out; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + const before = yield* executor.tools.describeAll(); + expect(before.length).toBeGreaterThan(0); + const rowsBefore = yield* Effect.promise(() => + config.db.findMany("tool", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + + // "A" rebuilds — and discovers something NEW (`extra`) that the + // persisted catalog does not have, so a caller handed A's discovery + // instead of the persisted rows is distinguishable. The proxy plays + // "B" between A's claim and A's unit. + demoDiscoversExtra.value = true; + raceState.armed = true; + const reported = yield* executor.connections + .refresh({ owner: "org", integration: INTEG, name: CONN }) + .pipe(Effect.ensuring(Effect.sync(() => void (demoDiscoversExtra.value = false)))); + expect(raceState.armed, "B interleaved").toBe(false); + expect(raceState.applied, "A's fenced unit was discarded").toBe(false); + + // Nothing of A's landed: the rows are still the earlier build's, the + // token is B's, and the sync stamp is null for the stale scan. + const rowsAfter = yield* Effect.promise(() => + config.db.findMany("tool", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + expect(rowsAfter.map((row) => row.generation).sort(), "A did not replace the rows").toEqual( + rowsBefore.map((row) => row.generation).sort(), + ); + // And A reported the persisted rows, VALIDATED against the manifest, + // not the ones it discovered and failed to write — so its caller + // cannot disagree with the next list. A discovered `extra`; persisted + // has no `extra`; the report must not. + expect( + reported.map((tool) => String(tool.name)).sort(), + "refresh reports what is persisted", + ).toEqual(["inspect", "run"]); + expect( + reported.map((tool) => String(tool.address)).sort(), + "refresh reports the persisted addresses", + ).toEqual( + rowsAfter + .map((row) => `tools.${row.integration}.${row.owner}.${row.connection}.${row.name}`) + .sort(), + ); + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(row?.tools_rebuild, "A's unit did not clear B's claim").toBe("build-B"); + expect(row?.tools_synced_at, "the connection stays stale-marked").toBeNull(); + + // The next read runs the stale scan, rebuilds, and serves one whole + // catalog with the token released. + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]); + const [after] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(after?.tools_rebuild, "the recovering build released the token").toBeNull(); + expect(after?.tools_manifest).not.toBeNull(); + }), + ); + + // D1 commits the winner's manifest before its row batch. A loser that + // reads the rows in that window must NOT report the previous build's rows + // as if they were current: `describeAll` refuses that state (manifest names + // the new generation, rows carry the old), so the loser reports nothing. + it.effect("a lost claim reports nothing while the winner's rows have not landed", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean } = { armed: false }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "replaceMany") { + return async (plan: unknown) => { + if (raceState.armed) { + raceState.armed = false; + // B claims AND has already committed its manifest for a + // generation whose rows are not in the table yet — the D1 + // guard-then-batch window. + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { + tools_synced_at: null, + tools_rebuild: "build-B", + tools_manifest: { generation: "gen-B", tools: 2, definitions: 6 }, + }, + }, + ); + } + return (target.replaceMany as (p: unknown) => Promise<{ applied: boolean }>)(plan); + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + raceState.armed = true; + const reported = yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); + expect(raceState.armed, "B interleaved").toBe(false); + // The rows in the table are the OLD build's; the manifest is B's. That + // is not a servable catalog, and the loser must not pretend it is. + expect(reported, "nothing is reported while the winner lands").toEqual([]); + }), + ); + + // The D1 race the guard-then-batch protocol cannot prevent: A's guard + // commits (A "won"), B claims + guards + lands its rows, THEN A's delayed + // batch lands rows A under manifest B. A got `applied: true` and must still + // not report its discovery: the persisted state is manifest B over rows A, + // which every list refuses. A reports nothing and stale-marks the row so + // the next read rebuilds. Modelled by rewriting the manifest to B's after + // A's `replaceMany` returns. + it.effect("a build whose rows landed under another build's manifest reports nothing", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean } = { armed: false }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "replaceMany") { + return async (plan: unknown) => { + const out = await ( + target.replaceMany as (p: unknown) => Promise<{ applied: boolean }> + )(plan); + if (raceState.armed && out.applied) { + raceState.armed = false; + // B's manifest overwrites A's after A's rows are in. + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { + tools_synced_at: Date.now(), + tools_rebuild: null, + tools_manifest: { generation: "gen-B", tools: 2, definitions: 6 }, + }, + }, + ); + } + return out; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + raceState.armed = true; + const reported = yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); + expect(raceState.armed, "B overwrote the manifest after A's rows landed").toBe(false); + expect(reported, "A reports nothing: its rows sit under B's manifest").toEqual([]); + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(row?.tools_synced_at, "the connection is stale-marked for the next read").toBeNull(); + }), + ); + + // A toolkit endpoint is a policy projection: connections it does not + // grant can never contribute a tool. A torn (or simply unrebuildable) + // catalog on one of THOSE must not fail the read for the connection the + // toolkit does grant. Only a provider that can PROVE it (a prepared + // connection-scope predicate) gets to exclude a connection; a bare + // rule-list provider keeps every connection strict. Modelled with a + // provider that grants only `main`, and a torn manifest on `other`. + it.effect("tools.describeAll ignores a torn catalog on a connection the projection blocks", () => + Effect.gen(function* () { + const mainOnly = definePlugin(() => ({ + id: "main-only" as const, + storage: () => ({}), + toolPolicyProvider: () => ({ + list: () => + Effect.succeed([ + { + id: "grant-main", + pattern: `${INTEG}.org.${CONN}.*`, + action: "approve" as const, + position: "a0", + }, + ]), + // One snapshot yields both the resolver and the scope predicate. + prepare: () => + Effect.succeed({ + resolve: ({ toolId }: { readonly toolId: string }) => + matchPattern(`${INTEG}.org.${CONN}.*`, toolId) + ? { action: "approve" as const, source: "user" as const } + : { action: "block" as const, source: "user" as const }, + canServeConnection: (connection: { readonly name: string }) => + connection.name === String(CONN), + }), + }), + }))(); + const config = makeTestConfig({ plugins: [demoPlugin, mainOnly] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + for (const name of [CONN, ConnectionName.make("other")]) { + yield* executor.connections.create({ + owner: "org", + name, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + } + const before = yield* executor.tools.describeAll(); + expect(before.map((tool) => String(tool.connection)).sort(), "only main is served").toEqual([ + "main", + "main", + ]); + + // Tear `other`: its manifest says N definitions, none are there, and it + // is stamped synced so the stale scan will not quietly repair it. + yield* Effect.promise(() => + config.db.deleteMany("definition", { + where: (b) => b.and(b("integration", "=", String(INTEG)), b("connection", "=", "other")), + }), + ); + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => String(tool.connection)).sort(), "main still served").toEqual([ + "main", + "main", + ]); + }), + ); + + // The inverse: a provider with no connection-scope predicate cannot prove + // anything about `other`, so its torn catalog fails the read — strict by + // default, never silently lenient. + it.effect( + "tools.describeAll stays strict under a provider without a connection-scope predicate", + () => + Effect.gen(function* () { + const mainOnlyNoScope = definePlugin(() => ({ + id: "main-only-noscope" as const, + storage: () => ({}), + toolPolicyProvider: () => ({ + list: () => + Effect.succeed([ + { + id: "grant-main", + pattern: `${INTEG}.org.${CONN}.*`, + action: "approve" as const, + position: "a0", + }, + ]), + }), + }))(); + const config = makeTestConfig({ plugins: [demoPlugin, mainOnlyNoScope] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + for (const name of [CONN, ConnectionName.make("other")]) { + yield* executor.connections.create({ + owner: "org", + name, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + } + yield* executor.tools.describeAll(); + yield* Effect.promise(() => + config.db.deleteMany("definition", { + where: (b) => + b.and(b("integration", "=", String(INTEG)), b("connection", "=", "other")), + }), + ); + const outcome = yield* Effect.result(executor.tools.describeAll()); + expect(Result.isFailure(outcome), "without proof, a torn connection fails the read").toBe( + true, + ); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 451fc0f54b..6bd1dec47c 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1,5 +1,6 @@ import { Cause, + Data, Deferred, Duration, Effect, @@ -176,6 +177,7 @@ import type { OwnerBinding, PluginCtx, PluginExtensions, + PreparedToolPolicy, ResolveToolsResult, StaticIntegrationDecl, StaticToolDecl, @@ -199,10 +201,15 @@ import { ORG_SUBJECT, type ExecutorOwnerPolicyContext, } from "./owner-policy"; -import { ToolAnnotationsView, ToolSchemaView, type IntegrationDetectionResult } from "./types"; +import { + ToolAnnotationsView, + ToolSchemaView, + type IntegrationDetectionResult, + type ToolProjection, +} from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; import { buildToolTypeScriptPreview } from "./schema-types"; -import { collectReferencedDefinitions } from "./schema-refs"; +import { collectReferencedDefinitions, reattachDefs } from "./schema-refs"; import { refreshAccessToken, exchangeClientCredentials, @@ -469,6 +476,19 @@ export type Executor = { readonly tools: { readonly list: (filter?: ToolListFilter) => Effect.Effect; readonly schema: (address: ToolAddress) => Effect.Effect; + /** + * Every visible dynamic tool with its self-contained input schema and + * resolved policy, in ONE pass: the tool rows, the shared `$defs`, and the + * policy rule set are each read once, then joined in memory. Built for + * surfaces that must advertise the whole catalog at once (the passthrough + * MCP mode), where a per-tool `schema()` round-trip would be an N+1 over + * thousands of rows. Blocked tools are omitted; static (plugin + * configuration) tools are omitted too — they are codemode affordances, + * not integration tools. + */ + readonly describeAll: ( + filter?: ToolListFilter, + ) => Effect.Effect; }; readonly providers: { @@ -894,6 +914,39 @@ const validateExecutorDbTables = (required: FumaTables, actual: FumaTables): voi }); }; +/** How many times `tools.describeAll` re-reads a catalog whose tool rows and + * definitions came from different rebuild generations. One rebuild landing + * mid-read needs one retry; the bound only exists so a pathological + * rebuild storm fails loudly instead of spinning. */ +const DESCRIBE_ALL_GENERATION_RETRIES = 3; + +/** What a finished catalog build leaves on its connection row (see the + * `tools_manifest` column): the active build and the row counts it wrote. */ +const CatalogManifest = Schema.Struct({ + /** Null for a build that wrote no rows at all (every row it would have + * written is absent, so there is no generation to match). */ + generation: Schema.NullOr(Schema.String), + tools: Schema.Number, + definitions: Schema.Number, +}); +type CatalogManifest = typeof CatalogManifest.Type; +const decodeCatalogManifest = Schema.decodeUnknownOption(CatalogManifest); +const emptyCatalogManifest = (): CatalogManifest => ({ + generation: null, + tools: 0, + definitions: 0, +}); + +/** `tools.describeAll` found connections whose rows are not one finished + * build. Carries exactly which, so recovery can stale-mark those alone. */ +class CatalogMismatch extends Data.TaggedError("CatalogMismatch")<{ + readonly connections: ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly connection: string; + }>; +}> {} + const storageFailureFromUnknown = (message: string, cause: unknown): StorageFailure => isStorageFailure(cause) ? cause : new StorageError({ message, cause }); @@ -1293,6 +1346,18 @@ type LooseStorageDb = { }, ) => Promise; readonly deleteMany: (tableName: string, options?: unknown) => Promise; + readonly replaceMany: (plan: { + readonly guard?: { + readonly table: string; + readonly where?: unknown; + readonly set: Record; + }; + readonly deletes: readonly { readonly table: string; readonly where?: unknown }[]; + readonly inserts: readonly { + readonly table: string; + readonly values: readonly Record[]; + }[]; + }) => Promise<{ readonly applied: boolean }>; readonly findFirst: ( tableName: string, options?: unknown, @@ -1350,6 +1415,21 @@ const makeCoreDb = (fuma: ReturnType) => ({ fuma.use(`${tableName}.deleteMany`, (db) => asLooseStorageDb(db).deleteMany(tableName, options), ), + /** Delete + insert across tables as one atomic unit, fenced by an optional + * guard update. See `AbstractQuery.replaceMany`. */ + replaceMany: (plan: { + readonly guard?: { + readonly table: CoreTableName; + readonly where?: CoreWhere; + readonly set: Record; + }; + readonly deletes: readonly { readonly table: CoreTableName; readonly where?: CoreWhere }[]; + readonly inserts: readonly { + readonly table: CoreTableName; + readonly values: readonly Record[]; + }[]; + }): Effect.Effect<{ readonly applied: boolean }, StorageFailure> => + fuma.use("replaceMany", (db) => asLooseStorageDb(db).replaceMany(plan)), findFirst: >( tableName: TName, options: TOptions, @@ -3534,25 +3614,82 @@ export const createExecutor = [], + definitionRows: readonly Record[], + manifest: CatalogManifest, + ) => + Effect.gen(function* () { + yield* core.updateMany("connection", { + where: connectionWhere, + set: { tools_synced_at: null, tools_rebuild: buildId }, + }); + const { applied } = yield* core.replaceMany({ + guard: { + table: "connection", + where: (b: AnyCb) => b.and(connectionWhere(b), b("tools_rebuild", "=", buildId)), + set: syncedSet(existingRow, manifest), + }, + deletes: [ + { table: "tool", where }, + { table: "definition", where }, + ], + inserts: [ + { table: "tool", values: toolRows }, + { table: "definition", values: definitionRows }, + ], + }); + if (!applied) { + yield* Effect.logInfo("executor tool sync lost its claim to a newer build", { + integration: String(ref.integration), + connection: String(ref.name), + }); + } + return applied; }); // A failing sync must not bury a recorded dead grant's `expired` // verdict: this sync's own credential resolution is what discovers @@ -3590,6 +3727,15 @@ export const createExecutor = ({ tenant: keys.tenant, owner: keys.owner, @@ -3709,6 +3847,7 @@ export const createExecutor = - rowToTool( - { - tenant: keys.tenant, - owner: keys.owner, - subject: keys.subject, - integration: String(ref.integration), - connection: String(ref.name), - plugin_id: integrationRow.plugin_id, - name: String(tool.name), - description: tool.description ?? "", - input_schema: tool.inputSchema ?? null, - output_schema: tool.outputSchema ?? null, - annotations: tool.annotations ?? null, - created_at: now, - updated_at: now, - } as ConnectionToolRow, - tool.annotations, - ), - ); + // Report what is PERSISTED, never what this build discovered — and + // only if what is persisted is one whole build, checked the way every + // list checks it (manifest vs rows). That holds for the loser (it + // wrote nothing) and for the apparent winner alike: on D1 the guard + // commits before the row batch, so a build can see `applied: true` + // and still have its rows land AFTER a later build's manifest, leaving + // manifest B over rows A. Returning this build's local discovery + // there would hand the caller a catalog the next list refuses. When + // the persisted state is not whole, stale-mark the connection so the + // next read rebuilds it, and report an empty catalog rather than a + // wrong one. This cannot go through `describeAll`: that runs the + // stale sync, which would re-enter this very connection's in-flight + // production. + const [rows, persistedDefinitions, connectionRow] = yield* Effect.all([ + core.findMany("tool", { where, select: [...TOOL_INVOCATION_COLUMNS, "generation"] }), + core.findMany("definition", { where }), + findConnectionRow(ref), + ]); + const manifest = connectionRow + ? Option.getOrNull(decodeCatalogManifest(decodeJsonColumn(connectionRow.tools_manifest))) + : null; + const whole = + manifest !== null && + rows.length === manifest.tools && + persistedDefinitions.length === manifest.definitions && + rows.every((row) => row.generation === manifest.generation) && + persistedDefinitions.every((row) => row.generation === manifest.generation); + if (!whole) { + yield* core.updateMany("connection", { + where: connectionWhere, + set: { tools_synced_at: null }, + }); + return []; + } + return rows.map((row) => rowToTool(row)); }); type ToolProductionError = IntegrationNotFoundError | StorageFailure; @@ -5351,13 +5506,7 @@ export const createExecutor = EffectivePolicy; - }; + | { readonly kind: "prepared"; readonly prepared: PreparedToolPolicy }; const compareProviderPolicyRule = ( a: ToolPolicyProviderRule, @@ -5397,9 +5546,9 @@ export const createExecutor = ({ + Effect.map((prepared) => ({ kind: "prepared" as const, - resolve, + prepared, })), ) : activeToolPolicyProvider.resolve @@ -5425,7 +5574,7 @@ export const createExecutor = => ruleSet.kind === "prepared" - ? Effect.succeed(ruleSet.resolve({ toolId, defaultRequiresApproval })) + ? Effect.succeed(ruleSet.prepared.resolve({ toolId, defaultRequiresApproval })) : ruleSet.kind === "provider" ? ruleSet.provider.resolve ? ruleSet.provider.resolve({ toolId, defaultRequiresApproval }) @@ -5475,116 +5624,137 @@ export const createExecutor = [row.slug, row] as const)); - // The TTL only matters when a loaded plugin actually lists a live remote - // catalog; otherwise skip it so age alone never widens the stale query. - const anyRemoteCatalog = Array.from(runtimes.values()).some( - (runtime) => runtime.plugin.remoteToolCatalog === true, - ); - const cutoff = - toolsSyncTtlMs == null || !anyRemoteCatalog ? null : Date.now() - toolsSyncTtlMs; - - // Bound the scan to potentially-stale rows: stale-marked (NULL stamp) or - // synced before the latest instant any trigger could fire at (the TTL - // cutoff / the newest config revision). Per-row trigger checks below - // re-verify against each row's own integration; in steady state this - // query returns nothing and the read pays one indexed lookup. - const latestRevision = integrations.reduce( - (max, row) => - row.config_revised_at == null - ? max - : Math.max(max ?? Number(row.config_revised_at), Number(row.config_revised_at)), - null, - ); - const staleBefore = - cutoff === null && latestRevision === null - ? null - : Math.max(cutoff ?? Number.MIN_SAFE_INTEGER, latestRevision ?? Number.MIN_SAFE_INTEGER); - - const connections = yield* core.findMany("connection", { - where: (b: AnyCb) => - staleBefore === null - ? b.isNull("tools_synced_at") - : b.or(b.isNull("tools_synced_at"), b("tools_synced_at", "<", staleBefore)), - }); - // Each rebuild is an independent upstream listing, so they run together - // rather than one after another: a host with many stale remote-catalog - // connections otherwise pays the sum of every server's latency on the - // read that trips the TTL. Only the listings overlap — `persistCatalog` - // keeps the catalog writes in a single-file queue, so this fan-out never - // opens two transactions on a one-connection database. - const rebuilds: Effect.Effect[] = []; - for (const connection of connections) { - const integrationRow = integrationBySlug.get(connection.integration); - if (!integrationRow) continue; - const runtime = runtimes.get(integrationRow.plugin_id); - // Only re-produce catalogs this executor can actually re-list — - // rebuilding under an unloaded plugin would clear a working catalog. - // (A loaded plugin without `resolveTools` still flows through: - // `produceConnectionTools` runs its clear-and-stamp cleanup path.) - if (!runtime) continue; - - const syncedAt = - connection.tools_synced_at == null ? null : Number(connection.tools_synced_at); - const revisedTime = - integrationRow.config_revised_at == null + // `scope` narrows the sync to one integration's connections. A read that + // only asked for that integration (`describeAll({ integration })`) must not + // scan, dial, or wait on every other stale connection in the workspace. + const syncStaleConnectionToolsScoped = (scope?: { readonly integration: IntegrationSlug }) => + Effect.gen(function* () { + // The platform view can never persist a rebuilt catalog (writes are + // denied at the storage boundary), so attempting the sync would only + // fire upstream `resolveTools` calls whose results are thrown away — + // network side effects on a read-only credential. Skip it entirely: + // read-only-ness of the platform read path is a stated invariant here, + // not an accident of the best-effort catch below. + if (config.platformView === true) return; + const integrations = yield* core.findMany("integration", { + where: (b: AnyCb) => + scope === undefined ? true : b("slug", "=", String(scope.integration)), + }); + if (integrations.length === 0) return; + const integrationBySlug = new Map(integrations.map((row) => [row.slug, row] as const)); + // The TTL only matters when a loaded plugin actually lists a live remote + // catalog; otherwise skip it so age alone never widens the stale query. + const anyRemoteCatalog = Array.from(runtimes.values()).some( + (runtime) => runtime.plugin.remoteToolCatalog === true, + ); + const cutoff = + toolsSyncTtlMs == null || !anyRemoteCatalog ? null : Date.now() - toolsSyncTtlMs; + + // Bound the scan to potentially-stale rows: stale-marked (NULL stamp) or + // synced before the latest instant any trigger could fire at (the TTL + // cutoff / the newest config revision). Per-row trigger checks below + // re-verify against each row's own integration; in steady state this + // query returns nothing and the read pays one indexed lookup. + const latestRevision = integrations.reduce( + (max, row) => + row.config_revised_at == null + ? max + : Math.max(max ?? Number(row.config_revised_at), Number(row.config_revised_at)), + null, + ); + const staleBefore = + cutoff === null && latestRevision === null ? null - : Number(integrationRow.config_revised_at); - - const staleMarked = syncedAt === null; - const configRevised = revisedTime !== null && (syncedAt ?? 0) < revisedTime; - const expired = - cutoff !== null && - runtime.plugin.remoteToolCatalog === true && - syncedAt !== null && - syncedAt < cutoff; - if (!staleMarked && !configRevised && !expired) continue; + : Math.max( + cutoff ?? Number.MIN_SAFE_INTEGER, + latestRevision ?? Number.MIN_SAFE_INTEGER, + ); - rebuilds.push( - produceConnectionTools( - integrationRow, - { - owner: connection.owner as Owner, - integration: IntegrationSlug.make(connection.integration), - name: ConnectionName.make(connection.name), - }, - "background", - ).pipe( - // Best-effort, but never silent: the read still succeeds on the - // stale-but-working catalog and the peer rebuilds still finish, - // while the operator gets the connection that failed and why. - // Without this a connection whose upstream is permanently broken - // re-fails on every read and leaves no trace anywhere. - Effect.catch((error) => - Effect.logWarning("executor stale tool sync failed", { - integration: connection.integration, - connection: connection.name, - error: describeSyncFailure(error), - }).pipe(Effect.as([] as readonly Tool[])), + // A connection with no catalog manifest (built before the manifest + // existed) is stale too: `tools.describeAll` refuses it until a rebuild + // stamps one, so the read that first sees it is the read that fixes it. + const connections = yield* core.findMany("connection", { + where: (b: AnyCb) => + b.and( + scope === undefined ? true : b("integration", "=", String(scope.integration)), + staleBefore === null + ? b.or(b.isNull("tools_synced_at"), b.isNull("tools_manifest")) + : b.or( + b.isNull("tools_synced_at"), + b.isNull("tools_manifest"), + b("tools_synced_at", "<", staleBefore), + ), ), - Effect.withSpan("executor.tools.sync_stale", { - attributes: { - "executor.integration": connection.integration, - "executor.connection": connection.name, + }); + // Each rebuild is an independent upstream listing, so they run together + // rather than one after another: a host with many stale remote-catalog + // connections otherwise pays the sum of every server's latency on the + // read that trips the TTL. Only the listings overlap — `persistCatalog` + // keeps the catalog writes in a single-file queue, so this fan-out never + // opens two transactions on a one-connection database. + const rebuilds: Effect.Effect[] = []; + for (const connection of connections) { + const integrationRow = integrationBySlug.get(connection.integration); + if (!integrationRow) continue; + const runtime = runtimes.get(integrationRow.plugin_id); + // Only re-produce catalogs this executor can actually re-list — + // rebuilding under an unloaded plugin would clear a working catalog. + // (A loaded plugin without `resolveTools` still flows through: + // `produceConnectionTools` runs its clear-and-stamp cleanup path.) + if (!runtime) continue; + + const syncedAt = + connection.tools_synced_at == null ? null : Number(connection.tools_synced_at); + const revisedTime = + integrationRow.config_revised_at == null + ? null + : Number(integrationRow.config_revised_at); + + const staleMarked = syncedAt === null || connection.tools_manifest == null; + const configRevised = revisedTime !== null && (syncedAt ?? 0) < revisedTime; + const expired = + cutoff !== null && + runtime.plugin.remoteToolCatalog === true && + syncedAt !== null && + syncedAt < cutoff; + if (!staleMarked && !configRevised && !expired) continue; + + rebuilds.push( + produceConnectionTools( + integrationRow, + { + owner: connection.owner as Owner, + integration: IntegrationSlug.make(connection.integration), + name: ConnectionName.make(connection.name), }, - }), - ), - ); - } - yield* Effect.all(rebuilds, { - concurrency: STALE_TOOLS_SYNC_CONCURRENCY, + "background", + ).pipe( + // Best-effort, but never silent: the read still succeeds on the + // stale-but-working catalog and the peer rebuilds still finish, + // while the operator gets the connection that failed and why. + // Without this a connection whose upstream is permanently broken + // re-fails on every read and leaves no trace anywhere. + Effect.catch((error) => + Effect.logWarning("executor stale tool sync failed", { + integration: connection.integration, + connection: connection.name, + error: describeSyncFailure(error), + }).pipe(Effect.as([] as readonly Tool[])), + ), + Effect.withSpan("executor.tools.sync_stale", { + attributes: { + "executor.integration": connection.integration, + "executor.connection": connection.name, + }, + }), + ), + ); + } + yield* Effect.all(rebuilds, { + concurrency: STALE_TOOLS_SYNC_CONCURRENCY, + }); }); - }); + const syncStaleConnectionTools = syncStaleConnectionToolsScoped(); // How long a tools read waits for the stale sync before answering from // the persisted rows (`ExecutorConfig.toolsSyncGraceMs`; `null` blocks @@ -5602,10 +5772,13 @@ export const createExecutor = + const awaitStaleSyncWithinGrace = ( + graceMs: number, + scope?: { readonly integration: IntegrationSlug }, + ) => Effect.gen(function* () { const fiber = yield* Effect.forkDetach( - syncStaleConnectionTools.pipe( + syncStaleConnectionToolsScoped(scope).pipe( Effect.catch((error) => Effect.logWarning("executor stale tool sync scan failed", { error: describeSyncFailure(error), @@ -5831,6 +6004,281 @@ export const createExecutor = => + Effect.gen(function* () { + // The stale sync is scoped to the requested integration, so a + // narrowed read never scans, dials, or waits on unrelated + // connections. + const scope = + filter?.integration === undefined ? undefined : { integration: filter.integration }; + if (toolsSyncGraceMs === null) { + yield* syncStaleConnectionToolsScoped(scope); + } else { + yield* awaitStaleSyncWithinGrace(toolsSyncGraceMs, scope); + } + const integrationWhere = (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined + ? true + : b("connection", "=", String(filter.connection)), + ); + // Unlike `toolsList`, this read NEEDS `input_schema`: it is the schema + // the surface advertises. `output_schema` stays out — nothing here + // serves it, and it is the other half of the per-row weight. + // + // Tool rows and their shared `$defs` must come from the SAME catalog + // generation: a detached stale-sync rebuild replaces both tables for a + // connection, and two reads that straddle it would pair an old + // schema's `$ref` with a refreshed definition of the same name. A + // transaction alone does not promise that on every backend (Postgres + // runs READ COMMITTED, D1 has no interactive transactions), so the + // join is checked, not assumed: a rebuild stamps every row it writes + // with the opaque `generation` id of the build, and the build's last + // statement records that id plus the row counts on the connection + // (`tools_manifest`). When a read does not match, it is re-read; a + // bounded number of attempts covers a rebuild landing mid-read without + // ever serving a mixed or partial catalog. + const readCatalog = Effect.all({ + rows: core.findMany("tool", { + where: integrationWhere, + select: [...TOOL_INVOCATION_COLUMNS, "input_schema", "generation"], + }), + // Shared definitions, keyed per connection below: `$ref`s are + // connection-local (`#/$defs/` resolves against the producing + // connection's `definition` rows), so the join key is the same + // (owner, integration, connection) triple the tool row carries. + definitionRows: core.findMany("definition", { where: integrationWhere }), + // Each connection's manifest: the build that last claimed it and + // the row counts it wrote. On D1 the manifest can commit before the + // row batch, so nothing about read order proves anything here; the + // proof below is exact generation + count agreement between the + // manifest and the rows actually present. + manifests: core.findMany("connection", { + where: (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined ? true : b("name", "=", String(filter.connection)), + ), + select: ["owner", "integration", "name", "tools_manifest"], + }), + }); + type ConnectionTriple = { + readonly owner: string; + readonly integration: string; + readonly connection: string; + }; + // Slugs and names are opaque strings; a tuple encoding is the only + // key that cannot collide, and the original triple rides in a side + // map so recovery never has to parse a key back apart. + const connectionKey = (row: ConnectionTriple): string => + JSON.stringify([row.owner, row.integration, row.connection]); + // Per connection, the rows served must be exactly ONE finished build. + // A build stamps every row it writes with one opaque `generation`, and + // writes the connection's manifest — that generation plus the row + // counts — with them (one transaction) or, on D1, just before them. + // Commit order proves nothing; agreement does. With a manifest the + // proof is complete: every tool row and every definition row carries the + // manifest's generation, and there are exactly as many of each as it + // says. That holds on a backend that commits each statement on its + // own (D1): a read that lands mid-rebuild sees either the old + // manifest with some rows already replaced (generation mismatch), or + // fewer rows than the manifest counts (still inserting), or the new + // manifest with everything in place. A competing writer in another + // isolate, which the per-executor write lock cannot serialize, is + // caught the same way: whatever it leaves behind does not match the + // manifest that was written last. A connection with NO manifest has + // no proof at all — it was built before the manifest existed, or its + // last rebuild died before stamping — and is refused outright; the + // stale-catalog scan treats a null manifest as stale, so the rebuild + // that stamps it is already underway by the time this is retried. + type GenerationRow = { + readonly owner: string; + readonly integration: string; + readonly connection: string; + readonly generation: string | null; + }; + // The connections whose rows do not form exactly one finished build. + const mixedGenerations = (snapshot: { + readonly rows: ReadonlyArray; + readonly definitionRows: ReadonlyArray; + readonly manifests: ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly name: string; + readonly tools_manifest: unknown; + }>; + }): ReadonlyArray => { + type Tally = { readonly generations: Set; count: number }; + const triples = new Map(); + const remember = (row: ConnectionTriple): string => { + const key = connectionKey(row); + if (!triples.has(key)) triples.set(key, row); + return key; + }; + const tally = (rows: ReadonlyArray): Map => { + const out = new Map(); + for (const row of rows) { + const key = remember(row); + const entry = out.get(key) ?? { generations: new Set(), count: 0 }; + entry.generations.add(row.generation); + entry.count += 1; + out.set(key, entry); + } + return out; + }; + const tools = tally(snapshot.rows); + const definitions = tally(snapshot.definitionRows); + const manifests = new Map(); + for (const row of snapshot.manifests) { + manifests.set( + remember({ owner: row.owner, integration: row.integration, connection: row.name }), + Option.getOrNull(decodeCatalogManifest(decodeJsonColumn(row.tools_manifest))), + ); + } + const keys = new Set([...tools.keys(), ...definitions.keys(), ...manifests.keys()]); + const mixed: ConnectionTriple[] = []; + for (const key of keys) { + const t = tools.get(key); + const d = definitions.get(key); + const manifest = manifests.get(key) ?? null; + const inconsistent = + // No manifest: nothing proves these rows are whole. + !manifest || + // Exact generation and exact counts on both sides. + (t?.count ?? 0) !== manifest.tools || + (d?.count ?? 0) !== manifest.definitions || + (t !== undefined && + (t.generations.size !== 1 || !t.generations.has(manifest.generation))) || + (d !== undefined && + (d.generations.size !== 1 || !d.generations.has(manifest.generation))); + if (inconsistent) mixed.push(triples.get(key)!); + } + return mixed; + }; + // A connection the active policy projection can never serve (a + // toolkit that grants nothing under it) must not fail this read: its + // catalog may be torn, unrebuildable, or simply someone else's. Only + // a provider that can PROVE that — from real grant-pattern overlap, + // not from a synthetic id a narrower grant would fail to match — + // gets to exclude a connection; otherwise every connection stays + // under strict validation. + // The scope predicate and the per-tool resolver come from ONE + // prepared snapshot, so a grant that changes between reads cannot + // make them disagree. + const policyRulesForScope = yield* listActivePolicyRuleSet(); + const canServeConnection = + policyRulesForScope.kind === "prepared" + ? (policyRulesForScope.prepared.canServeConnection ?? null) + : null; + const { rows, definitionRows } = yield* readCatalog.pipe( + Effect.flatMap((snapshot) => { + const mixed = mixedGenerations(snapshot).filter( + (triple) => + canServeConnection === null || + canServeConnection({ + integration: triple.integration, + owner: triple.owner, + name: triple.connection, + }), + ); + return mixed.length === 0 + ? Effect.succeed(snapshot) + : Effect.fail(new CatalogMismatch({ connections: mixed })); + }), + Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), + // Recovery backstop. A catalog that is still inconsistent after the + // retries is not a rebuild landing mid-read; it is one that landed + // WRONG — a build that died mid-claim, or rows a dead build left + // behind. Nothing else would rescan it (its stamp may be non-null), + // so stale-mark EXACTLY the inconsistent connections before + // surfacing: the next read rebuilds those, and only those. Ordinary + // storage failures are not catalog damage and trigger nothing. + Effect.catchTag("CatalogMismatch", (mismatch) => + Effect.forEach( + mismatch.connections, + (key) => + core + .updateMany("connection", { + where: (b: AnyCb) => + b.and( + byOwner(key.owner as Owner)(b), + b("integration", "=", key.integration), + b("name", "=", key.connection), + ), + set: { tools_synced_at: null }, + }) + .pipe(Effect.ignore), + { discard: true }, + ).pipe( + Effect.flatMap(() => + Effect.fail( + new StorageError({ + message: "tool catalog changed between reads", + cause: undefined, + }), + ), + ), + ), + ), + ); + const policyRules = policyRulesForScope; + const defsByConnection = new Map>(); + for (const def of definitionRows) { + const key = connectionKey(def); + let bucket = defsByConnection.get(key); + if (!bucket) { + bucket = new Map(); + defsByConnection.set(key, bucket); + } + bucket.set(def.name, decodeJsonColumn(def.schema)); + } + const EMPTY_DEFS: ReadonlyMap = new Map(); + + const projections: ToolProjection[] = []; + for (const row of rows) { + const tool = rowToTool(row); + if (!matchesToolFilter(tool, filter)) continue; + const effective = yield* resolvePolicyFromRuleSet( + normalizedPolicyId(tool), + policyRules, + tool.annotations?.requiresApproval, + ); + if (effective.action === "block") continue; + // The rule set already folded the plugin default in (`liftPlugin`), + // and an explicit user `approve` overrides it — the same answer + // `approvalRequired` gives on the invoke path. + const policy = + effective.action === "require_approval" + ? ("require_approval" as const) + : ("approve" as const); + const defs = defsByConnection.get(connectionKey(row)) ?? EMPTY_DEFS; + const inputSchema = + tool.inputSchema === undefined ? undefined : reattachDefs(tool.inputSchema, defs); + const readOnly = tool.annotations?.readOnly; + projections.push({ + address: tool.address, + integration: String(tool.integration), + owner: tool.owner, + connection: String(tool.connection), + name: String(tool.name), + description: tool.description, + ...(inputSchema === undefined ? {} : { inputSchema }), + policy, + ...(typeof readOnly === "boolean" ? { readOnly } : {}), + }); + } + return projections; + }).pipe(Effect.withSpan("executor.tools.describe_all")); + // ------------------------------------------------------------------ // Providers // ------------------------------------------------------------------ @@ -7137,6 +7585,7 @@ export const createExecutor = ( updateMany: (name, value) => db.updateMany(table(name), value), upsert: (name, value) => db.upsert(table(name), value), upsertMany: (name, value) => db.upsertMany(table(name), value), + replaceMany: (plan) => + db.replaceMany({ + ...(plan.guard ? { guard: { ...plan.guard, table: table(plan.guard.table) } } : {}), + deletes: plan.deletes.map((del) => ({ ...del, table: table(del.table) })), + inserts: plan.inserts.map((ins) => ({ ...ins, table: table(ins.table) })), + }), }; return Object.freeze(query); diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index 492b7420d4..774e82ba68 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -112,7 +112,12 @@ export type { Tool, ToolDef, ToolListFilter, ToolAnnotations } from "./tool"; export type { CredentialProvider, ProviderEntry } from "./provider"; // Public projections / detection. -export { ToolSchemaView, ToolAnnotationsView, IntegrationDetectionResult } from "./types"; +export { + ToolSchemaView, + ToolAnnotationsView, + ToolProjection, + IntegrationDetectionResult, +} from "./types"; // Health-check vocabulary (pure Schema + helpers). export { @@ -387,6 +392,7 @@ export { type OwnerBinding, type ToolPolicyProvider, type ToolPolicyProviderRule, + type PreparedToolPolicy, type IntegrationRecord, type StaticIntegrationDecl, type StaticToolDecl, diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 2ace32f891..a400f4b1e0 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -131,13 +131,36 @@ export interface ToolPolicyProvider { * requests), so caching on it would serve stale policy state. Each operation * gets a fresh snapshot. */ - readonly prepare?: () => Effect.Effect< - (input: { - readonly toolId: string; - readonly defaultRequiresApproval?: boolean; - }) => EffectivePolicy, - StorageFailure - >; + readonly prepare?: () => Effect.Effect; +} + +/** + * What one `prepare()` call hands back: a pure per-tool resolver and, for + * providers that can answer it, a pure connection-scope predicate — BOTH + * derived from the same storage snapshot, so a grant that changes between + * two reads cannot make the resolver serve a connection the predicate has + * just declared unservable (or vice versa). + */ +export interface PreparedToolPolicy { + readonly resolve: (input: { + readonly toolId: string; + readonly defaultRequiresApproval?: boolean; + }) => EffectivePolicy; + /** + * Can ANY tool of this connection be visible through this provider? + * Providers that are capability allowlists over connection patterns + * (toolkits) answer from real pattern overlap — a grant of + * `acme.org.main.issues.*` means yes for `acme/org/main` even though the + * connection-wide wildcard id would not match it. Core uses this to leave + * a connection's catalog out of a consistency check when nothing under it + * could ever be served; a provider that omits it keeps every connection + * under strict validation. + */ + readonly canServeConnection?: (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }) => boolean; } // --------------------------------------------------------------------------- diff --git a/packages/core/sdk/src/policies.test.ts b/packages/core/sdk/src/policies.test.ts index 1b5683aeba..fc64d0ecc3 100644 --- a/packages/core/sdk/src/policies.test.ts +++ b/packages/core/sdk/src/policies.test.ts @@ -773,3 +773,67 @@ describe("approve / require_approval interaction with annotations", () => { }), ); }); + +// --------------------------------------------------------------------------- +// tools.describeAll — the one-pass projection the passthrough MCP surface +// serves: every visible tool with its resolved policy, in a single read of the +// catalog + rule set. The fixture plugin marks `vercel.delete` as requiring +// approval by annotation; user rules layer on top exactly as they do for +// `tools.list` and `execute`. +// --------------------------------------------------------------------------- + +describe("executor.tools.describeAll", () => { + it.effect("returns every visible tool with the effective policy folded in", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + const all = yield* executor.tools.describeAll(); + const byAddress = new Map(all.map((tool) => [String(tool.address), tool])); + expect([...byAddress.keys()].sort()).toEqual([ + String(addr(GITHUB, "list")), + String(addr(VERCEL, "delete")), + String(addr(VERCEL, "deploy")), + ]); + // Plugin default: `delete` requires approval, `deploy` does not. + expect(byAddress.get(String(addr(VERCEL, "delete")))?.policy).toBe("require_approval"); + expect(byAddress.get(String(addr(VERCEL, "deploy")))?.policy).toBe("approve"); + expect(byAddress.get(String(addr(GITHUB, "list")))?.policy).toBe("approve"); + // The projection carries the routing triple a name mangler needs. + const deploy = byAddress.get(String(addr(VERCEL, "deploy")))!; + expect(deploy.integration).toBe("vercel"); + expect(deploy.owner).toBe("org"); + expect(deploy.connection).toBe(String(CONN)); + expect(deploy.name).toBe("deploy"); + }), + ); + + it.effect("omits blocked tools and honours require_approval / approve rules", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + yield* executor.policies.create({ owner: "org", pattern: "github.*", action: "block" }); + yield* executor.policies.create({ + owner: "org", + pattern: "vercel.*.*.deploy", + action: "require_approval", + }); + // An explicit approve overrides the plugin's requiresApproval default. + yield* executor.policies.create({ + owner: "org", + pattern: "vercel.*.*.delete", + action: "approve", + }); + const all = yield* executor.tools.describeAll(); + const byName = new Map(all.map((tool) => [`${tool.integration}.${tool.name}`, tool.policy])); + expect(byName.has("github.list")).toBe(false); + expect(byName.get("vercel.deploy")).toBe("require_approval"); + expect(byName.get("vercel.delete")).toBe("approve"); + }), + ); + + it.effect("narrows by integration like tools.list", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + const only = yield* executor.tools.describeAll({ integration: GITHUB }); + expect(only.map((tool) => tool.name)).toEqual(["list"]); + }), + ); +}); diff --git a/packages/core/sdk/src/tool.ts b/packages/core/sdk/src/tool.ts index c3bdb64f65..7e3aa0ab3d 100644 --- a/packages/core/sdk/src/tool.ts +++ b/packages/core/sdk/src/tool.ts @@ -12,6 +12,10 @@ export interface ToolAnnotations { readonly requiresApproval?: boolean; readonly approvalDescription?: string; readonly mayElicit?: boolean; + /** The tool never mutates upstream state (a GET, a GraphQL query, an + * upstream MCP `readOnlyHint`). Set only by plugins that can tell; absent + * means unknown, and a surface must not read it as "mutating". */ + readonly readOnly?: boolean; } /** A tool as produced by a plugin — the definition, no address yet (the SDK diff --git a/packages/core/sdk/src/types.ts b/packages/core/sdk/src/types.ts index 3a34664b04..fa52889e41 100644 --- a/packages/core/sdk/src/types.ts +++ b/packages/core/sdk/src/types.ts @@ -44,6 +44,34 @@ export const ToolSchemaView = Schema.Struct({ }); export type ToolSchemaView = typeof ToolSchemaView.Type; +// --------------------------------------------------------------------------- +// ToolProjection — one visible tool as an MCP passthrough surface serves it: +// the address, the resolved policy, and a SELF-CONTAINED input schema (shared +// `$defs` re-attached), returned by `executor.tools.describeAll()`. Built in +// one pass over the catalog so a workspace with thousands of tools costs a +// handful of reads, not one round-trip per tool. +// --------------------------------------------------------------------------- + +export const ToolProjection = Schema.Struct({ + address: ToolAddress, + integration: Schema.String, + owner: Schema.Literals(["org", "user"]), + connection: Schema.String, + name: Schema.String, + description: Schema.String, + /** JSON Schema with every referenced `$def` inlined under `$defs`. Absent + * when the tool declares no input. */ + inputSchema: Schema.optional(Schema.Unknown), + /** The effective policy action for this caller. `block`ed tools are never + * returned, so this is `approve` or `require_approval`. */ + policy: Schema.Literals(["approve", "require_approval"]), + /** Whether the plugin marks the tool as never mutating upstream state. Only + * plugins that know (HTTP method, GraphQL operation kind, upstream MCP hint) + * set it; absent means unknown. */ + readOnly: Schema.optional(Schema.Boolean), +}); +export type ToolProjection = typeof ToolProjection.Type; + // --------------------------------------------------------------------------- // Integration detection — optional capability on `PluginSpec.detect`. When a // user pastes a URL in the onboarding UI, `executor.integrations.detect(url)` diff --git a/packages/plugins/graphql/src/sdk/plugin.ts b/packages/plugins/graphql/src/sdk/plugin.ts index 176e6d7721..fd3adfdcbc 100644 --- a/packages/plugins/graphql/src/sdk/plugin.ts +++ b/packages/plugins/graphql/src/sdk/plugin.ts @@ -570,9 +570,11 @@ const annotationsFor = (binding: OperationBinding): ToolAnnotations => { return { requiresApproval: true, approvalDescription: `mutation ${binding.fieldName}`, + readOnly: false, }; } - return {}; + // A query is the read side of GraphQL by definition. + return { readOnly: true }; }; // --------------------------------------------------------------------------- diff --git a/packages/plugins/mcp/src/sdk/plugin.ts b/packages/plugins/mcp/src/sdk/plugin.ts index 4af0c4bf58..77aafdb628 100644 --- a/packages/plugins/mcp/src/sdk/plugin.ts +++ b/packages/plugins/mcp/src/sdk/plugin.ts @@ -543,9 +543,13 @@ const toToolDef = (entry: McpToolManifestEntry): ToolDef => { ...(entry.annotations ? { upstream: entry.annotations } : {}), ...(entry._meta ? { _meta: entry._meta } : {}), }; + const readOnly = entry.annotations?.readOnlyHint; const annotations: StampedAnnotations = { requiresApproval: destructive, ...(destructive ? { approvalDescription: entry.annotations?.title ?? entry.toolName } : {}), + // Carry the upstream read-only hint through verbatim: a passthrough MCP + // surface re-advertises it, and only the upstream server knows. + ...(typeof readOnly === "boolean" ? { readOnly } : {}), mcp: stamp, }; return { diff --git a/packages/plugins/openapi/src/sdk/invoke.ts b/packages/plugins/openapi/src/sdk/invoke.ts index 8d180e6952..c512b65ab1 100644 --- a/packages/plugins/openapi/src/sdk/invoke.ts +++ b/packages/plugins/openapi/src/sdk/invoke.ts @@ -1467,11 +1467,15 @@ export const REQUIRE_APPROVAL = new Set(["post", "put", "patch", "delete"]); export const annotationsForOperation = ( method: string, pathTemplate: string, -): { requiresApproval?: boolean; approvalDescription?: string } => { +): { requiresApproval?: boolean; approvalDescription?: string; readOnly?: boolean } => { const m = method.toLowerCase(); - if (!REQUIRE_APPROVAL.has(m)) return {}; + // A safe method (GET/HEAD/OPTIONS) is read-only by HTTP semantics; a + // passthrough MCP surface advertises that as `readOnlyHint` so a harness can + // skip its own confirmation for it. + if (!REQUIRE_APPROVAL.has(m)) return { readOnly: true }; return { requiresApproval: true, approvalDescription: `${method.toUpperCase()} ${pathTemplate}`, + readOnly: false, }; }; diff --git a/packages/plugins/toolkits/src/server.test.ts b/packages/plugins/toolkits/src/server.test.ts index bab67eb0e9..3b8bb0dcf7 100644 --- a/packages/plugins/toolkits/src/server.test.ts +++ b/packages/plugins/toolkits/src/server.test.ts @@ -199,4 +199,47 @@ describe("toolkitsPlugin", () => { ).toContain("google_docs.org.* approve"); }), ); + + // "Can any tool of this connection be visible?" must come from real grant + // overlap. A grant NARROWER than the connection (`github.org.main.issues.*`) + // still makes the connection servable, even though the connection-wide + // wildcard id `github.org.main.*` would not match that pattern — which is + // exactly the trap a synthetic-id probe falls into. + it.effect("prepareConnectionScope answers from grant overlap, not a synthetic wildcard id", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [toolkitsPlugin()] as const, + }); + const toolkit = yield* executor.toolkits.create({ owner: "org", name: "Narrow Kit" }); + yield* executor.toolkits.createConnection(toolkit.id, { + pattern: "github.org.main.issues.*", + }); + const canServe = yield* executor.toolkits.prepareConnectionScopeForSlug(toolkit.slug); + + // The narrowly granted connection IS servable. + expect(canServe({ integration: "github", owner: "org", name: "main" })).toBe(true); + // Sibling connections and other integrations are not. + expect(canServe({ integration: "github", owner: "org", name: "other" })).toBe(false); + expect(canServe({ integration: "linear", owner: "org", name: "main" })).toBe(false); + // An org toolkit never serves a personal connection. + expect(canServe({ integration: "github", owner: "user", name: "main" })).toBe(false); + + // Segment wildcards in the grant head still resolve. + const wide = yield* executor.toolkits.create({ owner: "org", name: "Wide Kit" }); + yield* executor.toolkits.createConnection(wide.id, { pattern: "github.*.*.issues.*" }); + const canServeWide = yield* executor.toolkits.prepareConnectionScopeForSlug(wide.slug); + expect(canServeWide({ integration: "github", owner: "org", name: "anything" })).toBe(true); + expect(canServeWide({ integration: "linear", owner: "org", name: "main" })).toBe(false); + + // The universal grant serves everything the owner model allows. + const all = yield* executor.toolkits.create({ owner: "org", name: "All Kit" }); + yield* executor.toolkits.createConnection(all.id, { pattern: "*" }); + const canServeAll = yield* executor.toolkits.prepareConnectionScopeForSlug(all.slug); + expect(canServeAll({ integration: "linear", owner: "org", name: "main" })).toBe(true); + + // An unknown toolkit serves nothing. + const none = yield* executor.toolkits.prepareConnectionScopeForSlug("no-such-kit"); + expect(none({ integration: "github", owner: "org", name: "main" })).toBe(false); + }), + ); }); diff --git a/packages/plugins/toolkits/src/server.ts b/packages/plugins/toolkits/src/server.ts index 7dacc3e456..9aadaee4f7 100644 --- a/packages/plugins/toolkits/src/server.ts +++ b/packages/plugins/toolkits/src/server.ts @@ -13,6 +13,7 @@ import { type PluginStorageFacade, type PluginStorageCollectionFacade, type StorageFailure, + type PreparedToolPolicy, type ToolPolicyAction, type ToolPolicyProvider, type ToolPolicyProviderRule, @@ -516,32 +517,77 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { // policies + connections on every tool, which is the per-tool N+1 that scales // with the whole catalog on the list surface. This is byte-for-byte the same // resolution, just hoisted out of the loop. + // Can any tool of this connection be visible through the toolkit? True + // when any granting pattern (a connection grant, or a legacy policy that + // acts as one) is UNDER the connection — i.e. begins with + // `..` followed by `.` or is exactly that prefix + // with a trailing wildcard — or is the universal `*`. A pattern rooted + // elsewhere can never match a tool id of this connection, whatever its + // remaining segments say. + const connectionScopeFrom = ( + isOrg: boolean, + policies: readonly ToolkitPolicyRecord[], + connections: readonly ToolkitConnectionRecord[], + ) => { + const legacyPolicyIds = legacyConnectionPolicyIds(policies, connections); + const grants = [ + ...connections.map((connection) => connection.pattern), + ...policies.filter((policy) => legacyPolicyIds.has(policy.id)).map((p) => p.pattern), + ]; + return (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }): boolean => { + if (isOrg && connection.owner === "user") return false; + const root = `${connection.integration}.${connection.owner}.${connection.name}`; + return grants.some((pattern) => { + if (pattern === "*") return true; + // A grant under the connection root, or a segment-wildcarded grant + // that still matches the root's three segments. + if (pattern === root || pattern.startsWith(`${root}.`)) return true; + const head = pattern.split(".").slice(0, 3).join("."); + return matchPattern(`${head}.*`, `${root}.x`); + }); + }; + }; + + // ONE read of the toolkit's policies + connections yields BOTH the per-tool + // resolver and the connection-scope predicate, so the two can never + // disagree about a grant that changed between reads. const preparePolicyResolverForSlug = ( slug: string, - ): Effect.Effect< - (input: { - readonly toolId: string; - readonly defaultRequiresApproval?: boolean; - }) => EffectivePolicy, - StorageFailure - > => + ): Effect.Effect => Effect.gen(function* () { const toolkit = yield* getBySlugEntry(slug); - if (!toolkit) return () => blockedPolicy(); + if (!toolkit) return { resolve: () => blockedPolicy(), canServeConnection: () => false }; const isOrg = toolkit.owner === "org"; const policies = yield* listPoliciesForRecord(toolkit.data.id); const connections = yield* listConnectionsForRecord(toolkit.data.id); - return (input: { readonly toolId: string; readonly defaultRequiresApproval?: boolean }) => { - if (isOrg && isPersonalDynamicToolId(input.toolId)) return blockedPolicy(); - return resolveToolkitPolicy( - input.toolId, - connections, - policies, - input.defaultRequiresApproval, - ); + return { + resolve: (input: { + readonly toolId: string; + readonly defaultRequiresApproval?: boolean; + }) => { + if (isOrg && isPersonalDynamicToolId(input.toolId)) return blockedPolicy(); + return resolveToolkitPolicy( + input.toolId, + connections, + policies, + input.defaultRequiresApproval, + ); + }, + canServeConnection: connectionScopeFrom(isOrg, policies, connections), }; }); + /** The scope predicate alone, for callers that want only it. Same snapshot + * rules as `preparePolicyResolverForSlug`, from which it is derived. */ + const prepareConnectionScopeForSlug = (slug: string) => + preparePolicyResolverForSlug(slug).pipe( + Effect.map((prepared) => prepared.canServeConnection ?? (() => false)), + ); + return { list, create, @@ -561,6 +607,7 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { policyRulesForSlug, resolvePolicyForSlug, preparePolicyResolverForSlug, + prepareConnectionScopeForSlug, }; }; @@ -682,7 +729,8 @@ const makePolicyProvider = ( resolve: ({ toolId, defaultRequiresApproval }) => extension.resolvePolicyForSlug(slug, toolId, defaultRequiresApproval), // Preferred path: core calls this once per operation, so the toolkit's - // policies + connections are fetched once instead of once per tool. + // policies + connections are fetched once instead of once per tool — and + // the connection-scope predicate rides on the same snapshot. prepare: () => extension.preparePolicyResolverForSlug(slug), });