diff --git a/.githooks/pre-commit b/.githooks/pre-commit index b9202975..1cfc9f32 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -3,3 +3,4 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" "$ROOT/scripts/verify-no-secrets.sh" --staged "$ROOT/scripts/generate-connector-contract.swift" --check +"$ROOT/scripts/generate-script-exec-contract.swift" --check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 755f87f3..96feb39e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,8 @@ Thank you for your interest in contributing. Please read the [Code of Conduct](C ## Requirements - **macOS** with **Xcode 27** (Swift 6.4+) -- **Docker Desktop** (Python guest runtime, web crawler, and file extractor images) +- **Docker Desktop** (Go worker image for guests, web crawler, and file extractor) +- **Go 1.27.1+** (`brew install go`) for local diagnostics; guest compile runs in Docker - Apple Developer account for code signing ## Getting started diff --git a/SECURITY.md b/SECURITY.md index 470cbe9b..02096041 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,7 +18,7 @@ We will acknowledge receipt and work on a fix before public disclosure when poss Derrick runs LLM agents with tools on the user's Mac. The design assumes: - **Untrusted model output** — tools and scripts are gated before execution. -- **Untrusted guest code** — Python plugins/scripts run in Docker with no network; the host performs HTTP. +- **Untrusted guest code** — Go plugins/scripts compile and run in Docker with no network; the host performs HTTP. - **Untrusted remote content** — fetched HTML and tool output are sanitized before display. - **Secrets stay on the host** — API keys and connector tokens live in Keychain or local `.env` (dev only); they are not injected into guest containers. @@ -26,7 +26,7 @@ Derrick runs LLM agents with tools on the user's Mac. The design assumes: | Layer | Mechanism | |-------|-----------| -| Script execution | Docker `--network none`, static Python verifier, LLM script reviewer | +| Script execution | Docker `--network none`, static Go verifier, in-container compile, LLM script reviewer | | Network egress | Host HTTP client, egress blacklist, user approval for new destinations | | Plugins | Factory build + review; hop-limited `http.request`; Keychain-attached auth | | Inter-process | Code-signed XPC peers, HMAC-signed service messages (release: Keychain secret) | diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index d7da35c2..8d30ce1e 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -23,7 +23,7 @@ Derrick integrates with user-configured APIs. You supply your own keys and are s ## Runtime -- **Docker Desktop** — script and plugin execution use the `python:3.14.7` guest image (see `docs/adr-swift-script-runtime.md` for superseded Swift guest notes). +- **Docker Desktop** — script and plugin execution use the `derrick-worker:go-v1` guest image. ## Project license diff --git a/docker/guest-runtime/Dockerfile b/docker/guest-runtime/Dockerfile deleted file mode 100644 index 5c99ce18..00000000 --- a/docker/guest-runtime/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Offline guest runtime for script_exec and plugin.invoke (Python primary). -# Build: docker build -f docker/guest-runtime/Dockerfile -t derrick-guest-runtime:python-v1 . -FROM python:3.14.7 - -RUN useradd --create-home --uid 10001 guest - -# uv — fast, lockfile-friendly dependency installs inside the guest image. -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv - -WORKDIR /home/guest -USER guest - -# Guests run with --network none; deps are baked into the image or installed at build time. -ENV UV_LINK_MODE=copy \ - PYTHONUNBUFFERED=1 diff --git a/docker/worker/.dockerignore b/docker/worker/.dockerignore new file mode 100644 index 00000000..b01fe143 --- /dev/null +++ b/docker/worker/.dockerignore @@ -0,0 +1,3 @@ +**/.git +**/.build +**/node_modules diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile new file mode 100644 index 00000000..0e11fc1a --- /dev/null +++ b/docker/worker/Dockerfile @@ -0,0 +1,33 @@ +# Unified Go worker image: web crawl, file extract (plugin guest binaries are copied per invoke). +# Build: docker build -f docker/worker/Dockerfile -t derrick-worker:go-v1 . +FROM golang:1.27.1 AS build + +WORKDIR /src +COPY workers/go/go.mod workers/go/go.sum ./ +RUN go mod download +COPY workers/go ./ + +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-web-crawler ./cmd/derrick-crawler +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-file-extractor ./cmd/derrick-file-extractor + +FROM debian:bookworm-slim + +LABEL derrick.worker.binaries="crawler,extractor" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends poppler-utils ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 worker + +COPY --from=build /usr/local/go /usr/local/go +COPY --from=build /out/derrick-web-crawler /usr/local/bin/derrick-web-crawler +COPY --from=build /out/derrick-file-extractor /usr/local/bin/derrick-file-extractor +RUN mkdir -p /data/in /data/out && chown -R worker:worker /data + +ENV PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV GOROOT=/usr/local/go +ENV GOTOOLCHAIN=local +ENV CGO_ENABLED=0 + +USER worker +WORKDIR /home/worker diff --git a/docs/Design.md b/docs/Design.md index 6bcc9bf7..5a9171d9 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -10,4 +10,4 @@ This application is Protocol first. All major features must have a Protocol and ## Plugins - Plugins are using the Agent Plugin standard -- Plugins run in the plugins docker containers as either Python or Go (Go is not yet supported) +- Plugins and `script_exec` run in the unified Go worker Docker image (`derrick-worker:go-v1`) diff --git a/master-todo.md b/master-todo.md index c6e09ec3..9cd98b98 100644 --- a/master-todo.md +++ b/master-todo.md @@ -77,7 +77,7 @@ Open: binary convert (xlsx) has no “here is your file” UI yet. Do not delete Each `script_exec` / `plugin.invoke`: 1. Wait for the offline queue (max 1). -2. `docker create` a unique `derrick-guest-runtime-` from `python:3.14.7`. +2. `docker create` a unique `derrick-guest-runtime-` from `derrick-worker:go-v1`. 3. Run until the host hop loop is done (terminal envelope, error, or in-use lease TTL). 4. `docker rm -f` immediately — that is “I’m done.” 5. Release the queue slot so the next script can create at once. diff --git a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift b/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift deleted file mode 100644 index 1490c869..00000000 --- a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift +++ /dev/null @@ -1,157 +0,0 @@ -import Foundation -import SQLite3 -import Structure - -public extension DBRepository { - func upsertNewsReader(_ spec: NewsReaderSpec) throws { - let topicsData = try JSONEncoder().encode(spec.topics) - let sourcesData = try JSONEncoder().encode(spec.sources) - let topics = String(data: topicsData, encoding: .utf8) ?? "[]" - let sources = String(data: sourcesData, encoding: .utf8) ?? "[]" - try withDatabaseHandle { handle in - try Self.execute(""" - INSERT INTO news_readers ( - id, name, topics_json, sources_json, mode, max_count, schedule, - last_error, last_fetched_at, created_at, updated_at - ) VALUES ( - \(quoted(spec.id)), - \(quoted(spec.name)), - \(quoted(topics)), - \(quoted(sources)), - \(quoted(spec.mode.rawValue)), - \(spec.maxCount), - \(quoted(spec.schedule.rawValue)), - \(sqlValue(spec.lastError)), - \(sqlValue(spec.lastFetchedAt.map { Self.iso8601Formatter().string(from: $0) })), - \(quoted(Self.iso8601Formatter().string(from: spec.createdAt))), - \(quoted(Self.iso8601Formatter().string(from: spec.updatedAt))) - ) - ON CONFLICT(id) DO UPDATE SET - name = excluded.name, - topics_json = excluded.topics_json, - sources_json = excluded.sources_json, - mode = excluded.mode, - max_count = excluded.max_count, - schedule = excluded.schedule, - last_error = excluded.last_error, - last_fetched_at = excluded.last_fetched_at, - updated_at = excluded.updated_at; - """, on: handle) - } - } - - func listNewsReaders() throws -> [NewsReaderSpec] { - try withDatabaseHandle { handle in - let sql = """ - SELECT id, name, topics_json, sources_json, mode, max_count, schedule, - last_error, last_fetched_at, created_at, updated_at - FROM news_readers - ORDER BY updated_at DESC; - """ - var statement: OpaquePointer? - guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else { - throw Self.sqliteError(handle: handle, fallback: "Failed to list news readers.") - } - defer { sqlite3_finalize(statement) } - var rows: [NewsReaderSpec] = [] - while sqlite3_step(statement) == SQLITE_ROW { - rows.append(try decodeNewsReader(statement: statement)) - } - return rows - } - } - - func deleteNewsReader(id: String) throws { - try withDatabaseHandle { handle in - try Self.execute("DELETE FROM news_readers WHERE id = \(quoted(id));", on: handle) - } - } - - func replaceNewsItems(readerID: String, items: [NewsItem]) throws { - try withDatabaseHandle { handle in - try Self.withImmediateTransaction(on: handle) { - try Self.execute("DELETE FROM news_items WHERE reader_id = \(quoted(readerID));", on: handle) - for item in items { - try Self.execute(""" - INSERT INTO news_items ( - id, reader_id, title, source_url, source_label, summary, published_at, fetched_at - ) VALUES ( - \(quoted(item.id)), - \(quoted(item.readerID)), - \(quoted(item.title)), - \(quoted(item.sourceURL)), - \(quoted(item.sourceLabel)), - \(sqlValue(item.summary)), - \(sqlValue(item.publishedAt.map { Self.iso8601Formatter().string(from: $0) })), - \(quoted(Self.iso8601Formatter().string(from: item.fetchedAt))) - ); - """, on: handle) - } - } - } - } - - func listNewsItems(readerID: String) throws -> [NewsItem] { - try withDatabaseHandle { handle in - let sql = """ - SELECT id, reader_id, title, source_url, source_label, summary, published_at, fetched_at - FROM news_items - WHERE reader_id = \(quoted(readerID)) - ORDER BY fetched_at DESC; - """ - var statement: OpaquePointer? - guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else { - throw Self.sqliteError(handle: handle, fallback: "Failed to list news items.") - } - defer { sqlite3_finalize(statement) } - var rows: [NewsItem] = [] - while sqlite3_step(statement) == SQLITE_ROW { - rows.append(try decodeNewsItem(statement: statement)) - } - return rows - } - } - - private func decodeNewsReader(statement: OpaquePointer) throws -> NewsReaderSpec { - func text(_ index: Int32) -> String { - String(cString: sqlite3_column_text(statement, index)) - } - func optionalText(_ index: Int32) -> String? { - sqlite3_column_type(statement, index) == SQLITE_NULL ? nil : text(index) - } - let topics = (try? JSONDecoder().decode([String].self, from: Data(text(2).utf8))) ?? [] - let sources = (try? JSONDecoder().decode([NewsSource].self, from: Data(text(3).utf8))) ?? [] - return NewsReaderSpec( - id: text(0), - name: text(1), - topics: topics, - sources: sources, - mode: NewsReaderMode(rawValue: text(4)) ?? .list, - maxCount: Int(sqlite3_column_int(statement, 5)), - schedule: NewsReaderSchedule(rawValue: text(6)) ?? .off, - lastError: optionalText(7), - lastFetchedAt: optionalText(8).flatMap { Self.iso8601Formatter().date(from: $0) }, - createdAt: Self.iso8601Formatter().date(from: text(9)) ?? .now, - updatedAt: Self.iso8601Formatter().date(from: text(10)) ?? .now - ) - } - - private func decodeNewsItem(statement: OpaquePointer) throws -> NewsItem { - func text(_ index: Int32) -> String { - String(cString: sqlite3_column_text(statement, index)) - } - func optionalText(_ index: Int32) -> String? { - sqlite3_column_type(statement, index) == SQLITE_NULL ? nil : text(index) - } - return NewsItem( - id: text(0), - readerID: text(1), - title: text(2), - sourceURL: text(3), - sourceLabel: text(4), - summary: optionalText(5), - publishedAt: optionalText(6).flatMap { Self.iso8601Formatter().date(from: $0) }, - fetchedAt: Self.iso8601Formatter().date(from: text(7)) ?? .now - ) - } -} diff --git a/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift b/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift index 6e3121c2..b2aee8fa 100644 --- a/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift +++ b/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift @@ -2,7 +2,7 @@ import Foundation import Structure public enum DatabaseSchema { - public static let latestVersion = 8 + public static let latestVersion = 9 public static func migrationSQL(version: Int, isUp: Bool) throws -> String { let migrationName = String(format: "%04d_%@", version, migrationFileBaseName(for: version)) @@ -39,6 +39,8 @@ public enum DatabaseSchema { return "messaging_agent_handled" case 8: return "messaging_thread_default_profile" + case 9: + return "drop_news_readers" default: return "unknown" } diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql index 6b4e5329..054ea0f3 100644 --- a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS news_readers ( mode TEXT NOT NULL, max_count INTEGER NOT NULL, schedule TEXT NOT NULL, + summary_text TEXT, last_error TEXT, last_fetched_at TEXT, created_at TEXT NOT NULL, diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql new file mode 100644 index 00000000..054ea0f3 --- /dev/null +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS news_readers ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + topics_json TEXT NOT NULL DEFAULT '[]', + sources_json TEXT NOT NULL DEFAULT '[]', + mode TEXT NOT NULL, + max_count INTEGER NOT NULL, + schedule TEXT NOT NULL, + summary_text TEXT, + last_error TEXT, + last_fetched_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS news_items ( + id TEXT PRIMARY KEY NOT NULL, + reader_id TEXT NOT NULL, + title TEXT NOT NULL, + source_url TEXT NOT NULL, + source_label TEXT NOT NULL, + summary TEXT, + published_at TEXT, + fetched_at TEXT NOT NULL, + UNIQUE(reader_id, source_url), + FOREIGN KEY(reader_id) REFERENCES news_readers(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_news_items_reader + ON news_items(reader_id, fetched_at DESC); diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql new file mode 100644 index 00000000..cbb5bff7 --- /dev/null +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS news_items; +DROP TABLE IF EXISTS news_readers; diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift deleted file mode 100644 index bbc475d1..00000000 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift +++ /dev/null @@ -1,60 +0,0 @@ -import XCTest -import Structure -@testable import DBRepository - -final class DBNewsReaderTests: XCTestCase { - func testNewsReaderRoundTripAndItems() async throws { - let repository = try makeRepository() - _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - - let spec = NewsReaderSpec( - name: "Markets", - topics: ["Financial", "rates"], - sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/world/rss.xml")], - mode: .summaries, - maxCount: 10, - schedule: .daily - ) - try await repository.upsertNewsReader(spec) - let listed = try await repository.listNewsReaders() - XCTAssertEqual(listed.count, 1) - XCTAssertEqual(listed[0].name, "Markets") - XCTAssertEqual(listed[0].topics, ["Financial", "rates"]) - XCTAssertEqual(listed[0].mode, .summaries) - - let item = NewsItem( - readerID: spec.id, - title: "Rates rise", - sourceURL: "https://example.com/rates", - sourceLabel: "BBC", - summary: "A summary" - ) - try await repository.replaceNewsItems(readerID: spec.id, items: [item]) - let items = try await repository.listNewsItems(readerID: spec.id) - XCTAssertEqual(items.count, 1) - XCTAssertEqual(items[0].sourceURL, "https://example.com/rates") - - try await repository.deleteNewsReader(id: spec.id) - let remainingReaders = try await repository.listNewsReaders() - let remainingItems = try await repository.listNewsItems(readerID: spec.id) - XCTAssertTrue(remainingReaders.isEmpty) - XCTAssertTrue(remainingItems.isEmpty) - } - - private func makeRepository() throws -> DBRepository { - let directory = FileManager.default.temporaryDirectory.appendingPathComponent( - UUID().uuidString, - isDirectory: true - ) - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - return DBRepository( - configuration: DBRepositoryConfiguration( - applicationName: "ui", - databaseName: "derrick", - databaseDirectoryURL: directory, - username: "app-user", - password: "app-secret" - ) - ) - } -} diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift index a546a2c0..343da150 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift @@ -81,23 +81,9 @@ final class DBRepositoryTests: XCTestCase { func testSchemaUpgradeDoesNotWipeExistingRows() async throws { let repository = try makeRepository() _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"keep-me"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), - "app.derrick/plugin": artifact, - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "keep-me", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: [:], - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "keep-me" ) try await repository.savePluginFactoryRelease(release) let url = await repository.databaseURL @@ -108,7 +94,7 @@ final class DBRepositoryTests: XCTestCase { _ = try await repository.migrateSessionMemory(username: "app-user", password: "app-secret") XCTAssertEqual(try schemaVersion(at: url), DatabaseSchema.latestVersion) - XCTAssertTrue(try tableExists(named: "news_readers", at: url)) + XCTAssertFalse(try tableExists(named: "news_readers", at: url)) XCTAssertTrue(try tableExists(named: "agent_profiles", at: url)) XCTAssertTrue(try tableExists(named: "messaging_agent_handled", at: url)) let loaded = try await repository.pluginFactoryRelease(pluginID: "keep-me", version: "1.0.0") @@ -119,25 +105,11 @@ final class DBRepositoryTests: XCTestCase { func testApprovedPluginFactoryReleasePersistsAndVerifies() async throws { let repository = try makeRepository() _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) let skillFiles = ["skills/weather/SKILL.md": "# Weather"] - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), - "app.derrick/plugin": artifact, - "skills/weather/SKILL.md": Data("# Weather".utf8), - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "weather-tool", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: skillFiles, - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "weather-tool", + skillFiles: skillFiles ) try await repository.savePluginFactoryRelease(release) @@ -164,25 +136,11 @@ final class DBRepositoryTests: XCTestCase { ) ) _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) let skillFiles = ["skills/weather/SKILL.md": "# Weather"] - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), - "app.derrick/plugin": artifact, - "skills/weather/SKILL.md": Data("# Weather".utf8), - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "weather-tool", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: skillFiles, - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "weather-tool", + skillFiles: skillFiles ) try await repository.savePluginFactoryRelease(release) @@ -633,4 +591,38 @@ final class DBRepositoryTests: XCTestCase { } return String(cString: c) } + + /// Hash must match `packageFiles()`, which uses `guestSource` at the Go guest path. + private func makeGoFactoryRelease( + pluginID: String, + manifestName: String, + skillFiles: [String: String] = [:] + ) -> PluginFactoryRelease { + let artifact = Data("compiled".utf8) + let guestSource = "package main" + let manifestJSON = "{\"name\":\"\(manifestName)\"}" + let runtimeJSON = #"{"language":"go"}"# + var files: [String: Data] = [ + "plugin.json": Data(manifestJSON.utf8), + "app.derrick/runtime.json": Data(runtimeJSON.utf8), + "app.derrick/plugin.go": Data(guestSource.utf8), + "app.derrick/plugin": artifact, + ] + for (path, body) in skillFiles { + files[path] = Data(body.utf8) + } + let release = PluginFactoryRelease( + pluginID: pluginID, + version: "1.0.0", + manifestJSON: manifestJSON, + runtimeJSON: runtimeJSON, + guestSource: guestSource, + compiledArtifact: artifact, + skillFiles: skillFiles, + contentHash: PluginContentHash.hash(files: files), + reviewSummary: "approved" + ) + XCTAssertTrue(release.verifyIntegrity(), "test fixture hash must match packageFiles()") + return release + } } diff --git a/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift b/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift index 22131acd..96c27f92 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift @@ -52,7 +52,7 @@ public actor DaemonRuntime { service: .daemon, status: ok ? .ok : .degraded, detail: ok ? nil : "not bootstrapped", - guestRuntimeImage: DerrickGuestRuntime.pythonGuestDockerImage, + guestRuntimeImage: DerrickGuestRuntime.guestDockerImage, executableFingerprint: DaemonSelfRetirement.launchedFingerprint ) } diff --git a/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift b/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift index e617f6a0..a74708e3 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift @@ -31,11 +31,15 @@ public enum HITLApprovalNotifier: Sendable { let isNetwork = isNetworkToolName(row.toolName) let host = host(fromNetworkToolName: row.toolName) - let title = isNetwork ? "Network access needed" : "Approval needed" + let blacklistPattern = blacklistPattern(from: row.argumentsJSON) + let title = isNetwork + ? (blacklistPattern == nil ? "Network access needed" : "Network blacklist") + : "Approval needed" let body: String - if isNetwork, let host { - let suffix = Self.registrableSuffix(for: host) - body = "Allow *.\(suffix)? Tap to approve or deny. Always Allow covers all subdomains." + if isNetwork, let blacklistPattern { + body = "This request matches blacklist \(blacklistPattern). Tap to allow this run, remove from blacklist, or deny." + } else if isNetwork, let host { + body = "Network access to \(host). Tap to approve or deny." } else { let preview = truncated(row.argumentsJSON, limit: 160) body = preview.isEmpty @@ -80,12 +84,13 @@ public enum HITLApprovalNotifier: Sendable { return host.isEmpty ? nil : host } - /// Last two labels — same rule as egress permanent allow (`*.apple.com` ← `securemetrics.apple.com`). - private static func registrableSuffix(for host: String) -> String { - let normalized = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - let parts = normalized.split(separator: ".").map(String.init) - guard parts.count >= 2 else { return normalized } - return parts.suffix(2).joined(separator: ".") + private static func blacklistPattern(from argumentsJSON: String) -> String? { + guard let data = argumentsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + object["kind"] as? String == "blacklist" else { + return nil + } + return object["pattern"] as? String } private static func truncated(_ text: String, limit: Int) -> String { diff --git a/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift b/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift index ce7af8ef..34a8199e 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift @@ -36,56 +36,70 @@ enum PluginFactoryCreateWorkflow { ) async throws -> MCPToolCallResultDTO ) async throws { let input = try PluginFactoryCreateInput.decodeJSON(request.inputJSON) - guard input.pluginType == .connector else { + guard input.pluginType == .connector || input.pluginType == .custom else { try await fail( workflowID: workflowID, stage: "type", - message: "Only connector plugins can be built in the factory. News lists are created from the News reader wizard.", + message: "This plugin type cannot be built in the factory.", repositoryProvider: repositoryProvider ) return } - guard let vendor = input.vendor else { + guard let pluginID = input.pluginID, !pluginID.isEmpty else { try await fail( workflowID: workflowID, - stage: "vendor", - message: "Choose a messaging vendor for this connector.", + stage: "name", + message: "Name this plugin before creating it.", repositoryProvider: repositoryProvider ) return } - guard let pluginID = input.pluginID, !pluginID.isEmpty else { + guard !input.description.isEmpty else { try await fail( workflowID: workflowID, - stage: "name", - message: "Name this connector before creating it.", + stage: "description", + message: "Describe what the plugin should do, then try again.", repositoryProvider: repositoryProvider ) return } - guard let auth = input.auth else { + + if input.pluginType == .custom { + try await runCustomBuild( + workflowID: workflowID, + request: request, + input: input, + pluginID: pluginID, + baseContext: baseContext, + repositoryProvider: repositoryProvider, + executeTool: executeTool + ) + return + } + + guard let vendor = input.vendor else { try await fail( workflowID: workflowID, - stage: "auth", - message: "Save the connector credentials before creating it.", + stage: "vendor", + message: "Could not determine which messaging service this plugin targets.", repositoryProvider: repositoryProvider ) return } - guard auth.authScheme.isSupportedInWizard else { + guard let auth = input.auth else { try await fail( workflowID: workflowID, stage: "auth", - message: "OAuth connectors are not available yet. Use a bot token or API key.", + message: "Save the plugin credentials before creating it.", repositoryProvider: repositoryProvider ) return } - guard !input.description.isEmpty else { + guard auth.authScheme.isSupportedInWizard else { try await fail( workflowID: workflowID, - stage: "description", - message: "Choose a vendor and create the connector again.", + stage: "auth", + message: "OAuth connectors are not available yet. Use a bot token or API key.", repositoryProvider: repositoryProvider ) return @@ -194,6 +208,87 @@ enum PluginFactoryCreateWorkflow { ) } + private static func runCustomBuild( + workflowID: String, + request: WorkflowStartRequest, + input: PluginFactoryCreateInput, + pluginID: String, + baseContext: ExecutionContextWire, + repositoryProvider: @escaping @Sendable () async throws -> DBRepository, + executeTool: @escaping ( + String, + String, + ExecutionContextWire, + ServicePrincipal, + String?, + String?, + String, + String + ) async throws -> MCPToolCallResultDTO + ) async throws { + try await log( + workflowID: workflowID, + stage: "factory", + message: "Writing SKILL.md, building the guest program, and running trial tests…", + repositoryProvider: repositoryProvider + ) + let goal = input.customBuildGoal() + let buildArgs = try buildArguments(goal: goal, hostManifest: nil) + let buildResult = try await executeTool( + AllowedMCPTool.pluginFactoryBuild.rawValue, + buildArgs, + baseContext, + request.principal, + request.helperAPIKey, + request.helperReviewerModelJSON, + workflowID, + "factory" + ) + if buildResult.isError { + try await fail( + workflowID: workflowID, + stage: "factory", + message: userFacingToolError(buildResult, fallback: "Plugin factory could not finish building the plugin."), + repositoryProvider: repositoryProvider + ) + return + } + + guard let summary = decodeBuildResult(buildResult.text), + summary.ok != false, + let savedID = summary.pluginID?.trimmingCharacters(in: .whitespacesAndNewlines), + !savedID.isEmpty + else { + let decoded = decodeBuildResult(buildResult.text) + let raw = decoded?.error + ?? decoded?.reviewSummary + ?? "Plugin factory did not return a saved plugin." + try await fail( + workflowID: workflowID, + stage: "factory", + message: PluginFactoryCreateFailureMessage.userFacing(raw), + repositoryProvider: repositoryProvider + ) + return + } + + let resultJSON = try JSONEncoder.service.encode( + PluginFactoryCreateResult( + pluginID: savedID, + version: summary.version ?? "1.0.0", + vendor: "custom", + reviewSummary: summary.reviewSummary ?? "" + ) + ) + let resultText = String(decoding: resultJSON, as: UTF8.self) + try await complete( + workflowID: workflowID, + message: "Plugin saved as /\(savedID).", + resultJSON: resultText, + repositoryProvider: repositoryProvider + ) + } + private static func crawlArguments( startURL: String, vendor: PluginFactoryCreateInput.ConnectorVendor, diff --git a/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift b/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift index c2c11edc..717f1f0b 100644 --- a/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift +++ b/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift @@ -542,10 +542,10 @@ import Testing _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["send_message"]}}} """ let guestSource = "import json, sys\njson.dump([], sys.stdout)" - let runtimeJSON = #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + let runtimeJSON = #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# let draft = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", diff --git a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift index 478c0adc..ec7cd578 100644 --- a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift +++ b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift @@ -93,11 +93,9 @@ public enum DockerRunRequestValidator: Sendable { return .disallowedDockerSubcommand(subcommand) } - if subcommand == "image" { - guard let second = dockerArgs.dropFirst().first, - DockerHostLaunch.allowedImageSubcommands.contains(second) else { - return .disallowedDockerSubcommand("image \(dockerArgs.dropFirst().first ?? "")") - } + if subcommand == "image", + let error = validateImageArguments(dockerArgs) { + return error } if subcommand == "exec", let error = validateExecArguments(dockerArgs) { @@ -206,20 +204,17 @@ public enum DockerRunRequestValidator: Sendable { case "sh": guard args.count == 3, args[1] == "-c", - args[2] == "cat > /tmp/guest.py" + args[2] == "cat > /tmp/guest && chmod +x /tmp/guest" + || args[2] == DockerWorkerRuntime.guestWriteSourceShell + || args[2] == DockerWorkerRuntime.guestCompileShell + || args[2] == DockerWorkerRuntime.guestReadBinaryShell else { return .disallowedDockerFlag("exec \(command)") } - case "python3": - guard args.count == 2, args[1] == "/tmp/guest.py" else { - return .disallowedDockerFlag("exec \(command)") - } - case "/usr/local/bin/derrick-web-crawler": - guard args == ["/usr/local/bin/derrick-web-crawler"] else { - return .disallowedDockerFlag("exec \(command)") - } - case "/usr/local/bin/derrick-file-extractor": - guard args == ["/usr/local/bin/derrick-file-extractor"] else { + case DockerWorkerRuntime.crawlerBinary, + DockerWorkerRuntime.extractorBinary, + DockerWorkerRuntime.guestBinaryPath: + guard args == [command] else { return .disallowedDockerFlag("exec \(command)") } default: @@ -244,12 +239,45 @@ public enum DockerRunRequestValidator: Sendable { guard args.count == 5 else { return .disallowedDockerFlag("build extra arguments") } - guard DockerProductImagePolicy.isAllowedWebCrawlerBuild( + if DockerProductImagePolicy.isAllowedWorkerBuild( + dockerfilePath: dockerfile, + imageTag: tag, + contextPath: context + ) { + return nil + } + if DockerProductImagePolicy.isAllowedWebCrawlerBuild( dockerfilePath: dockerfile, imageTag: tag, contextPath: context - ) else { - return .disallowedDockerFlag("build product image") + ) { + return nil + } + return .disallowedDockerFlag("build product image") + } + + private static func validateImageArguments( + _ dockerArgs: [String] + ) -> DockerRunRequestValidationError? { + let args = Array(dockerArgs.dropFirst()) + guard let second = args.first, + DockerHostLaunch.allowedImageSubcommands.contains(second) else { + return .disallowedDockerSubcommand("image \(args.first ?? "")") + } + guard second == "inspect" else { return nil } + if args.count == 2 { + return nil + } + guard args.count == 4, + args[1] == "--format", + args[2] == "{{.Id}}" else { + return .disallowedDockerFlag("image inspect") + } + let tag = args[3] + guard tag == DockerWorkerRuntime.image + || tag == DockerProductImagePolicy.webCrawlerImage + || tag == DerrickGuestRuntime.guestDockerImage else { + return .disallowedDockerFlag("image inspect tag") } return nil } diff --git a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift index e806f4ad..bd387784 100644 --- a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift +++ b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift @@ -211,8 +211,11 @@ struct DockerRunnerXPCTests { ["version"], ["image", "inspect", "img"], ["pull", "img"], - ["exec", "-i", "c", "python3", "/tmp/guest.py"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest.py"], + ["exec", "-i", "c", DockerWorkerRuntime.guestBinaryPath], + ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest && chmod +x /tmp/guest"], + ["exec", "-i", "c", "sh", "-c", DockerWorkerRuntime.guestWriteSourceShell], + ["exec", "c", "sh", "-c", DockerWorkerRuntime.guestCompileShell], + ["exec", "c", "sh", "-c", DockerWorkerRuntime.guestReadBinaryShell], ["exec", "-i", "c", "/usr/local/bin/derrick-web-crawler"], ["create", "--label", "app.derrick=runtime", "--entrypoint", "/bin/sleep", "--name", "c", "derrick-web-crawler:swift-6.4-v1", "infinity"], [ @@ -229,7 +232,7 @@ struct DockerRunnerXPCTests { "--memory", "1g", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", - "python:3.14.7", + DockerWorkerRuntime.image, "/bin/sleep", "infinity", ], @@ -256,13 +259,13 @@ struct DockerRunnerXPCTests { } } - @Test func rejectsInvalidPythonGuestExecCommands() { + @Test func rejectsInvalidGoGuestExecCommands() { for args in [ - ["exec", "-i", "c", "python3", "/tmp/other.py"], - ["exec", "-i", "c", "python3", "/tmp/guest.py", "extra"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/other.py"], + ["exec", "-i", "c", "python3", "/tmp/guest.go"], + ["exec", "-i", "c", DockerWorkerRuntime.guestBinaryPath, "extra"], + ["exec", "-i", "c", "sh", "-c", "cat > /tmp/other"], ["exec", "-i", "c", "sh", "-c", "rm -rf /"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest.py", "extra"], + ["exec", "-i", "c", "sh", "-c", "go build /tmp/plugin.go"], ["exec", "-i", "c", "swift", "/tmp/plugin.swift"], ["exec", "-i", "c", "/tmp/plugin"], ["exec", "-i", "c", "swiftc", "-O", "/tmp/plugin.swift", "-o", "/tmp/plugin"], @@ -354,7 +357,7 @@ struct DockerRunnerXPCTests { @Test func rejectsCreateWithoutRuntimeLabel() { let r = request(arguments: DockerHostLaunch.dockerCLIArguments([ - "create", "--name", "c", "python:3.14.7", "/bin/sleep", "infinity", + "create", "--name", "c", DockerWorkerRuntime.image, "/bin/sleep", "infinity", ])) #expect(DockerRunRequestValidator.validate(r) == .disallowedDockerFlag("create missing runtime label")) } diff --git a/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift b/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift index 0359bf12..dfe1be76 100644 --- a/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift +++ b/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift @@ -26,7 +26,7 @@ enum FactoryHarnessMain { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("FactoryHarness: building slack full-sync connector…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: DirectShellDocker.executor()) + let executor = GoPluginFactoryDockerExecutor(executor: DirectShellDocker.executor()) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 3) ).build( diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift index 889e2aac..e721d2c3 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift @@ -34,7 +34,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { """ The host already assigned plugin_id \(host.pluginID) and these secret ids: \ \(host.secrets.map(\.id).joined(separator: ", ")). \ - Return python_source and test_input_json. Do not pick a different plugin_id or secret ids. + Return go_source and test_input_json. Do not pick a different plugin_id or secret ids. """ ) } @@ -75,41 +75,8 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { private static func builderSystemPrompt(for userGoal: String) -> String { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. - Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), python_source (string), - test_input_json (string containing valid JSON — a serialized object, not prose), - skill_files (array of objects with path and body), - secrets (array of objects with id, label, and kind; required for connector plugins), - role (string, optional: "connector" or "standard"), - messaging_ops (array of strings, required for connector role). - plugin_id must use lowercase letters, numbers, hyphens, and dots only - (for example my-connector). Never use underscores in plugin_id. - If the plugin needs a username, password, token, or API key, declare them in secrets. - kind must be username, password, token, or api_key. id is a stable Keychain key - such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in python_source. - Set role to "connector" when the plugin sends and receives messages with an external - messaging service (any chat or mail connector). Omit role or use "standard" otherwise. - For role connector, include messaging_ops: an array of implemented ops - (send_message, poll_inbox, sync_threads). It must match the user goal scope and test_input_json. - Do not return manifest_json. The host creates the canonical Agent Plugin manifest. - If skill_files is not needed, return an empty array. Every skill file path must be exactly - skills//SKILL.md. Never use manifest.json or other paths in skill_files. - \(DerrickGuestPython.modelContract) + \(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) - Before returning the draft, self-check the implementation: - - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - - Match host responses by the emitted request_id. - - Use only the Python standard library (no pip, requests, urllib, socket, or subprocess). - - The direct test input must exercise the terminal result path with matching http_results fixtures. - For messaging connector plugins (role connector) that call a vendor HTTP API: - - Declare secrets in the manifest only. Never hard-code credentials. - - Parse each http_results body as JSON when the vendor returns JSON. - - When scope includes send_message, the final result.emit must include sent_message. - - Direct tests for poll_inbox must include a non-empty messages array; runtime empty messages with vendor success is success. - - When scope includes sync_threads, the final result.emit must include a non-empty threads array. - Each thread needs vendor_thread_id (opaque vendor ID) and title (human label for the host channel picker). - Emit only conversations the saved secret can access; for Slack skip channels where is_member is false. When vendor documentation is supplied in the user prompt, use it only to fill may_call HTTP details. """ } @@ -120,7 +87,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { "plugin_id": AgentSchema(type: .string), "version": AgentSchema(type: .string), "description": AgentSchema(type: .string), - "python_source": AgentSchema(type: .string), + "go_source": AgentSchema(type: .string), "test_input_json": AgentSchema(type: .string), "skill_files": AgentSchema( type: .array, @@ -149,7 +116,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { "messaging_ops": AgentSchema(type: .array, items: AgentSchema(type: .string)), ], required: [ - "plugin_id", "version", "description", "python_source", + "plugin_id", "version", "description", "go_source", "test_input_json", "skill_files", ] ) @@ -181,7 +148,7 @@ public actor LiveFactoryReviewer: PluginFactoryReviewer { test_input_json: \(String(decoding: draft.testInput, as: UTF8.self)) - Python source: + Go source: \(draft.guestSource) Direct test output: @@ -199,12 +166,8 @@ public actor LiveFactoryReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ - You are Derrick's independent plugin alignment and safety reviewer. - Return exactly one JSON object: - {"decision":"approved|rejected","summary":"...","findings":[ - {"severity":"info|warning|blocking","category":"alignment|safety|correctness|privacy|supplyChain","message":"..."} - ]} - Reject unsafe or non-deterministic code. Approve when direct test output matches fixtures through result.emit. + Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. + \(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) """ } diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift index 8e102534..54110bc1 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift @@ -41,7 +41,7 @@ public enum ReferenceSlackConnectorDraft { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ "description":"Slack messaging connector",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"permissions":["channels:history","channels:read","chat:write","groups:history","groups:read","im:history","im:read","mpim:history","mpim:read","users:read"],"messaging_ops":[\(opsJSON)]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"permissions":["channels:history","channels:read","chat:write","groups:history","groups:read","im:history","im:read","mpim:history","mpim:read","users:read"],"messaging_ops":[\(opsJSON)]}}} """ return PluginFactoryDraft( manifestJSON: manifestJSON, @@ -52,104 +52,278 @@ public enum ReferenceSlackConnectorDraft { } private static let fullSyncSource = """ - import json, sys - def emit(x): json.dump(x, sys.stdout, separators=(",", ":")) - def sorted_results(results): - seen = set() - out = [] - for item in sorted(results or [], key=lambda r: str(r.get("request_id",""))): - rid = str(item.get("request_id","")) - if rid and rid not in seen: - seen.add(rid) - out.append(item) + package main + + import ( + "encoding/json" + "os" + "sort" + ) + + func emit(v any) { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) + } + + func sortedResults(results []map[string]any) []map[string]any { + if len(results) == 0 { + return nil + } + sort.Slice(results, func(i, j int) bool { + left, _ := results[i]["request_id"].(string) + right, _ := results[j]["request_id"].(string) + return left < right + }) + seen := map[string]bool{} + out := []map[string]any{} + for _, item := range results { + rid, _ := item["request_id"].(string) + if rid == "" || seen[rid] { + continue + } + seen[rid] = true + out = append(out, item) + } return out - def message_from(msg, default_channel): - if not isinstance(msg, dict): return None - channel = msg.get("channel") or default_channel - ts = msg.get("ts") - if not channel or not ts: return None - thread_ts = msg.get("thread_ts") - parent = None - if thread_ts and str(thread_ts) != str(ts): - parent = str(thread_ts) - reply_count = int(msg.get("reply_count") or 0) - row = {"vendor_thread_id":channel,"vendor_message_id":str(ts),"direction":"inbound","sender":msg.get("user") or "slack","body":msg.get("text") or "","created_at":str(ts),"reply_count":reply_count} - if parent: + } + + func asMap(v any) map[string]any { + if m, ok := v.(map[string]any); ok { + return m + } + return map[string]any{} + } + + func asString(v any) string { + if s, ok := v.(string); ok { + return s + } + return "" + } + + func messageFrom(msg map[string]any, defaultChannel string) map[string]any { + channel := asString(msg["channel"]) + if channel == "" { + channel = defaultChannel + } + ts := asString(msg["ts"]) + if channel == "" || ts == "" { + return nil + } + threadTS := asString(msg["thread_ts"]) + parent := "" + if threadTS != "" && threadTS != ts { + parent = threadTS + } + replyCount := 0 + if n, ok := msg["reply_count"].(float64); ok { + replyCount = int(n) + } + row := map[string]any{ + "vendor_thread_id": channel, + "vendor_message_id": ts, + "direction": "inbound", + "sender": func() string { + if s := asString(msg["user"]); s != "" { + return s + } + return "slack" + }(), + "body": asString(msg["text"]), + "created_at": ts, + "reply_count": replyCount, + } + if parent != "" { row["parent_vendor_message_id"] = parent + } return row - def main(): - event = json.load(sys.stdin) - params = event.get("params") or {} - op = params.get("messaging_op") - if event.get("kind") == "manual" and op == "sync_threads": - emit([{"verb":"http.request","request_id":"sync-1","method":"GET","url":"https://slack.com/api/conversations.list?types=public_channel,private_channel&limit=200&exclude_archived=true","headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + + func main() { + var event map[string]any + if err := json.NewDecoder(os.Stdin).Decode(&event); err != nil { return - if event.get("kind") == "manual" and op == "poll_inbox": - channel = params.get("vendor_thread_id") or params.get("channel") - parent = params.get("parent_vendor_message_id") or params.get("thread_ts") - if not channel: - emit([{"verb":"result.emit","messages":[]}]) + } + params := asMap(event["params"]) + op := asString(params["messaging_op"]) + kind := asString(event["kind"]) + + switch { + case kind == "manual" && op == "sync_threads": + emit([]map[string]any{{ + "verb": "http.request", "request_id": "sync-1", "method": "GET", + "url": "https://slack.com/api/conversations.list?types=public_channel,private_channel&limit=200&exclude_archived=true", + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) + return + case kind == "manual" && op == "poll_inbox": + channel := asString(params["vendor_thread_id"]) + if channel == "" { + channel = asString(params["channel"]) + } + parent := asString(params["parent_vendor_message_id"]) + if parent == "" { + parent = asString(params["thread_ts"]) + } + if channel == "" { + emit([]map[string]any{{"verb": "result.emit", "messages": []map[string]any{}}}) return - if parent: - url = "https://slack.com/api/conversations.replies?channel=" + str(channel) + "&ts=" + str(parent) + "&limit=50" - emit([{"verb":"http.request","request_id":"replies-1","method":"GET","url":url,"headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + if parent != "" { + url := "https://slack.com/api/conversations.replies?channel=" + channel + "&ts=" + parent + "&limit=50" + emit([]map[string]any{{ + "verb": "http.request", "request_id": "replies-1", "method": "GET", "url": url, + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) return - url = "https://slack.com/api/conversations.history?channel=" + str(channel) + "&limit=50" - emit([{"verb":"http.request","request_id":"poll-1","method":"GET","url":url,"headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + url := "https://slack.com/api/conversations.history?channel=" + channel + "&limit=50" + emit([]map[string]any{{ + "verb": "http.request", "request_id": "poll-1", "method": "GET", "url": url, + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) return - if event.get("kind") == "message_in_room" and op == "send_message": - channel = params.get("vendor_thread_id") - text = params.get("text", "") - parent = params.get("parent_vendor_message_id") or params.get("thread_ts") - payload = {"channel":channel,"text":text} - if parent: + case kind == "message_in_room" && op == "send_message": + channel := asString(params["vendor_thread_id"]) + text := asString(params["text"]) + parent := asString(params["parent_vendor_message_id"]) + if parent == "" { + parent = asString(params["thread_ts"]) + } + payload := map[string]any{"channel": channel, "text": text} + if parent != "" { payload["thread_ts"] = parent - emit([{"verb":"http.request","request_id":"send-1","method":"POST","url":"https://slack.com/api/chat.postMessage","headers":{"Authorization":"Bearer {{secret:bot_token}}","Content-Type":"application/json"},"json":payload}]) + } + emit([]map[string]any{{ + "verb": "http.request", "request_id": "send-1", "method": "POST", + "url": "https://slack.com/api/chat.postMessage", + "headers": map[string]any{ + "Authorization": "Bearer {{secret:bot_token}}", + "Content-Type": "application/json", + }, + "json": payload, + }}) return - if event.get("kind") == "http_results" and op == "sync_threads": - threads = [] - for item in sorted_results(event.get("http_results")): - if item.get("request_id") != "sync-1": continue - payload = json.loads(item.get("body") or "{}") - if payload.get("ok") is False: - emit([{"verb":"result.emit","title":"Slack list failed","summary":str(payload.get("error") or "unknown")}]) + case kind == "http_results" && op == "sync_threads": + threads := []map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + if asString(item["request_id"]) != "sync-1" { + continue + } + var payload map[string]any + _ = json.Unmarshal([]byte(asString(item["body"])), &payload) + if payload["ok"] == false { + emit([]map[string]any{{ + "verb": "result.emit", + "title": "Slack list failed", + "summary": asString(payload["error"]), + }}) return - for ch in sorted(payload.get("channels") or [], key=lambda c: str(c.get("id",""))): - if ch.get("is_member") is False: + } + channels, _ := payload["channels"].([]any) + sort.Slice(channels, func(i, j int) bool { + left := asMap(channels[i]) + right := asMap(channels[j]) + return asString(left["id"]) < asString(right["id"]) + }) + for _, chAny := range channels { + ch := asMap(chAny) + if ch["is_member"] == false { continue - cid = ch.get("id") - name = ch.get("name") or cid - if cid: - threads.append({"vendor_thread_id":cid,"title":"#" + str(name)}) - emit([{"verb":"result.emit","threads":threads}]) + } + cid := asString(ch["id"]) + name := asString(ch["name"]) + if name == "" { + name = cid + } + if cid != "" { + threads = append(threads, map[string]any{ + "vendor_thread_id": cid, + "title": "#" + name, + }) + } + } + } + emit([]map[string]any{{"verb": "result.emit", "threads": threads}}) return - if event.get("kind") == "http_results" and op == "poll_inbox": - channel = params.get("vendor_thread_id") or params.get("channel") - messages = [] - for item in sorted_results(event.get("http_results")): - if item.get("request_id") not in ("poll-1", "replies-1"): continue - payload = json.loads(item.get("body") or "{}") - if payload.get("ok") is False: - emit([{"verb":"result.emit","title":"Slack blocked this thread","summary":str(payload.get("error") or "unknown")}]) + case kind == "http_results" && op == "poll_inbox": + channel := asString(params["vendor_thread_id"]) + if channel == "" { + channel = asString(params["channel"]) + } + messages := []map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + rid := asString(item["request_id"]) + if rid != "poll-1" && rid != "replies-1" { + continue + } + var payload map[string]any + _ = json.Unmarshal([]byte(asString(item["body"])), &payload) + if payload["ok"] == false { + emit([]map[string]any{{ + "verb": "result.emit", + "title": "Slack blocked this thread", + "summary": asString(payload["error"]), + }}) return - for msg in sorted(payload.get("messages") or [], key=lambda m: str(m.get("ts",""))): - parsed = message_from(msg, channel) - if parsed: messages.append(parsed) - emit([{"verb":"result.emit","messages":messages}]) + } + rawMessages, _ := payload["messages"].([]any) + sort.Slice(rawMessages, func(i, j int) bool { + left := asMap(rawMessages[i]) + right := asMap(rawMessages[j]) + return asString(left["ts"]) < asString(right["ts"]) + }) + for _, msgAny := range rawMessages { + parsed := messageFrom(asMap(msgAny), channel) + if parsed != nil { + messages = append(messages, parsed) + } + } + } + emit([]map[string]any{{"verb": "result.emit", "messages": messages}}) return - if event.get("kind") == "http_results" and op == "send_message": - body = {} - for item in sorted_results(event.get("http_results")): - if item.get("request_id") == "send-1": - body = json.loads(item.get("body") or "{}") - if body.get("ok"): - ts = body.get("ts") or (body.get("message") or {}).get("ts") - emit([{"verb":"result.emit","sent_message":{"vendor_message_id":ts,"created_at":ts}}]) - else: - emit([{"verb":"result.emit","summary":"send failed"}]) + case kind == "http_results" && op == "send_message": + body := map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + if asString(item["request_id"]) == "send-1" { + _ = json.Unmarshal([]byte(asString(item["body"])), &body) + } + } + if body["ok"] == true { + ts := asString(body["ts"]) + if ts == "" { + ts = asString(asMap(body["message"])["ts"]) + } + emit([]map[string]any{{ + "verb": "result.emit", + "sent_message": map[string]any{ + "vendor_message_id": ts, + "created_at": ts, + }, + }}) + } else { + emit([]map[string]any{{"verb": "result.emit", "summary": "send failed"}}) + } return - emit([{"verb":"result.emit","summary":"unsupported"}]) - if __name__ == "__main__": - main() + default: + emit([]map[string]any{{"verb": "result.emit", "summary": "unsupported"}}) + } + } """ } diff --git a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift new file mode 100644 index 00000000..d242193a --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift @@ -0,0 +1,57 @@ +import Foundation +import Structure + +/// Reads and verifies pinned Docker product image digests. +public enum DockerImageInspector: Sendable { + public static func localImageID( + tag: String, + executor: @escaping DockerCLIExecutor + ) async throws -> DockerImageDigest { + let response = try await executor( + ["image", "inspect", "--format", "{{.Id}}", tag], + Data(), + 30 + ) + guard response.exitCode == 0 else { + throw DockerImageDigestError.imageMissing(tag) + } + let raw = String(decoding: response.stdout, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard let digest = DockerImageDigest(hexDigest: raw) else { + throw DockerImageDigestError.imageMissing(tag) + } + return digest + } + + public static func verifyPinned( + tag: String, + expected: DockerImageDigest, + executor: @escaping DockerCLIExecutor + ) async throws { + let actual = try await localImageID(tag: tag, executor: executor) + guard actual == expected else { + throw DockerImageDigestError.digestMismatch(tag: tag, expected: expected, actual: actual) + } + } + + /// Returns false when the image exists but predates required worker binaries. + public static func workerImageHasCurrentBinaries( + tag: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor + ) async -> Bool { + let format = "{{index .Config.Labels \"\(DockerWorkerRuntime.binariesLabelKey)\"}}" + do { + let response = try await executor( + ["image", "inspect", "--format", format, tag], + Data(), + 30 + ) + guard response.exitCode == 0 else { return false } + let label = String(decoding: response.stdout, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + return label == DockerWorkerRuntime.binariesLabelValue + } catch { + return false + } + } +} diff --git a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift index 3522edc8..1c1b3403 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift @@ -2,30 +2,53 @@ import Foundation import DockerRunnerXPC import Structure -/// Ensures trusted product Docker images exist (pull or local build). +/// Ensures trusted product Docker images exist (local build) and match pinned digests. public enum DockerProductImagePrewarmer: Sendable { - public static func ensureWebCrawlerImage( + public static func ensureWorkerImage( executor: @escaping DockerCLIExecutor ) async throws { try await ensureImage( - tag: DockerProductImagePolicy.webCrawlerImage, - dockerfileRelativePath: DockerProductImagePolicy.webCrawlerDockerfileRelativePath, - contextRelativePath: DockerProductImagePolicy.webCrawlerBuildContextRelativePath, + tag: DockerProductImagePolicy.workerImage, + dockerfileRelativePath: DockerProductImagePolicy.workerDockerfileRelativePath, + contextRelativePath: DockerProductImagePolicy.workerBuildContextRelativePath, + pinnedDigest: DockerWorkerRuntime.pinnedDigest, + buildValidator: DockerProductImagePolicy.isAllowedWorkerBuild, executor: executor, buildTimeoutSeconds: 1_200 ) } + /// Legacy alias used by crawler startup paths. + public static func ensureWebCrawlerImage( + executor: @escaping DockerCLIExecutor + ) async throws { + try await ensureWorkerImage(executor: executor) + } + public static func ensureImage( tag: String, dockerfileRelativePath: String, contextRelativePath: String, + pinnedDigest: DockerImageDigest, + buildValidator: @escaping (String, String, String) -> Bool, executor: @escaping DockerCLIExecutor, buildTimeoutSeconds: Int = 1_200 ) async throws { let inspect = try await executor(["image", "inspect", tag], Data(), 30) if inspect.exitCode == 0 { - return + let binariesCurrent = await DockerImageInspector.workerImageHasCurrentBinaries( + tag: tag, + executor: executor + ) + if binariesCurrent { + try await DockerImageInspector.verifyPinned( + tag: tag, + expected: pinnedDigest, + executor: executor + ) + return + } + // Stale worker image (missing required binaries). Rebuild overwrites the tag. } guard let repoRoot = DerrickRepositoryRoot.locate() else { @@ -39,6 +62,9 @@ public enum DockerProductImagePrewarmer: Sendable { guard FileManager.default.fileExists(atPath: context.path) else { throw DockerProductImagePrewarmerError.dockerfileMissing(context.path) } + guard buildValidator(dockerfile.path, tag, context.path) else { + throw DockerProductImagePrewarmerError.buildFailed(tag, "build policy rejected image build") + } let build = try await executor( [ @@ -58,37 +84,23 @@ public enum DockerProductImagePrewarmer: Sendable { detail.isEmpty ? "exit \(build.exitCode)" : detail ) } + + try await DockerImageInspector.verifyPinned( + tag: tag, + expected: pinnedDigest, + executor: executor + ) } } -/// One in-flight crawler image build per process. -/// -/// Chat and daemon start this in the background. A `web.crawl` that arrives -/// while it is still running waits on the same task and does not start a second -/// `docker build`. +/// Legacy alias gate — delegates to `WorkerImageGate` so crawler/script paths share one build. public actor WebCrawlerImageGate { public static let shared = WebCrawlerImageGate() - private var inFlight: Task? - public init() {} public func ensureReady(executor: @escaping DockerCLIExecutor) async throws { - if let inFlight { - try await inFlight.value - return - } - let task = Task { - try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) - } - inFlight = task - do { - try await task.value - inFlight = nil - } catch { - inFlight = nil - throw error - } + try await WorkerImageGate.shared.ensureReady(executor: executor) } } @@ -97,8 +109,6 @@ public enum DockerProductImagePrewarmerError: Error, LocalizedError, Equatable, case dockerfileMissing(String) case buildFailed(String, String) - /// First compiler `error:` line, if the docker build log has one. Not shown as the - /// user-facing `errorDescription` (that stays a short human sentence). public var compilerDiagnostic: String? { switch self { case .buildFailed(_, let detail): @@ -111,11 +121,11 @@ public enum DockerProductImagePrewarmerError: Error, LocalizedError, Equatable, public var errorDescription: String? { switch self { case .repositoryRootNotFound: - return "The web crawler image is not installed and Derrick could not find its source to build it." + return "The worker image is not installed and Derrick could not find its source to build it." case .dockerfileMissing: - return "The web crawler image is not installed and Derrick could not find its build files." + return "The worker image is not installed and Derrick could not find its build files." case .buildFailed: - return "Derrick could not build the web crawler image. Make sure Docker Desktop is running, has enough disk space, and can reach the network." + return "Derrick could not build the worker image. Make sure Docker Desktop is running, has enough disk space, and can reach the network." } } diff --git a/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift index 21abfdce..0d85bbf6 100644 --- a/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift +++ b/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift @@ -5,8 +5,9 @@ import Structure /// Own queue (max 1). Job folders are bind-mounted; the image must already /// exist (`docker image inspect` happens outside the permit). public struct FileExtractorDockerExecutor: Sendable { - public static let image = "derrick-file-extractor:swift-6.4-v1" + public static let image = DockerWorkerRuntime.image public static let containerPrefix = "derrick-file-extractor" + public static let binaryPath = DockerWorkerRuntime.extractorBinary public static let maximumTimeoutSeconds = 180 private let executor: DockerCLIExecutor @@ -27,10 +28,7 @@ public struct FileExtractorDockerExecutor: Sendable { timeoutSeconds: Int ) async throws -> DockerCLIResult { let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) - let imageCheck = try await executor(["image", "inspect", Self.image], Data(), 30) - guard imageCheck.exitCode == 0 else { - throw FileExtractorDockerExecutorError.imageUnavailable(Self.image) - } + try await WorkerImageGate.shared.ensureReady(executor: executor) let executor = self.executor do { return try await queue.withPermit { @@ -48,7 +46,7 @@ public struct FileExtractorDockerExecutor: Sendable { startStep: "start file extractor container", body: { name in try await executor( - ["exec", "-i", name, "/usr/local/bin/derrick-file-extractor"], + ["exec", "-i", name, Self.binaryPath], input, timeout ) diff --git a/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift b/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift index 21b09bae..35edd1f6 100644 --- a/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift @@ -64,8 +64,17 @@ public enum FileExtractorToolModule: MCPToolModule { let payload = try JSONEncoder().encode(workerRequest) let dockerResult = try await run(payload, workspace, parsed.timeoutSeconds) let workerJSON = String(decoding: dockerResult.stdout, as: UTF8.self) - let worker = (try? JSONDecoder().decode(FileExtractorWireResult.self, from: dockerResult.stdout)) - ?? FileExtractorWireResult(ok: false, files: [], diagnostics: [workerJSON]) + let worker: FileExtractorWireResult + if (try? GuestContractValidation.validateFileExtractorResultJSON(dockerResult.stdout)) != nil { + worker = (try? JSONDecoder().decode(FileExtractorWireResult.self, from: dockerResult.stdout)) + ?? FileExtractorWireResult(ok: false, files: [], diagnostics: [workerJSON]) + } else { + worker = FileExtractorWireResult( + ok: false, + files: [], + diagnostics: ["File extractor returned invalid JSON output."] + ) + } let exported = (try? workspace.publishOutputs()) ?? [] if dockerResult.exitCode != 0 && !worker.ok { return try failure( diff --git a/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift b/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift index 629d0bc6..abdae01c 100644 --- a/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift @@ -34,7 +34,7 @@ public enum JobOrchestrationToolModule { ]), "tool_arguments": .object([ "type": .string("object"), - "description": .string("Frozen effector args. For web.crawl use {start_url,goal,max_pages,max_depth,timeout_seconds}. For script_exec use {description,reason,script} where script is standalone Python reading JSON from stdin and writing Derrick envelope JSON to stdout.") + "description": .string("Frozen effector args. For web.crawl use {start_url,goal,max_pages,max_depth,timeout_seconds}. For script_exec use {description,reason,script} where script is standalone Go reading JSON from stdin and writing Derrick envelope JSON to stdout.") ]), "wake_after": .object([ "type": .string("boolean"), diff --git a/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift b/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift index baf82d7a..3bcb7363 100644 --- a/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift @@ -18,7 +18,7 @@ public enum PluginFactoryToolModule: MCPToolModule { ]), "host_manifest_json": .object([ "type": .string("string"), - "description": .string("Host-owned Agent Plugin plugin.json. When set, the builder only supplies Python and tests.") + "description": .string("Host-owned Agent Plugin plugin.json. When set, the builder only supplies Go source and tests.") ]), ]), "required": .array([.string("goal")]) diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift new file mode 100644 index 00000000..2c230202 --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift @@ -0,0 +1,266 @@ +import Foundation +import Plugin +import Structure + +/// Offline Go guest executor for `script_exec` and `plugin.invoke`. +/// +/// Compiles `package main` source and runs the Linux binary inside the pinned +/// worker image. Untrusted source never touches the host toolchain. +public struct GoGuestDockerExecutor: Sendable { + public static let containerPrefix = "derrick-guest-runtime" + private static let compileTimeoutSeconds = 120 + private static let writeTimeoutSeconds = 60 + + public let image: String + private let executor: DockerCLIExecutor + private let queue: DerrickDockerRunQueue + + public init( + image: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor, + queue: DerrickDockerRunQueue = .guest + ) { + self.image = image.trimmingCharacters(in: .whitespacesAndNewlines) + self.executor = executor + self.queue = queue + } + + /// Compile guest source in a one-shot container and return the Linux binary bytes. + public func compileSource(_ source: String) async throws -> Data { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await prepareCompiledGuest(source: source, in: name) + return try await readBinary(from: name) + } + } + + /// Compile once, run once (factory single-hop path). + public func runSource( + source: String, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + try await withCompiledGuest(source: source) { name in + try await runCompiledGuest( + container: name, + input: input, + timeoutSeconds: timeoutSeconds + ) + } + } + + /// Compile once, then run the body with a live container name (multi-hop loops). + public func withCompiledGuest( + source: String, + _ body: @escaping @Sendable (String) async throws -> T + ) async throws -> T { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await prepareCompiledGuest(source: source, in: name) + return try await body(name) + } + } + + /// Run a previously compiled guest binary in a one-shot container. + public func runArtifact( + artifact: Data, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await write(binary: artifact, to: name) + return try await runCompiledGuest( + container: name, + input: input, + timeoutSeconds: timeoutSeconds + ) + } + } + + /// Execute `/tmp/guest` in an existing guest container. + public func runCompiledGuest( + container name: String, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + result( + from: try await executor( + [ + "exec", "-i", name, + DockerWorkerRuntime.guestBinaryPath, + ], + input, + min(max(timeoutSeconds, 1), GuestRuntimeLimits.maxTimeoutSeconds) + ) + ) + } + + private func prepareCompiledGuest(source: String, in container: String) async throws { + try await writeSource(source, to: container) + try await compileGuest(in: container) + } + + private func withGuestContainer( + _ body: @escaping @Sendable (String) async throws -> T + ) async throws -> T { + try await DockerImageInspector.verifyPinned( + tag: image, + expected: DockerWorkerRuntime.pinnedDigest, + executor: executor + ) + let image = self.image + let executor = self.executor + do { + return try await queue.withPermit { + try await OneshotDockerContainer.run( + executor: executor, + prefix: Self.containerPrefix, + createArguments: { name in + [ + "create", + ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ + "--network", "none", + "--name", name, + "--env", "HOME=/tmp", + "--env", "GOCACHE=/tmp/gocache", + "--env", "GOTMPDIR=/tmp", + "--read-only", + "--tmpfs", "/tmp:rw,exec,nosuid,size=256m", + "--pids-limit", "128", + "--cpus", "2.0", + "--memory", "1g", + "--security-opt", "no-new-privileges", + "--cap-drop", "ALL", + image, + "/bin/sleep", + "infinity", + ] + }, + createStep: "create go guest runtime container", + startStep: "start go guest runtime container", + body: body + ) + } + } catch let error as OneshotDockerContainerError { + throw mappedGuestError(error) + } + } + + private func writeSource(_ source: String, to container: String) async throws { + try check( + try await executor( + ["exec", "-i", container, "sh", "-c", DockerWorkerRuntime.guestWriteSourceShell], + Data(source.utf8), + Self.writeTimeoutSeconds + ), + step: "write Go guest source" + ) + } + + private func write(binary: Data, to container: String) async throws { + try check( + try await executor( + [ + "exec", "-i", container, "sh", "-c", + "cat > \(DockerWorkerRuntime.guestBinaryPath) && chmod +x \(DockerWorkerRuntime.guestBinaryPath)", + ], + binary, + Self.writeTimeoutSeconds + ), + step: "write Go guest binary" + ) + } + + private func compileGuest(in container: String) async throws { + try check( + try await executor( + ["exec", container, "sh", "-c", DockerWorkerRuntime.guestCompileShell], + Data(), + Self.compileTimeoutSeconds + ), + step: "compile Go guest" + ) + } + + private func readBinary(from container: String) async throws -> Data { + let response = try await executor( + ["exec", container, "sh", "-c", DockerWorkerRuntime.guestReadBinaryShell], + Data(), + Self.writeTimeoutSeconds + ) + try check(response, step: "read compiled Go guest") + guard !response.stdout.isEmpty else { + throw GoGuestDockerExecutorError.commandFailed( + "read compiled Go guest", + "compiled binary was empty" + ) + } + return response.stdout + } + + private func result(from response: DockerCLIResult) -> PluginFactoryExecutionResult { + PluginFactoryExecutionResult( + exitCode: response.exitCode, + stdout: response.stdout, + stderr: response.stderr + ) + } + + private func check(_ response: DockerCLIResult, step: String) throws { + guard response.exitCode == 0 else { + throw GoGuestDockerExecutorError.commandFailed(step, detail(from: response)) + } + } + + private func mappedGuestError(_ error: OneshotDockerContainerError) -> Error { + switch error { + case .commandFailed(let step, let detail): + return GoGuestDockerExecutorError.commandFailed(step, detail) + case .imageUnavailable: + return error + } + } + + private func detail(from result: DockerCLIResult) -> String { + let stderr = String(decoding: result.stderr, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + return stderr.isEmpty ? "exit \(result.exitCode)" : stderr + } +} + +public enum GoGuestDockerExecutorError: Error, LocalizedError, Equatable, Sendable { + case commandFailed(String, String) + + public var errorDescription: String? { + switch self { + case .commandFailed(let step, let detail): + return "\(step) failed: \(detail)" + } + } +} + +/// Shared gate for worker image readiness (crawl, extract, plugin guest). +public actor WorkerImageGate { + public static let shared = WorkerImageGate() + + private var inFlight: Task? + + public func ensureReady(executor: @escaping DockerCLIExecutor) async throws { + if let inFlight { + try await inFlight.value + return + } + let task = Task { + try await DockerProductImagePrewarmer.ensureWorkerImage(executor: executor) + } + inFlight = task + do { + try await task.value + inFlight = nil + } catch { + inFlight = nil + throw error + } + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift new file mode 100644 index 00000000..cc8c9ab6 --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift @@ -0,0 +1,61 @@ +import Foundation +import Plugin +import Structure + +/// Production adapter for the Go plugin factory. +public struct GoPluginFactoryDockerExecutor: PluginFactoryExecutor, PluginFactoryCompiledGuestExecutor, Sendable { + private let runtime: GoGuestDockerExecutor + + public var image: String { runtime.image } + + public init( + image: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor + ) { + runtime = GoGuestDockerExecutor(image: image, executor: executor) + } + + public func runGuestSource( + source: String, + input: Data + ) async throws -> PluginFactoryExecutionResult { + try await runtime.runSource(source: source, input: input) + } + + public func runGuestSourceHops( + source: String, + testInput: Data + ) async throws -> PluginFactoryHopTestRun { + let script = try PluginFactoryTestScript.parse(testInput) + return try await runtime.withCompiledGuest(source: source) { container in + var hopResults: [PluginFactoryExecutionResult] = [] + var lastResult = PluginFactoryExecutionResult(exitCode: 1) + + for hop in script.hops { + let input = try hop.encodeValidated() + let result = try await runtime.runCompiledGuest( + container: container, + input: input + ) + hopResults.append(result) + lastResult = result + guard result.exitCode == 0 else { + return PluginFactoryHopTestRun(final: result, hopResults: hopResults) + } + } + + return PluginFactoryHopTestRun(final: lastResult, hopResults: hopResults) + } + } + + public func packageGuestSource(source: String) async throws -> Data { + try await runtime.compileSource(source) + } + + public func runPackagedArtifact( + _ artifact: Data, + input: Data + ) async throws -> PluginFactoryExecutionResult { + try await runtime.runArtifact(artifact: artifact, input: input) + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift b/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift new file mode 100644 index 00000000..7a3acb17 --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift @@ -0,0 +1,17 @@ +import Foundation +import Plugin +import Structure + +/// Conservative source checks for standalone Go guest scripts. +public enum GoScriptVerifier: Sendable { + public static func validate( + source: String, + dependencies: [String: String] = [:] + ) -> [String] { + var findings = GuestGoSourceValidator.validate(source: source) + if !dependencies.isEmpty { + findings.append("Guest script dependencies are not supported.") + } + return findings + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift index 070b6625..ed9aae65 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift @@ -2,7 +2,7 @@ import Foundation import Plugin import Structure -/// Shared host hop loop for offline Python guest programs. +/// Shared host hop loop for offline Go guest programs. public enum GuestHopLoop: Sendable { public typealias HTTPResultEventBuilder = @Sendable ( [PluginEnvelope], @@ -62,6 +62,7 @@ public enum GuestHopLoop: Sendable { } lastSummary = envelope.payload["summary"]?.stringValue ?? envelope.payload["content"]?.stringValue + ?? envelope.payload["markdown"]?.stringValue ?? envelope.payload["html"]?.stringValue ?? envelope.payload["text"]?.stringValue ?? envelope.payload["title"]?.stringValue diff --git a/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift b/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift index 54dd21d9..207bc7f7 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift @@ -2,7 +2,7 @@ import Foundation import Plugin import Structure -/// Runs an approved factory release through the offline Python guest runtime. +/// Runs an approved factory release through the offline Go guest runtime. public enum GuestPluginRunner: Sendable { public static func run( release: PluginFactoryRelease, @@ -15,14 +15,20 @@ public enum GuestPluginRunner: Sendable { let invokeID = UUID().uuidString let initialEvent = (try? PluginHopEvent.decodeValidated(input)) ?? PluginHopEvent(kind: .manual) - let executor = PythonGuestDockerExecutor(executor: dockerExecutor) + let executor = GoGuestDockerExecutor(executor: dockerExecutor) + guard !release.compiledArtifact.isEmpty else { + throw GoGuestDockerExecutorError.commandFailed( + "load plugin artifact", + "compiled artifact is empty" + ) + } return try await GuestHopLoop.runForPluginInvoke( initialEvent: initialEvent, invokeID: invokeID, timeoutSeconds: timeoutSeconds, execute: { hopInput in - try await executor.runSource( - source: release.guestSource, + try await executor.runArtifact( + artifact: release.compiledArtifact, input: hopInput, timeoutSeconds: timeoutSeconds ) diff --git a/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift b/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift index 629c803c..b7069634 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift @@ -62,7 +62,7 @@ public actor HostHTTPClient { guard !trimmed.isEmpty, let url = URL(string: wireURL), url.scheme != nil, url.host != nil else { return HostHTTPFetch(status: 0, headers: [:], body: "", error: "invalid_url") } - var currentURL = url + var currentURL = NewsSourceURL.canonicalFetchURL(url) var currentMethod = request.method var currentBody = wire.body let envelopeHeaders = wire.headers diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift deleted file mode 100644 index ad464879..00000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift +++ /dev/null @@ -1,133 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Offline Python guest executor for script_exec and plugin.invoke. -/// -/// Recreate-on-handoff: one fresh `--network none` container per run, deleted -/// when the hop loop finishes (host done). The image is reused if already pulled. -public struct PythonGuestDockerExecutor: Sendable { - public static let containerPrefix = "derrick-guest-runtime" - - public let image: String - private let executor: DockerCLIExecutor - private let queue: DerrickDockerRunQueue - - public init( - image: String = DerrickGuestRuntime.pythonGuestDockerImage, - executor: @escaping DockerCLIExecutor, - queue: DerrickDockerRunQueue = .guest - ) { - self.image = image.trimmingCharacters(in: .whitespacesAndNewlines) - self.executor = executor - self.queue = queue - } - - public func runSource( - source: String, - input: Data, - timeoutSeconds: Int = 300 - ) async throws -> PluginFactoryExecutionResult { - try await withGuestContainer { name in - try await write(source: Data(source.utf8), to: name) - return result( - from: try await executor( - ["exec", "-i", name, "python3", "/tmp/guest.py"], - input, - min(max(timeoutSeconds, 1), GuestRuntimeLimits.maxTimeoutSeconds) - ) - ) - } - } - - private func withGuestContainer( - _ body: @escaping @Sendable (String) async throws -> T - ) async throws -> T { - try await OneshotDockerContainer.ensurePulledImage(image, executor: executor) - let image = self.image - let executor = self.executor - do { - return try await queue.withPermit { - try await OneshotDockerContainer.run( - executor: executor, - prefix: Self.containerPrefix, - createArguments: { name in - [ - "create", - ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ - "--network", "none", - "--name", name, - "--env", "HOME=/tmp", - "--read-only", - "--tmpfs", "/tmp:rw,exec,nosuid,size=128m", - "--pids-limit", "128", - "--cpus", "2.0", - "--memory", "1g", - "--security-opt", "no-new-privileges", - "--cap-drop", "ALL", - image, - "/bin/sleep", - "infinity", - ] - }, - createStep: "create guest runtime container", - startStep: "start guest runtime container", - body: body - ) - } - } catch let error as OneshotDockerContainerError { - throw mappedGuestError(error) - } - } - - private func write(source: Data, to container: String) async throws { - try check( - try await executor( - ["exec", "-i", container, "sh", "-c", "cat > /tmp/guest.py"], - source, - 60 - ), - step: "write Python source" - ) - } - - private func result(from response: DockerCLIResult) -> PluginFactoryExecutionResult { - PluginFactoryExecutionResult( - exitCode: response.exitCode, - stdout: response.stdout, - stderr: response.stderr - ) - } - - private func check(_ response: DockerCLIResult, step: String) throws { - guard response.exitCode == 0 else { - throw PythonGuestDockerExecutorError.commandFailed(step, detail(from: response)) - } - } - - private func mappedGuestError(_ error: OneshotDockerContainerError) -> Error { - switch error { - case .commandFailed(let step, let detail): - return PythonGuestDockerExecutorError.commandFailed(step, detail) - case .imageUnavailable: - return error - } - } - - private func detail(from result: DockerCLIResult) -> String { - let stderr = String(decoding: result.stderr, as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) - return stderr.isEmpty ? "exit \(result.exitCode)" : stderr - } -} - -public enum PythonGuestDockerExecutorError: Error, LocalizedError, Equatable, Sendable { - case commandFailed(String, String) - - public var errorDescription: String? { - switch self { - case .commandFailed(let step, let detail): - return "\(step) failed: \(detail)" - } - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift deleted file mode 100644 index d52be34d..00000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Production adapter for the Python factory. -public struct PythonPluginFactoryDockerExecutor: PluginFactoryExecutor, Sendable { - private let runtime: PythonGuestDockerExecutor - - public var image: String { runtime.image } - - public init( - image: String = DerrickGuestRuntime.pythonGuestDockerImage, - executor: @escaping DockerCLIExecutor - ) { - runtime = PythonGuestDockerExecutor(image: image, executor: executor) - } - - public func runGuestSource( - source: String, - input: Data - ) async throws -> PluginFactoryExecutionResult { - try await runtime.runSource(source: source, input: input) - } - - public func packageGuestSource(source: String) async throws -> Data { - Data(source.utf8) - } - - public func runPackagedArtifact( - _ artifact: Data, - input: Data - ) async throws -> PluginFactoryExecutionResult { - guard let source = String(data: artifact, encoding: .utf8) else { - throw PythonGuestDockerExecutorError.commandFailed( - "decode Python artifact", - "artifact is not valid UTF-8" - ) - } - return try await runtime.runSource(source: source, input: input) - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift deleted file mode 100644 index 8825d2bf..00000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift +++ /dev/null @@ -1,13 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Conservative source checks for standalone Python guest scripts. -public enum PythonScriptVerifier: Sendable { - public static func validate( - source: String, - dependencies: [String: String] = [:] - ) -> [String] { - GuestPythonSourceValidator.validate(source: source, dependencies: dependencies) - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift index 5145a427..c1a380d7 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift @@ -3,7 +3,7 @@ import MCP import Plugin import Structure -/// Runs standalone Python guest source and dispatches host-owned capability hops. +/// Runs standalone Go guest source and dispatches host-owned capability hops. public enum ScriptExecutionRuntime { public static func run( arguments: [String: Value], @@ -16,12 +16,12 @@ public enum ScriptExecutionRuntime { ) async throws -> String { let started = Date() let parsed = try parse(arguments) - let language = GuestScriptLanguage.python + let language = GuestScriptLanguage.go logger("[script_exec] \(language.rawValue) source chars=\(parsed.script.count)") if GuestScriptLanguage.requestedLanguageIsUnsupported(arguments) { return finish(blocked( - findings: ["script_exec only runs Python. Swift guest scripts are not supported."], + findings: ["script_exec only runs Go. Python and other guest languages are not supported."], stage: .staticValidation, started: started, parsed: parsed, @@ -30,7 +30,7 @@ public enum ScriptExecutionRuntime { } let staticStarted = Date() - let staticFindings = PythonScriptVerifier.validate( + let staticFindings = GoScriptVerifier.validate( source: parsed.script, dependencies: parsed.dependencies ) @@ -113,30 +113,34 @@ public enum ScriptExecutionRuntime { logger("[script_exec] skipping LLM reviewer") } + let compileStarted = Date() let timeout = GuestRuntimeLimits.effectiveScriptTimeoutSeconds( requested: parsed.timeoutSeconds ) let invokeID = UUID().uuidString do { - let executor = PythonGuestDockerExecutor(executor: stdinExecutor) - let result = try await GuestHopLoop.run( - initialEvent: initialEvent, - invokeID: invokeID, - timeoutSeconds: timeout, - verifier: language.verifierID, - execute: { input in - try await executor.runSource( - source: parsed.script, - input: input, - timeoutSeconds: timeout - ) - }, - logger: logger, - hopHandler: hopHandler - ) + let executor = GoGuestDockerExecutor(executor: stdinExecutor) + let result = try await executor.withCompiledGuest(source: parsed.script) { container in + try await GuestHopLoop.run( + initialEvent: initialEvent, + invokeID: invokeID, + timeoutSeconds: timeout, + verifier: language.verifierID, + execute: { input in + try await executor.runCompiledGuest( + container: container, + input: input, + timeoutSeconds: timeout + ) + }, + logger: logger, + hopHandler: hopHandler + ) + } + let compileMS = ScriptPhaseTiming.elapsedMS(from: compileStarted) let metrics = ScriptPhaseTiming.scriptMetrics(parsed.script) var phaseTiming = result.phaseTiming ?? ScriptPhaseTiming() - phaseTiming.staticValidateMS = staticValidateMS + phaseTiming.staticValidateMS = staticValidateMS + compileMS phaseTiming.totalMS = ScriptPhaseTiming.elapsedMS(from: started) phaseTiming.scriptCharCount = metrics.chars phaseTiming.scriptLineCount = metrics.lines @@ -156,11 +160,17 @@ public enum ScriptExecutionRuntime { phaseTiming: phaseTiming ) return finish(decorated, logger: logger) - } catch let error as PythonGuestDockerExecutorError { + } catch let error as GoGuestDockerExecutorError { logger("[script_exec] guest runtime failed: \(error.localizedDescription)") + let stage: ScriptFailureStage + if case .commandFailed(let step, _) = error, step.contains("compile") { + stage = .typecheck + } else { + stage = .execution + } return finish(runtimeFailure( findings: [error.localizedDescription], - stage: .execution, + stage: stage, started: started, parsed: parsed, assessment: reviewerAssessment, diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift index 87272b17..322efb92 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift @@ -144,8 +144,8 @@ public enum ScriptExecutionVerifier { private static func readonlyViolations(in script: String) -> [String] { let patterns: [(String, String)] = [ - (#"(?m)\bopen\s*\("#, "Readonly mode cannot mutate filesystem."), - (#"(?m)\b(subprocess|os\.system|os\.popen|socket|urllib|requests|httpx)\b"#, "Readonly mode cannot execute nested commands or access the network.") + (#"os\.(Open|ReadFile|WriteFile|Remove|Create|Mkdir)"#, "Readonly mode cannot mutate filesystem."), + (#"\"net/http\"|\"net\"|exec\.Command|crypto/tls"#, "Readonly mode cannot execute nested commands or access the network.") ] return patterns.compactMap { pattern, message in script.range(of: pattern, options: .regularExpression) != nil ? message : nil @@ -159,7 +159,7 @@ extension ScriptExecutionResult { durationMS: Int, maxSeconds: Int = GuestRuntimeLimits.containerRunMaxTTLSeconds, phaseTiming: ScriptPhaseTiming? = nil, - verifier: String = "python-check-v1" + verifier: String = "go-check-v1" ) -> ScriptExecutionResult { let explanation = GuestRuntimeLimits.containerLeaseExceededExplanation(maxSeconds: maxSeconds) return ScriptExecutionResult( diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift index 12e2e3db..dd3c4741 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift @@ -20,11 +20,11 @@ public enum ScriptExecutionToolModule: MCPToolModule { ]), "script": .object([ "type": .string("string"), - "description": .string("Standalone Python source. Reads one JSON event from standard input and writes a JSON array of Derrick envelopes to standard output. Use http.request envelopes for host HTTP and result.emit/message.post for terminal output.") + "description": .string("Standalone Go `package main` source. Reads one hop-event JSON object from standard input and writes an envelope-list JSON array to standard output. Use http.request envelopes for host HTTP and result.emit/message.post for terminal output.") ]), "language": .object([ "type": .string("string"), - "description": .string("Must be python when set. Swift guest scripts are not supported.") + "description": .string("Must be go when set. Python and Swift guest scripts are not supported.") ]), "user_prompt": .object([ "type": .string("string"), diff --git a/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift b/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift index 6ea9c131..a05bfb75 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift @@ -9,5 +9,5 @@ import Foundation import Structure public var ReviewerSystemPrompt: String { - DerrickBundledText.mustLoad("script_reviewer_instructions.md") + ScriptExecContractPrompts.reviewerGuide() } diff --git a/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift index c53322be..1b8ca9e0 100644 --- a/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift +++ b/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift @@ -7,23 +7,20 @@ import Structure /// The image is trusted product code. User input is passed only as JSON on /// stdin; it is never interpolated into a shell command. public struct WebCrawlerDockerExecutor: Sendable { - public static let image = "derrick-web-crawler:swift-6.4-v1" + public static let image = DockerWorkerRuntime.image public static let containerPrefix = "derrick-web-crawler" + public static let binaryPath = DockerWorkerRuntime.crawlerBinary public static let maximumTimeoutSeconds = 900 public static let dockerNetwork = "bridge" private let executor: DockerCLIExecutor private let queue: DerrickDockerRunQueue - private let imageGate: WebCrawlerImageGate - public init( executor: @escaping DockerCLIExecutor, - queue: DerrickDockerRunQueue = .crawler, - imageGate: WebCrawlerImageGate = .shared + queue: DerrickDockerRunQueue = .crawler ) { self.executor = executor self.queue = queue - self.imageGate = imageGate } public func run( @@ -31,7 +28,7 @@ public struct WebCrawlerDockerExecutor: Sendable { timeoutSeconds: Int ) async throws -> DockerCLIResult { let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) - try await imageGate.ensureReady(executor: executor) + try await WorkerImageGate.shared.ensureReady(executor: executor) let prepared = try await WebCrawlerDockerInputPreparer.enrich(input) let executor = self.executor do { @@ -53,7 +50,7 @@ public struct WebCrawlerDockerExecutor: Sendable { startStep: "start crawler container", body: { name in try await executor( - ["exec", "-i", name, "/usr/local/bin/derrick-web-crawler"], + ["exec", "-i", name, Self.binaryPath], prepared.data, timeout ) diff --git a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift index 2c6bc083..bfa60b2b 100644 --- a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift @@ -63,6 +63,17 @@ public enum WebCrawlerToolModule: MCPToolModule { ).encodedJSON() } + do { + try GuestContractValidation.validateWebCrawlerResultJSON(dockerResult.stdout) + } catch { + return try failure( + status: .failed, + stage: .execution, + code: "web_crawl_invalid_output", + message: "Crawler returned invalid JSON output." + ).encodedJSON() + } + guard let result = try? JSONDecoder().decode( WebCrawlerWireResult.self, from: dockerResult.stdout @@ -138,15 +149,16 @@ public enum WebCrawlerToolModule: MCPToolModule { let timeoutSeconds = intValue(arguments["timeout_seconds"]) ?? 120 guard !startURL.isEmpty else { throw WebCrawlerToolError.invalidStartURL } - guard let url = URL(string: startURL), - let scheme = url.scheme?.lowercased(), + guard let rawURL = URL(string: startURL), + let scheme = rawURL.scheme?.lowercased(), scheme == "http" || scheme == "https", - url.host?.isEmpty == false, - url.user == nil, - url.password == nil + rawURL.host?.isEmpty == false, + rawURL.user == nil, + rawURL.password == nil else { throw WebCrawlerToolError.invalidStartURL } + let url = NewsSourceURL.canonicalFetchURL(rawURL, contextHint: goal) guard !goal.isEmpty else { throw WebCrawlerToolError.emptyGoal } guard goal.count <= 2_000 else { throw WebCrawlerToolError.goalTooLong } if let reason = maliciousGoalReason(goal) { @@ -163,7 +175,7 @@ public enum WebCrawlerToolModule: MCPToolModule { } return WebCrawlerWireRequest( - startURL: startURL, + startURL: url.absoluteString, goal: goal, maxPages: maxPages, maxDepth: maxDepth, @@ -274,6 +286,13 @@ private struct WebCrawlerWireResult: Decodable, Sendable { case stopReason = "stop_reason" case diagnostics } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + ok = try container.decode(Bool.self, forKey: .ok) + stopReason = try container.decode(String.self, forKey: .stopReason) + diagnostics = try container.decodeIfPresent([String].self, forKey: .diagnostics) ?? [] + } } private enum WebCrawlerToolError: Error, LocalizedError, Sendable { diff --git a/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift b/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift index f26bde73..dac39f40 100644 --- a/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift +++ b/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift @@ -271,7 +271,7 @@ struct E2EEnvironment { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("[E2E] factory build scope=\(scope.rawValue)…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: dockerExecutor) + let executor = GoPluginFactoryDockerExecutor(executor: dockerExecutor) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 5) ).build( diff --git a/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift b/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift index 7c72d075..9fd283b6 100644 --- a/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift +++ b/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift @@ -83,7 +83,7 @@ enum SlackConnectorInstallReference { userGoal: goal, hostManifest: input.hostManifest, builder: E2EFactoryBuilder(scope: .fullSync), - executor: PythonPluginFactoryDockerExecutor(executor: dockerExecutor), + executor: GoPluginFactoryDockerExecutor(executor: dockerExecutor), reviewer: E2EHarnessReviewer(), logger: { fputs("\($0)\n", stderr) } ) diff --git a/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift b/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift index 06aa406e..202ed356 100644 --- a/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift +++ b/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift @@ -79,7 +79,7 @@ struct LiveHarnessEnvironment { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("[live] building full-sync connector via LLM factory…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: dockerExecutor) + let executor = GoPluginFactoryDockerExecutor(executor: dockerExecutor) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 3) ).build( diff --git a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift index 1896241f..b3970914 100644 --- a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift +++ b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift @@ -7,21 +7,109 @@ import Plugin import WebCrawler @testable import MCPServer -@Suite struct MCPServerTests { - private static let dummyPythonScript = """ - import json, sys - _ = sys.stdin.read() - json.dump([{"verb":"result.emit","summary":"ok"}], sys.stdout) +@Suite(.serialized) struct MCPServerTests { + private static let dummyGoScript = """ + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{"verb": "result.emit", "summary": "ok"}}) + } """ - private static let dummyStdin: @Sendable ([String], Data, Int) async throws -> DockerCLIResult = { arguments, _, _ in - if arguments.contains("python3"), arguments.contains("/tmp/guest.py") { + private static func isGuestBinaryExec(_ arguments: [String]) -> Bool { + arguments.contains(DockerWorkerRuntime.guestBinaryPath) + && !arguments.contains("cat >") + } + + private static func mockWorkerImageInspect(_ arguments: [String]) -> DockerCLIResult? { + guard arguments.first == "image", arguments.contains("inspect") else { + return nil + } + if arguments.contains("{{.Id}}") { + let digest = DockerWorkerRuntime.pinnedDigest.rawValue + "\n" + return DockerCLIResult(exitCode: 0, stdout: Data(digest.utf8), stderr: Data()) + } + if arguments.contains(where: { $0.contains(DockerWorkerRuntime.binariesLabelKey) }) { + let label = DockerWorkerRuntime.binariesLabelValue + "\n" + return DockerCLIResult(exitCode: 0, stdout: Data(label.utf8), stderr: Data()) + } + return DockerCLIResult(exitCode: 0, stdout: Data("[]".utf8), stderr: Data()) + } + + private actor ImageBuildLatch { + private(set) var succeeded = false + func markSucceeded() { succeeded = true } + } + + private static func missingUntilBuiltExecutor( + recorder: DockerCallRecorder, + latch: ImageBuildLatch, + failFirstBuild: Bool = false, + buildDelay: Duration? = nil + ) -> DockerCLIExecutor { + { args, _, _ in + await recorder.append(args) + if args.first == "build" { + if let buildDelay { + try await Task.sleep(for: buildDelay) + } + if failFirstBuild { + let builds = await recorder.calls.filter { $0.first == "build" }.count + if builds == 1 { + return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data("boom".utf8)) + } + } + await latch.markSucceeded() + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if args.first == "image" { + if await latch.succeeded, let mocked = mockWorkerImageInspect(args) { + return mocked + } + return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) + } + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + } + + private static let dummyCompiledGuest = Data([0x7f, 0x45, 0x4c, 0x46, 0x02]) + + private static func mockGuestDocker(_ arguments: [String]) -> DockerCLIResult? { + if arguments.contains("cat > /tmp/guest") { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestWriteSourceShell) { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestCompileShell) { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestReadBinaryShell) { + return DockerCLIResult(exitCode: 0, stdout: dummyCompiledGuest, stderr: Data()) + } + if isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), stderr: Data() ) } + return mockWorkerImageInspect(arguments) + } + + private static let dummyStdin: @Sendable ([String], Data, Int) async throws -> DockerCLIResult = { arguments, _, _ in + if let mocked = mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } @@ -234,46 +322,57 @@ import WebCrawler #expect(allowed?.contains("docs.slack.dev") == true) } - @Test func dockerProductImagePrewarmerSkipsBuildWhenImagePresent() async throws { + @Test func workerImageLabelDetectsMissingBinaries() async { let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in await recorder.append(args) + if args.first == "image", args.contains("inspect"), args.contains("--format") { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } - try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) - #expect(await recorder.calls == [["image", "inspect", DockerProductImagePolicy.webCrawlerImage]]) + let current = await DockerImageInspector.workerImageHasCurrentBinaries(executor: executor) + #expect(!current) } - @Test func dockerProductImagePrewarmerBuildsWhenImageMissing() async throws { - guard DerrickRepositoryRoot.locate() != nil else { return } + @Test func dockerProductImagePrewarmerSkipsBuildWhenImagePresent() async throws { let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + if let mocked = Self.mockWorkerImageInspect(args) { + return mocked } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) let calls = await recorder.calls - #expect(calls.count == 2) - #expect(calls[0] == ["image", "inspect", DockerProductImagePolicy.webCrawlerImage]) - #expect(calls[1].first == "build") - #expect(calls[1].contains(DockerProductImagePolicy.webCrawlerImage)) - #expect(calls[1].last?.hasSuffix("/\(DockerProductImagePolicy.webCrawlerBuildContextRelativePath)") == true) + #expect(calls.first == ["image", "inspect", DockerProductImagePolicy.workerImage]) + #expect(calls.contains { $0.contains("--format") && $0.contains(where: { $0.contains(DockerWorkerRuntime.binariesLabelKey) }) }) + #expect(calls.contains { $0.contains("{{.Id}}") }) + #expect(!calls.contains { $0.first == "build" }) + } + + @Test func dockerProductImagePrewarmerBuildsWhenImageMissing() async throws { + guard DerrickRepositoryRoot.locate() != nil else { return } + let recorder = DockerCallRecorder() + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor(recorder: recorder, latch: latch) + try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) + let calls = await recorder.calls + #expect(calls[0] == ["image", "inspect", DockerProductImagePolicy.workerImage]) + #expect(calls.contains { $0.first == "build" && $0.contains(DockerProductImagePolicy.workerImage) }) + #expect(calls.contains { $0.contains("{{.Id}}") }) + #expect(calls.filter { $0.first == "build" }.count == 1) } @Test func crawlerImageBuildFailureMessageOmitsBuildkitDump() { let error = DockerProductImagePrewarmerError.buildFailed( - DockerProductImagePolicy.webCrawlerImage, + DockerProductImagePolicy.workerImage, "#0 building with \"default\" instance using docker driver" ) let text = error.localizedDescription #expect(!text.contains("#0 building")) - #expect(text.lowercased().contains("web crawler")) + #expect(text.lowercased().contains("worker image")) #expect(text.lowercased().contains("disk")) #expect(error.compilerDiagnostic == nil) } @@ -285,7 +384,7 @@ import WebCrawler error: Build failed """ let error = DockerProductImagePrewarmerError.buildFailed( - DockerProductImagePolicy.webCrawlerImage, + DockerProductImagePolicy.workerImage, detail ) #expect(error.compilerDiagnostic?.contains("CryptoKit") == true) @@ -295,17 +394,12 @@ import WebCrawler @Test func crawlerImageBuildIsSingleFlight() async throws { guard DerrickRepositoryRoot.locate() != nil else { return } let recorder = DockerCallRecorder() - let executor: DockerCLIExecutor = { args, _, _ in - await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - try await Task.sleep(for: .milliseconds(80)) - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor( + recorder: recorder, + latch: latch, + buildDelay: .milliseconds(80) + ) let gate = WebCrawlerImageGate() try await withThrowingTaskGroup(of: Void.self) { group in group.addTask { try await gate.ensureReady(executor: executor) } @@ -315,26 +409,18 @@ import WebCrawler } let calls = await recorder.calls #expect(calls.filter { $0.first == "build" }.count == 1) - #expect(calls.filter { $0.first == "image" }.count == 1) + #expect(calls.filter { $0 == ["image", "inspect", DockerProductImagePolicy.workerImage] }.count == 1) } @Test func crawlerImageBuildFailureAllowsRetry() async throws { guard DerrickRepositoryRoot.locate() != nil else { return } let recorder = DockerCallRecorder() - let executor: DockerCLIExecutor = { args, _, _ in - await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - let builds = await recorder.calls.filter { $0.first == "build" }.count - if builds == 1 { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data("boom".utf8)) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor( + recorder: recorder, + latch: latch, + failFirstBuild: true + ) let gate = WebCrawlerImageGate() do { try await gate.ensureReady(executor: executor) @@ -383,6 +469,9 @@ import WebCrawler let runner = FileExtractorDockerExecutor( executor: { arguments, _, _ in await recorder.append(arguments) + if let mocked = Self.mockWorkerImageInspect(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) @@ -419,11 +508,17 @@ import WebCrawler timeoutSeconds: 5 ) Issue.record("expected missing extractor image") + } catch is DockerProductImagePrewarmerError { + // Image is missing; prewarmer tries a rebuild and that mock also fails. + } catch is DockerImageDigestError { + // Pin check after a failed inspect. } catch let error as FileExtractorDockerExecutorError { #expect(error == .imageUnavailable(FileExtractorDockerExecutor.image)) } let calls = await recorder.calls - #expect(calls == [["image", "inspect", FileExtractorDockerExecutor.image]]) + #expect(calls.contains { $0.first == "image" && $0.contains("inspect") }) + #expect(!calls.contains { $0.first == "create" }) + #expect(!calls.contains { $0.first == "start" }) } @Test func orphanSweeperRemovesLabeledAndPrefixedContainers() async throws { @@ -825,9 +920,9 @@ import WebCrawler #expect(results?.first?["json"] == nil) } - @Test func pythonGuestRuntimeUsesPinnedImage() { - #expect(DerrickGuestRuntime.pythonGuestDockerImage == "python:3.14.7") - #expect(PythonGuestDockerExecutor.containerPrefix == "derrick-guest-runtime") + @Test func goGuestRuntimeUsesWorkerImage() { + #expect(DerrickGuestRuntime.guestDockerImage == DockerWorkerRuntime.image) + #expect(GoGuestDockerExecutor.containerPrefix == "derrick-guest-runtime") #expect(DerrickDockerRunQueue.guest.maxConcurrentContainers == 1) #expect(DerrickDockerRunQueue.crawler.maxConcurrentContainers == 2) #expect(DerrickDockerRunQueue.extractor.maxConcurrentContainers == 1) @@ -847,7 +942,7 @@ import WebCrawler @Test func oneshotEnsurePulledImageSkipsPullWhenImageExists() async throws { let recorder = DockerCallRecorder() - try await OneshotDockerContainer.ensurePulledImage("python:3.14.7") { arguments, _, _ in + try await OneshotDockerContainer.ensurePulledImage(DockerWorkerRuntime.image) { arguments, _, _ in await recorder.append(arguments) return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } @@ -858,7 +953,7 @@ import WebCrawler @Test func oneshotEnsurePulledImagePullsWhenMissing() async throws { let recorder = DockerCallRecorder() - try await OneshotDockerContainer.ensurePulledImage("python:3.14.7") { arguments, _, _ in + try await OneshotDockerContainer.ensurePulledImage(DockerWorkerRuntime.image) { arguments, _, _ in await recorder.append(arguments) if arguments.first == "image" { return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) @@ -867,7 +962,7 @@ import WebCrawler } let calls = await recorder.calls #expect(calls.contains { $0.starts(with: ["image", "inspect"]) }) - #expect(calls.contains { $0.first == "pull" && $0.contains("python:3.14.7") }) + #expect(calls.contains { $0.first == "pull" && $0.contains(DockerWorkerRuntime.image) }) } @Test func dockerRunQueueSerializesWhenMaxIsOne() async throws { @@ -915,7 +1010,7 @@ import WebCrawler return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, prefix: "derrick-guest-runtime", - createArguments: { name in ["create", "--name", name, "python:3.14.7"] }, + createArguments: { name in ["create", "--name", name, DockerWorkerRuntime.image] }, createStep: "create guest runtime container", startStep: "start guest runtime container", body: { _ in @@ -931,40 +1026,35 @@ import WebCrawler } } - @Test func pythonSourceVerifierRejectsNetworkAndDependencies() { - let findings = PythonScriptVerifier.validate( - source: "import sys\nimport requests", + @Test func goSourceVerifierRejectsNetworkAndDependencies() { + let findings = GoScriptVerifier.validate( + source: "package main\nimport \"net/http\"", dependencies: ["example": "1.0.0"] ) - #expect(findings.contains("Direct network access is not allowed; emit http.request envelopes.")) - #expect(findings.contains("Guest plugin dependencies are not supported; use the standard library.")) + #expect(findings.contains("Go guest must not import \"net/http\".")) + #expect(findings.contains("Guest script dependencies are not supported.")) } - @Test func pythonSourceVerifierRequiresStdin() { - let findings = PythonScriptVerifier.validate(source: "print('[]')") - #expect(findings.contains("Python source must read its JSON event from standard input.")) + @Test func goSourceVerifierRequiresPackageMain() { + let findings = GoScriptVerifier.validate(source: "package plugin") + #expect(findings.contains("Go guest source must declare package main.")) } - @Test func pythonExecutorUsesReadOnlyOfflineContainer() async throws { + @Test func goExecutorUsesReadOnlyOfflineContainer() async throws { let recorder = DockerCallRecorder() - let runner = PythonGuestDockerExecutor( - image: "python:3.14.7", + let runner = GoGuestDockerExecutor( + image: DockerWorkerRuntime.image, executor: { arguments, _, _ in await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) + if let mocked = Self.mockGuestDocker(arguments) { + return mocked } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) ) _ = try await runner.runSource( - source: "import json, sys\njson.dump([], sys.stdout)", + source: Self.dummyGoScript, input: Data(#"{"kind":"script"}"#.utf8) ) @@ -975,15 +1065,17 @@ import WebCrawler #expect(create.contains("--read-only")) #expect(create.contains("--label")) #expect(create.contains(DerrickDockerRuntimeIdentity.labelAssignment)) - let exec = calls.first(where: { $0.contains("python3") }) ?? [] - #expect(exec.contains("/tmp/guest.py")) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestWriteSourceShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestCompileShell) }) + let exec = calls.first(where: { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) ?? [] + #expect(exec.contains(DockerWorkerRuntime.guestBinaryPath)) #expect(calls.contains { $0.first == "rm" && $0.contains("-f") }) } - @Test func pythonGuestDockerCommandsPassXPCValidation() async throws { + @Test func goGuestDockerCommandsPassXPCValidation() async throws { let recorder = DockerCallRecorder() - let runner = PythonGuestDockerExecutor( - image: "python:3.14.7", + let runner = GoGuestDockerExecutor( + image: DockerWorkerRuntime.image, executor: { arguments, _, _ in await recorder.append(arguments) if let error = DockerRunRequestValidator.validate( @@ -995,26 +1087,22 @@ import WebCrawler stderr: Data(error.launchErrorMessage.utf8) ) } - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) + if let mocked = Self.mockGuestDocker(arguments) { + return mocked } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) ) let result = try await runner.runSource( - source: "import json, sys\njson.dump([], sys.stdout)", + source: Self.dummyGoScript, input: Data(#"{"kind":"script"}"#.utf8) ) #expect(result.exitCode == 0) let calls = await recorder.calls - #expect(calls.contains { $0.contains("sh") && $0.contains("cat > /tmp/guest.py") }) - #expect(calls.contains { $0.contains("python3") && $0.contains("/tmp/guest.py") }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestWriteSourceShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestCompileShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) } @Test func leftoverSwiftRuntimePrefixIsStillSwept() { @@ -1022,15 +1110,15 @@ import WebCrawler #expect(GuestRuntimeLimits.maxTimeoutSeconds == 300) } - @Test func pythonScriptCanReturnHTMLResult() async throws { + @Test func goScriptCanReturnHTMLResult() async throws { let resultText = try await ScriptExecutionRuntime.run( arguments: [ "description": .string("render a safe card"), "reason": .string("manual HTML output check"), - "script": .string(Self.dummyPythonScript) + "script": .string(Self.dummyGoScript) ], stdinExecutor: { arguments, _, _ in - if arguments.contains("python3"), arguments.contains("/tmp/guest.py") { + if Self.isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data( @@ -1039,6 +1127,9 @@ import WebCrawler stderr: Data() ) } + if let mocked = Self.mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, reviewer: StubReviewer( @@ -1058,13 +1149,13 @@ import WebCrawler #expect(result.output?.value == "

Safe

") } - @Test func scriptExecRejectsSwiftLanguage() async throws { + @Test func scriptExecRejectsUnsupportedLanguage() async throws { let resultText = try await ScriptExecutionRuntime.run( arguments: [ - "description": .string("legacy swift"), + "description": .string("legacy python"), "reason": .string("should be blocked"), - "script": .string(Self.dummyPythonScript), - "language": .string("swift") + "script": .string(Self.dummyGoScript), + "language": .string("python") ], stdinExecutor: Self.dummyStdin, reviewer: nil, @@ -1074,10 +1165,10 @@ import WebCrawler let result = try #require(ToolExecutionOutcome.decode(from: resultText)) #expect(result.status == .blocked) #expect(result.stage == .validation) - #expect(resultText.contains("only runs Python")) + #expect(resultText.contains("only runs Go")) } - @Test func pythonScriptToolBlocksFilesystemAccess() async throws { + @Test func goScriptToolBlocksFilesystemAccess() async throws { let bridge = try await MCPLocalBridge.make { server in await server.registerScriptExecutionTool( stdinExecutor: Self.dummyStdin, @@ -1098,7 +1189,7 @@ import WebCrawler arguments: [ "description": .string("attempt write"), "reason": .string("test"), - "script": .string("import sys\n_ = sys.stdin.read()\nopen('/tmp/a','w')") + "script": .string("package main\nimport \"os\"\nfunc main() { _, _ = os.Open(\"/tmp/a\") }") ] ) @@ -1108,18 +1199,18 @@ import WebCrawler @Test func leftoverSwiftGuestImageIsTreatedAsStaleHygieneTag() { #expect(DerrickGuestRuntime.swiftPluginDockerImage.contains("swift")) - #expect(DerrickGuestRuntime.pythonGuestDockerImage == "python:3.14.7") + #expect(DerrickGuestRuntime.guestDockerImage == DockerWorkerRuntime.image) } - @Test func guestPluginRunnerRunsPythonRelease() async throws { + @Test func guestPluginRunnerRunsGoRelease() async throws { let recorder = DockerCallRecorder() let release = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", manifestJSON: "{}", - runtimeJSON: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"#, - guestSource: "import json, sys\njson.dump([{\"verb\":\"result.emit\",\"summary\":\"ok\"}], sys.stdout)", - compiledArtifact: Data(), + runtimeJSON: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"#, + guestSource: Self.dummyGoScript, + compiledArtifact: Self.dummyCompiledGuest, skillFiles: [:], contentHash: try PluginContentHash(hex: String(repeating: "c", count: 64)), reviewSummary: "ok" @@ -1129,21 +1220,23 @@ import WebCrawler input: Data(#"{"kind":"manual"}"#.utf8), dockerExecutor: { arguments, _, _ in await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { + if Self.isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), stderr: Data() ) } + if let mocked = Self.mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } ) #expect(result.exitCode == 0) #expect(String(decoding: result.stdout, as: UTF8.self).contains("result.emit")) let calls = await recorder.calls - #expect(calls.contains { $0.contains("python3") }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) #expect(!calls.contains { $0.contains("swift") }) } @@ -1274,12 +1367,12 @@ import WebCrawler #expect(hops == PluginContract.maxPluginInvokeHops) } - @Test func pythonGuestContainerArgumentsStayNetworkIsolated() { + @Test func goGuestContainerArgumentsStayNetworkIsolated() { let name = "derrick-guest-runtime-test" let args = [ "create", "--network", "none", "--name", name, "--read-only", "--tmpfs", "/tmp:rw,exec,nosuid,size=128m", - DerrickGuestRuntime.pythonGuestDockerImage, "/bin/sleep", "infinity", + DerrickGuestRuntime.guestDockerImage, "/bin/sleep", "infinity", ] #expect(args.contains("--name")) #expect(args.contains(name)) @@ -1301,7 +1394,7 @@ import WebCrawler "mode": .string("write"), "description": .string("create report file"), "reason": .string("user asked for file output"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "expected_effects": .array([.string("write /tmp/report.txt")]), "allow_network": .bool(true) ] @@ -1332,7 +1425,7 @@ import WebCrawler "mode": .string("readonly"), "description": .string("inspect csv"), "reason": .string("analyze user-provided data"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "user_prompt": .string("summarize this csv"), "allow_network": .bool(true) ] @@ -1448,7 +1541,7 @@ import WebCrawler "mode": .string("readonly"), "description": .string("fetch page"), "reason": .string("test"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "allow_network": .bool(true) ] ) diff --git a/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift b/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift deleted file mode 100644 index b95215d0..00000000 --- a/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation -import MCPServer -import Plugin -import Testing -import Structure - -@Suite struct PythonPluginFactoryDockerExecutorTests { - @Test func packagesAndRunsPythonSource() async throws { - let recorder = DockerCallRecorder() - let runner = PythonPluginFactoryDockerExecutor( - image: "python:3.14.7", - executor: { arguments, _, _ in - await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - ) - let draft = try await runner.runGuestSource( - source: "import json, sys\njson.dump([], sys.stdout)", - input: Data(#"{"kind":"manual"}"#.utf8) - ) - #expect(draft.exitCode == 0) - let packaged = try await runner.packageGuestSource( - source: "import json, sys\njson.dump([], sys.stdout)" - ) - let released = try await runner.runPackagedArtifact( - packaged, - input: Data(#"{"kind":"manual"}"#.utf8) - ) - #expect(released.exitCode == 0) - let calls = await recorder.calls - #expect(calls.contains { $0.contains("python3") && $0.contains("/tmp/guest.py") }) - } -} diff --git a/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift b/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift new file mode 100644 index 00000000..e4bd5293 --- /dev/null +++ b/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift @@ -0,0 +1,35 @@ +import Foundation +import Structure + +public enum GuestGoSourceValidator: Sendable { + private static let forbiddenImports = [ + "\"net/http\"", + "\"net\"", + "\"os/exec\"", + "\"crypto/tls\"", + ] + + private static let forbiddenCalls = [ + "os.Open", + "os.ReadFile", + "os.WriteFile", + "exec.Command", + "http.Get", + "http.Post", + "http.Client", + ] + + public static func validate(source: String) -> [String] { + var findings: [String] = [] + for token in forbiddenImports where source.contains(token) { + findings.append("Go guest must not import \(token).") + } + for token in forbiddenCalls where source.contains(token) { + findings.append("Go guest must not call \(token).") + } + if !source.contains("package main") { + findings.append("Go guest source must declare package main.") + } + return findings + } +} diff --git a/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift b/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift deleted file mode 100644 index 2bc56739..00000000 --- a/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift +++ /dev/null @@ -1,44 +0,0 @@ -import Foundation -import Structure - -/// Conservative source checks for Python guest plugins and factory drafts. -public enum GuestPythonSourceValidator: Sendable { - public static func validate( - source: String, - dependencies: [String: String] = [:] - ) -> [String] { - var findings: [String] = [] - let trimmed = source.trimmingCharacters(in: .whitespacesAndNewlines) - - if trimmed.isEmpty { - findings.append("Python source is empty.") - } - - let forbiddenTokens: [(String, String)] = [ - ("import socket", "Direct socket access is not allowed; emit http.request envelopes."), - ("from socket", "Direct socket access is not allowed; emit http.request envelopes."), - ("import urllib", "Direct network access is not allowed; emit http.request envelopes."), - ("from urllib", "Direct network access is not allowed; emit http.request envelopes."), - ("import requests", "Direct network access is not allowed; emit http.request envelopes."), - ("import httpx", "Direct network access is not allowed; emit http.request envelopes."), - ("import subprocess", "Process execution is not allowed."), - ("os.system(", "Process execution is not allowed."), - ("os.popen(", "Process execution is not allowed."), - ("open(", "Filesystem access is not allowed in guest plugins."), - ] - for (token, message) in forbiddenTokens where source.contains(token) { - findings.append(message) - } - - let readsStdin = source.contains("sys.stdin") - || source.contains("input(") - || source.contains("stdin.read") - if !readsStdin { - findings.append("Python source must read its JSON event from standard input.") - } - if !dependencies.isEmpty { - findings.append("Guest plugin dependencies are not supported; use the standard library.") - } - return findings - } -} diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift index b5883093..046e54e2 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift @@ -76,12 +76,8 @@ public struct PluginFactorySession: Sendable { "Fix every item below in your next JSON draft response:", ] parts.append(contentsOf: findings.map { "- \($0)" }) - parts.append( - """ - Connector test_input_json must use a hops array replayed by the factory. Match each http.request \ - request_id to an http_results fixture. Declare the same ops in messaging_ops and params.messaging_op. - """ - ) + parts.append(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) + parts.append(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) return parts.joined(separator: "\n") case .reviewRejected(let summary, let findings): var parts = [ @@ -92,23 +88,8 @@ public struct PluginFactorySession: Sendable { parts.append("Findings:") parts.append(contentsOf: findings.map { "- \($0)" }) } - parts.append( - """ - Connector protocol (do not add rules): - \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) - """ - ) - parts.append( - """ - Before returning the next draft, update test_input_json to a hops array replayed by the factory: - {"hops":[{"kind":"message_in_room","params":{"messaging_op":"send_message",...}},\ - {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ - "params":{...}}]} - Include http_results fixtures for every messaging_op you implement. Match request_id values \ - in fixtures to the http.request envelopes your python_source emits. De-duplicate http_results \ - by request_id using stable sorting — do not overwrite duplicates by response order. - """ - ) + parts.append(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) + parts.append(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) return parts.joined(separator: "\n") default: return error.localizedDescription @@ -259,10 +240,14 @@ public struct PluginFactory: Sendable { } let runtimeJSON = try runtimeJSON(for: manifest) + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: draft.manifestJSON + ) var files: [String: Data] = [ "plugin.json": Data(draft.manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.py": Data(draft.guestSource.utf8), + guestPath: Data(draft.guestSource.utf8), "app.derrick/plugin": artifact, ] for (path, body) in draft.skillFiles { @@ -293,9 +278,9 @@ public struct PluginFactory: Sendable { do { let manifest = try AgentPluginManifest.decode(data) guard let entrypoint = manifest.derrick?.entrypoint, - entrypoint.hasSuffix(".py") else { + entrypoint.hasSuffix(".go") else { throw PluginFactoryError.invalidManifest( - "extensions.app.derrick.entrypoint must point to a Python file." + "extensions.app.derrick.entrypoint must point to a Go file." ) } guard !["create-plugin", "edit-plugin"].contains(manifest.name.rawValue) else { @@ -323,7 +308,7 @@ public struct PluginFactory: Sendable { } private func validateSource(_ source: String) throws { - let findings = GuestPythonSourceValidator.validate(source: source) + let findings = GuestGoSourceValidator.validate(source: source) if let first = findings.first { throw PluginFactoryError.invalidSource(first) } @@ -331,10 +316,10 @@ public struct PluginFactory: Sendable { private func runtimeJSON(for manifest: AgentPluginManifest) throws -> String { guard let entrypoint = manifest.derrick?.entrypoint else { - throw PluginFactoryError.invalidManifest("A Python entrypoint is required.") + throw PluginFactoryError.invalidManifest("A Go entrypoint is required.") } let object: [String: String] = [ - "language": "python", + "language": "go", "entrypoint": entrypoint, ] let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift index b5e45a75..2583c1e7 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift @@ -2,6 +2,7 @@ import Foundation import Structure public extension PluginFactoryRelease { - /// All approved releases run as Python guests. - var guestLanguage: PluginGuestLanguage { .python } + var guestLanguage: PluginGuestLanguage { + PluginFactoryRuntime.decode(from: runtimeJSON)?.language ?? .go + } } diff --git a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift index e84fea76..5fa5adec 100644 --- a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift +++ b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift @@ -4,35 +4,79 @@ import Testing @testable import Plugin @Suite struct PluginFactoryTests { - private func guestPythonSource(emit: String = #"[]"#) -> String { + private func guestGoSource(summary: String = "ok") -> String { """ - import json, sys - _ = json.load(sys.stdin) - json.dump(\(emit), sys.stdout) + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{"verb": "result.emit", "summary": "\(summary)"}}) + } """ } - @Test func guestLanguageIsAlwaysPython() { + @Test func releaseVerifiesWithEntrypointGuestPath() throws { + let runtimeJSON = #"{"entrypoint":"./app.derrick/plugin.go","language":"go"}"# + let manifestJSON = """ + {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connector-1","version":"1.0.0",\ + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector"}}} + """ + let guestSource = "package main\n" + let artifact = Data("binary".utf8) + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: manifestJSON + ) + #expect(guestPath == "app.derrick/plugin.go") + let files: [String: Data] = [ + "plugin.json": Data(manifestJSON.utf8), + "app.derrick/runtime.json": Data(runtimeJSON.utf8), + guestPath: Data(guestSource.utf8), + "app.derrick/plugin": artifact, + ] + let release = PluginFactoryRelease( + pluginID: "slack-connector-1", + version: "1.0.0", + manifestJSON: manifestJSON, + runtimeJSON: runtimeJSON, + guestSource: guestSource, + compiledArtifact: artifact, + skillFiles: [:], + contentHash: PluginContentHash.hash(files: files), + reviewSummary: "ok" + ) + #expect(release.verifyIntegrity()) + } + + @Test func guestLanguageIsGoFromRuntimeJSON() { let release = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", manifestJSON: "{}", - runtimeJSON: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"#, - guestSource: guestPythonSource(), + runtimeJSON: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"#, + guestSource: guestGoSource(), compiledArtifact: Data(), skillFiles: [:], contentHash: try! PluginContentHash(hex: String(repeating: "b", count: 64)), reviewSummary: "ok" ) - #expect(release.guestLanguage == .python) + #expect(release.guestLanguage == .go) } - @Test func pluginFactoryRuntimeDecodesPythonEntrypoint() { + @Test func pluginFactoryRuntimeDecodesGoEntrypoint() { let runtime = PluginFactoryRuntime.decode( - from: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + from: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# ) - #expect(runtime?.language == .python) - #expect(runtime?.entrypoint.hasSuffix(".py") == true) + #expect(runtime?.language == .go) + #expect(runtime?.entrypoint.hasSuffix(".go") == true) } @Test func envelopeDecoderRejectsNestedResultAliases() { @@ -47,7 +91,7 @@ import Testing let reviewer = RecordingFactoryReviewer(result: PluginFactoryReview(approved: true, summary: "safe")) let draft = PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data(#"{"kind":"manual"}"#.utf8), skillFiles: ["skills/weather/SKILL.md": "# Weather\n\nReturn weather."] ) @@ -60,12 +104,12 @@ import Testing #expect(release.pluginID == "weather-tool") #expect(release.version == "1.2.3") - #expect(release.runtimeJSON.contains("\"language\":\"python\"")) - #expect(release.runtimeJSON.contains("plugin.py")) + #expect(release.runtimeJSON.contains("\"language\":\"go\"")) + #expect(release.runtimeJSON.contains("plugin.go")) #expect(!release.contentHash.rawValue.isEmpty) #expect(release.verifyIntegrity()) var tampered = release.packageFiles() - tampered["app.derrick/plugin.py"] = Data("changed".utf8) + tampered["app.derrick/plugin.go"] = Data("changed".utf8) #expect(!PluginFactoryRelease.verifyIntegrity(files: tampered, expected: release.contentHash)) #expect(await executor.draftRunCount == 1) #expect(await executor.packageCount == 1) @@ -83,7 +127,7 @@ import Testing _ = try await PluginFactory().build( draft: PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource() + guestSource: guestGoSource() ), executor: executor, reviewer: reviewer @@ -110,9 +154,17 @@ import Testing draft: PluginFactoryDraft( manifestJSON: manifestJSON(), guestSource: """ - import json, sys - _ = json.load(sys.stdin) - print("not a plugin envelope") + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + _ = json.NewDecoder(os.Stdin).Decode(&map[string]any{}) + os.Stdout.WriteString("not a plugin envelope") + } """, testInput: Data(#"{"kind":"manual"}"#.utf8) ), @@ -121,10 +173,7 @@ import Testing ) Issue.record("Expected invalid output") } catch let error as PluginFactoryError { - #expect( - error.localizedDescription.contains("invalid plugin output") - || error.localizedDescription.contains("Python draft test failed") - ) + #expect(error.localizedDescription.contains("invalid plugin output")) #expect(await reviewer.callCount == 0) } } @@ -222,7 +271,7 @@ import Testing @Test func reservedPluginIDsCannotBeCreated() async { let draft = PluginFactoryDraft( manifestJSON: manifestJSON().replacingOccurrences(of: "weather-tool", with: "create-plugin"), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), ) do { _ = try await PluginFactory().build( @@ -242,13 +291,13 @@ import Testing @Test func missingSchemaIsRejectedAtTheFactoryBoundary() async { let manifest = """ - {"name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ do { _ = try await PluginFactory().build( draft: PluginFactoryDraft( manifestJSON: manifest, - guestSource: guestPythonSource() + guestSource: guestGoSource() ), executor: RecordingFactoryExecutor(), reviewer: RecordingFactoryReviewer( @@ -268,12 +317,12 @@ import Testing pluginID: "weather-tool", version: "1.0.0", description: "Weather summaries.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), ) let draft = try response.draft() let manifest = try AgentPluginManifest.decode(Data(draft.manifestJSON.utf8)) #expect(manifest.schema == PluginContract.agentPluginSchema) - #expect(manifest.derrick?.entrypoint == "./app.derrick/plugin.py") + #expect(manifest.derrick?.entrypoint == "./app.derrick/plugin.go") } @Test func builderNormalizesUnderscorePluginIDAndWritesSecretLabels() throws { @@ -281,7 +330,7 @@ import Testing pluginID: "slack_connection", version: "1.0.0", description: "Slack send and receive.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), secrets: [ try PluginSecretField(id: "username", label: "Slack username", kind: .username), try PluginSecretField(id: "password", label: "Slack password", kind: .password), @@ -300,7 +349,7 @@ import Testing pluginID: "slack-connection", version: "1.0.0", description: "Slack send and receive.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), role: .connector, messagingOps: ["send_message"] ) @@ -319,7 +368,7 @@ import Testing pluginID: "slack-connection", version: "1.0.0", description: "Slack full sync.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), role: .connector ) let draft = try response.draft() @@ -329,7 +378,7 @@ import Testing @Test func connectorTestScriptRequiresHopsAndFixtures() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) #expect(throws: PluginFactoryError.self) { @@ -337,7 +386,7 @@ import Testing } let draft = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data( #"{"kind":"message_in_room","params":{"messaging_op":"send_message"}}"#.utf8 ), @@ -374,7 +423,7 @@ import Testing @Test func fullSyncTestScriptRequiresReplyThreadPollHop() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) let channelOnly = Data( @@ -391,7 +440,7 @@ import Testing ) let missingReplies = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: channelOnly, userGoal: fullSyncGoal() ) @@ -401,7 +450,7 @@ import Testing let withReplies = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data( """ {"hops":[ @@ -424,7 +473,7 @@ import Testing @Test func dualFixtureAllowsUnsortedHttpResultsLoop() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) let testInput = Data( @@ -446,17 +495,7 @@ import Testing ) let draft = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: """ - import json, sys - event = json.load(sys.stdin) - for item in event.get("http_results") or []: - if item.get("request_id") == "send-1": - body = json.loads(item.get("body") or "{}") - if body.get("ok"): - json.dump([{"verb":"result.emit","sent_message":{"vendor_message_id":"1.0","created_at":"1.0"}}], sys.stdout) - sys.exit(0) - json.dump([{"verb":"result.emit","summary":"failed"}], sys.stdout) - """, + guestSource: connectorGuestGoSource(), testInput: testInput, userGoal: fullSyncGoal() ) @@ -519,7 +558,7 @@ import Testing @Test func missingRoleDefaultsToStandard() throws { let json = """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ let manifest = try AgentPluginManifest.decode(Data(json.utf8)) #expect(manifest.derrick?.role == .standard) @@ -528,7 +567,7 @@ import Testing @Test func invalidRoleIsRejected() { let json = """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"slack"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"slack"}}} """ do { _ = try AgentPluginManifest.decode(Data(json.utf8)) @@ -555,7 +594,7 @@ import Testing pluginID: "weather-tool", version: "1.0.0", description: "Weather summaries.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), skillFiles: [ PluginFactorySkillFile(path: "SKILL.md", body: "Invalid layout.") ] @@ -575,14 +614,14 @@ import Testing private func draft() -> PluginFactoryDraft { PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data(#"{"kind":"manual"}"#.utf8) ) } private func manifestJSON() -> String { """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.2.3","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.2.3","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ } } @@ -615,24 +654,51 @@ private func validConnectorTestInput() -> Data { private func connectorDraft(testInput: Data) -> PluginFactoryDraft { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ return PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: """ - import json, sys - event = json.load(sys.stdin) - def emit(v): - json.dump(v, sys.stdout, separators=(",", ":")) - if event.get("http_results"): - emit([{"verb":"result.emit","sent_message":{"vendor_message_id":"1.0","created_at":"1710000001.0"}}]) - else: - emit([{"verb":"http.request","request_id":"send-1","method":"POST","url":"https://slack.com/api/chat.postMessage"}]) - """, + guestSource: connectorGuestGoSource(), testInput: testInput ) } +private func connectorGuestGoSource() -> String { + """ + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + if _, ok := event["http_results"]; ok { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{ + "verb": "result.emit", + "sent_message": map[string]any{ + "vendor_message_id": "1.0", + "created_at": "1710000001.0", + }, + }}) + return + } + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{ + "verb": "http.request", + "request_id": "send-1", + "method": "POST", + "url": "https://slack.com/api/chat.postMessage", + }}) + } + """ +} + private actor SequenceFactoryBuilder: PluginFactoryBuilder { let drafts: [PluginFactoryDraft] private(set) var callCount = 0 diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift b/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift index 4ac3d26d..9f35e742 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift @@ -9,7 +9,7 @@ import Foundation public struct ContainerLifecyclePolicy: Sendable, Hashable { /// Maximum crawler containers at once (oneshot; own queue). public let maxNetworkContainers: Int - /// Maximum offline Python guest containers at once (`script_exec` / `plugin.invoke`). + /// Maximum offline Go guest containers at once (`script_exec` / `plugin.invoke`). public let maxOfflineContainers: Int /// Maximum file-extractor containers at once (oneshot; own queue). public let maxFileExtractContainers: Int diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift b/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift index 7dace2ea..5e46fd36 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift @@ -50,7 +50,7 @@ public enum DerrickBundledText: Sendable { public static func formatCodeForModel( _ source: String, heading: String, - language: String = "python" + language: String = "go" ) -> String { """ # \(heading) diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift index 3cd83418..cb0c5969 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift @@ -12,12 +12,8 @@ public enum DerrickGuestRuntime: Sendable { /// Leftover Swift guest image tag reported by older daemons. Hygiene retires a mismatch. public static let swiftPluginDockerImage = "swiftlang/swift:nightly-6.4.x-noble" - /// Pullable Python image for offline guests (script_exec primary). - public static let pythonGuestDockerImage = "python:3.14.7" - - /// Custom image with uv for packaged connector plugins. - /// Build: `docker build -f docker/guest-runtime/Dockerfile -t derrick-guest-runtime:python-v1 .` - public static let pythonGuestDockerImageWithUV = "derrick-guest-runtime:python-v1" + /// Unified Go worker image for offline guests (`script_exec` / `plugin.invoke`). + public static let guestDockerImage = DockerWorkerRuntime.image } public struct ServiceHealthReport: Codable, Sendable, Hashable { diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift b/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift index 0a69752b..5604b30b 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift @@ -6,10 +6,14 @@ public enum EffectorAdmissionPolicy: Sendable { context: ExecutionContextWire?, principal: ServicePrincipal ) -> Bool { - if case .job = principal { return true } - guard let context else { return false } - if context.capabilities.contains(.syncWebCrawl) { return true } - if context.workflow?.kind == .pluginFactoryCreate { return true } + switch principal { + case .job, .agent: + return true + default: + break + } + if let context, context.capabilities.contains(.syncWebCrawl) { return true } + if let context, context.workflow?.kind == .pluginFactoryCreate { return true } return false } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift index 1acf6952..5d3808f0 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift @@ -45,7 +45,7 @@ public struct MCPToolCallRequest: Codable, Sendable, Hashable { /// JSON `HelperModelWire` for script security reviewer model selection. /// When nil, MCPService uses the default helper model. public let helperReviewerModelJSON: String? - /// When true, MCPService allows synchronous `web.crawl` (interactive `/create-plugin` turns). + /// When true, plugin factory creation is active for this call. /// Deprecated: use `executionContextJSON` (ExecutionContextWire). public let pluginFactoryCreationActive: Bool /// JSON `ExecutionContextWire` for cross-boundary policy and effector admission. diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift index 0a72e146..10ef8ffb 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift @@ -92,8 +92,8 @@ public enum PluginFactoryCreateFailureMessage: Sendable { if message.count > 160 { return true } let prefixes = [ "Invalid Agent Plugin manifest", - "Invalid Python guest source", - "Python draft test failed", + "Invalid Go guest source", + "Go draft test failed", "Plugin review rejected", "Draft validation failed:", ] diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift index 001ae388..a152366d 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift @@ -4,7 +4,6 @@ import Foundation public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { public enum PluginType: String, Codable, Sendable, CaseIterable { case connector - case newsReader = "news_reader" case custom } @@ -93,6 +92,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { public let description: String public let pluginID: String? public let auth: ConnectorAuthDiscovery? + public let skillMarkdown: String? public init( pluginType: PluginType, @@ -101,7 +101,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { scope: ConnectorScope = .fullSync, description: String, pluginID: String? = nil, - auth: ConnectorAuthDiscovery? = nil + auth: ConnectorAuthDiscovery? = nil, + skillMarkdown: String? = nil ) { self.pluginType = pluginType self.vendor = vendor @@ -109,6 +110,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { self.scope = scope self.pluginID = pluginID?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty self.auth = auth + self.skillMarkdown = skillMarkdown?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty self.description = Self.resolvedDescription( userDescription: description, vendor: vendor, @@ -117,6 +119,37 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { ) } + public static func makeFromSkillDraft( + _ draft: PluginSkillDraft, + auth: ConnectorAuthDiscovery? = nil + ) throws -> PluginFactoryCreateInput { + let pluginID = try draft.normalizedPluginID() + let description = draft.factoryDescription() + let skillMarkdown = draft.skillMarkdown() + switch draft.plannedKind { + case .messagingConnector: + guard let vendor = draft.inferredConnectorVendor else { + throw PluginSkillDraftError.missingConnectorVendor + } + return PluginFactoryCreateInput( + pluginType: .connector, + vendor: vendor, + scope: .fullSync, + description: description, + pluginID: pluginID, + auth: auth, + skillMarkdown: skillMarkdown + ) + case .customCapability: + return PluginFactoryCreateInput( + pluginType: .custom, + description: description, + pluginID: pluginID, + skillMarkdown: skillMarkdown + ) + } + } + /// Builds connector workflow input. The factory goal uses the fixed scope sentence, not free-text extras. public static func makeConnector( vendor: ConnectorVendor, @@ -137,7 +170,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { scope: scope, description: userDescription, pluginID: resolvedID, - auth: auth ?? (try? ConnectorAuthDiscovery.slackBotTokenFallback()) + auth: auth ?? (try? ConnectorAuthDiscovery.slackBotTokenFallback()), + skillMarkdown: nil ) } @@ -182,6 +216,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { case description case pluginID case auth + case skillMarkdown } public init(from decoder: Decoder) throws { @@ -195,6 +230,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { pluginID = try container.decodeIfPresent(String.self, forKey: .pluginID)? .trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty auth = try container.decodeIfPresent(ConnectorAuthDiscovery.self, forKey: .auth) + skillMarkdown = try container.decodeIfPresent(String.self, forKey: .skillMarkdown)? + .trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty description = Self.resolvedDescription( userDescription: rawDescription, vendor: vendor, @@ -212,6 +249,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { try container.encode(description, forKey: .description) try container.encodeIfPresent(pluginID, forKey: .pluginID) try container.encodeIfPresent(auth, forKey: .auth) + try container.encodeIfPresent(skillMarkdown, forKey: .skillMarkdown) } public func encodedJSON() throws -> String { @@ -247,8 +285,11 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { extra.append("Host permission labels: \(auth.permissions.joined(separator: ", "))") } } + if let skillMarkdown, !skillMarkdown.isEmpty { + extra.append("SKILL.md draft:\n\(skillMarkdown)") + } extra.append( - "The host writes plugin.json. Return python_source and test_input_json only. Do not invent a plugin_id or secrets list." + "The host writes plugin.json. Return go_source and test_input_json only. Do not invent a plugin_id or secrets list." ) return try ConnectorContractPrompts.factoryGoal( vendorLabel: vendorLabel, @@ -267,31 +308,43 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { } } - /// Failure stage hint for returning the wizard to the right step. + public func customBuildGoal() -> String { + var lines = [ + "Create an Agent Plugin capability.", + description, + ] + if let skillMarkdown, !skillMarkdown.isEmpty { + lines.append("SKILL.md draft:\n\(skillMarkdown)") + } + lines.append( + "Return go_source, test_input_json, and skill_files. Include a valid plugin.json via the builder contract when no host manifest is supplied." + ) + return lines.joined(separator: "\n\n") + } + + /// Failure stage hint for returning the plugin studio to the right step. public enum FailureStep: String, Sendable { - case type - case vendor - case name - case auth - case news - case description - case creating + case goal + case skill + case preview + case credentials + case build } public static func failureStep(forStage stage: String?) -> FailureStep { switch stage?.lowercased() { - case "type": - return .type - case "name": - return .name - case "auth", "discover": - return .auth - case "news", "paywall": - return .news - case "crawl", "docs", "vendor", "factory", "build", "review", "description": - return .vendor + case "goal": + return .goal + case "skill", "name", "description", "type", "vendor": + return .skill + case "preview": + return .preview + case "auth", "discover", "credentials": + return .credentials + case "crawl", "docs", "factory", "build", "review": + return .build default: - return .creating + return .build } } } diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift b/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift deleted file mode 100644 index ebfd4f9b..00000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift +++ /dev/null @@ -1,154 +0,0 @@ -import Foundation - -public enum NewsFeedParser { - public static func parse(data: Data, sourceLabel: String, fallbackPageURL: URL) -> [ParsedNewsEntry] { - let text = String(data: data, encoding: .utf8) - ?? String(data: data, encoding: .isoLatin1) - ?? "" - if NewsPaywall.looksLikeFeed(text) { - return parseXMLFeed(text, sourceLabel: sourceLabel) - } - if let entry = htmlFallback(text: text, sourceLabel: sourceLabel, url: fallbackPageURL) { - return [entry] - } - return [] - } - - public struct ParsedNewsEntry: Sendable, Hashable { - public var title: String - public var sourceURL: String - public var summary: String? - public var publishedAt: Date? - - public init(title: String, sourceURL: String, summary: String? = nil, publishedAt: Date? = nil) { - self.title = title - self.sourceURL = sourceURL - self.summary = summary - self.publishedAt = publishedAt - } - } - - private static func parseXMLFeed(_ xml: String, sourceLabel: String) -> [ParsedNewsEntry] { - _ = sourceLabel - var entries: [ParsedNewsEntry] = [] - let itemBlocks = slices(of: xml, start: "") - + slices(of: xml, start: "") - for block in itemBlocks { - let title = firstTag(block, names: ["title"]) ?? "" - let link = firstTag(block, names: ["link"]) - ?? attribute(named: "href", in: firstRawTag(block, name: "link") ?? "") - ?? firstTag(block, names: ["guid", "id"]) - ?? "" - let summary = firstTag(block, names: ["description", "summary", "content"]) - let dateText = firstTag(block, names: ["pubDate", "published", "updated", "dc:date"]) - let cleanedTitle = stripTags(title).trimmingCharacters(in: .whitespacesAndNewlines) - let cleanedLink = stripTags(link).trimmingCharacters(in: .whitespacesAndNewlines) - guard !cleanedTitle.isEmpty, let url = URL(string: cleanedLink), url.scheme != nil else { - continue - } - entries.append( - ParsedNewsEntry( - title: cleanedTitle, - sourceURL: url.absoluteString, - summary: summary.map(stripTags).flatMap { $0.isEmpty ? nil : $0 }, - publishedAt: parseDate(dateText) - ) - ) - } - return entries - } - - private static func htmlFallback(text: String, sourceLabel: String, url: URL) -> ParsedNewsEntry? { - _ = sourceLabel - let title = firstTag(text, names: ["title"]) - .map(stripTags)? - .trimmingCharacters(in: .whitespacesAndNewlines) - guard let title, !title.isEmpty else { return nil } - return ParsedNewsEntry(title: title, sourceURL: url.absoluteString, summary: nil, publishedAt: nil) - } - - private static func slices(of text: String, start: String, end: String) -> [String] { - var result: [String] = [] - let startLower = start.lowercased() - let endLower = end.lowercased() - let lower = text.lowercased() - var idx = lower.startIndex - while let startRange = lower[idx...].range(of: startLower) { - guard let endRange = lower[startRange.upperBound...].range(of: endLower) else { break } - let sliceStart = startRange.lowerBound - let sliceEnd = endRange.upperBound - result.append(String(text[sliceStart.. String? { - for name in names { - let lower = xml.lowercased() - let open = "<\(name.lowercased())" - guard let openStart = lower.range(of: open) else { continue } - guard let tagClose = xml[openStart.upperBound...].firstIndex(of: ">") else { continue } - let innerStart = xml.index(after: tagClose) - let closeToken = "" - guard let close = lower[innerStart...].range(of: closeToken) else { continue } - return String(xml[innerStart.. String? { - let lower = xml.lowercased() - let open = "<\(name.lowercased())" - guard let openStart = lower.range(of: open) else { return nil } - guard let tagClose = xml[openStart.upperBound...].firstIndex(of: ">") else { return nil } - return String(xml[openStart.lowerBound...tagClose]) - } - - private static func attribute(named name: String, in tag: String) -> String? { - let pattern = "\(name)\\s*=\\s*\"([^\"]+)\"" - guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) else { - return nil - } - let range = NSRange(tag.startIndex.. 1, - let inner = Range(match.range(at: 1), in: tag) - else { - return nil - } - return String(tag[inner]) - } - - private static func stripTags(_ raw: String) -> String { - var value = raw.replacingOccurrences(of: "<[^>]+>", with: "", options: .regularExpression) - let entities: [(String, String)] = [ - ("&", "&"), - ("<", "<"), - (">", ">"), - (""", "\""), - ("'", "'"), - ("'", "'"), - (" ", " "), - ] - for (from, to) in entities { - value = value.replacingOccurrences(of: from, with: to) - } - return value - } - - private static func parseDate(_ raw: String?) -> Date? { - guard let raw, !raw.isEmpty else { return nil } - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - let rfc = DateFormatter() - rfc.locale = Locale(identifier: "en_US_POSIX") - rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss Z" - if let date = rfc.date(from: trimmed) { return date } - rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss zzz" - if let date = rfc.date(from: trimmed) { return date } - let iso = ISO8601DateFormatter() - iso.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - if let date = iso.date(from: trimmed) { return date } - iso.formatOptions = [.withInternetDateTime] - return iso.date(from: trimmed) - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift deleted file mode 100644 index f09967e2..00000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift +++ /dev/null @@ -1,187 +0,0 @@ -import Foundation - -public enum NewsReaderMode: String, Codable, Sendable, Hashable, CaseIterable { - case list - case summaries - - public var displayName: String { - switch self { - case .list: return "List articles" - case .summaries: return "Summaries" - } - } -} - -public enum NewsReaderSchedule: String, Codable, Sendable, Hashable, CaseIterable { - case off - case hourly - case daily - - public var displayName: String { - switch self { - case .off: return "Only when opened" - case .hourly: return "Every hour" - case .daily: return "Every day" - } - } -} - -public struct NewsSource: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var label: String - public var url: String - - public init(id: String = UUID().uuidString, label: String, url: String) { - self.id = id - self.label = label - self.url = url.trimmingCharacters(in: .whitespacesAndNewlines) - } -} - -public struct NewsReaderSpec: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var name: String - public var topics: [String] - public var sources: [NewsSource] - public var mode: NewsReaderMode - public var maxCount: Int - public var schedule: NewsReaderSchedule - public var lastError: String? - public var lastFetchedAt: Date? - public var createdAt: Date - public var updatedAt: Date - - public init( - id: String = UUID().uuidString, - name: String, - topics: [String], - sources: [NewsSource], - mode: NewsReaderMode = .list, - maxCount: Int = 20, - schedule: NewsReaderSchedule = .off, - lastError: String? = nil, - lastFetchedAt: Date? = nil, - createdAt: Date = .now, - updatedAt: Date = .now - ) { - self.id = id - self.name = name.trimmingCharacters(in: .whitespacesAndNewlines) - self.topics = topics.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty } - self.sources = sources.filter { !$0.url.isEmpty } - self.mode = mode - self.maxCount = min(50, max(1, maxCount)) - self.schedule = schedule - self.lastError = lastError - self.lastFetchedAt = lastFetchedAt - self.createdAt = createdAt - self.updatedAt = updatedAt - } -} - -public struct NewsItem: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var readerID: String - public var title: String - public var sourceURL: String - public var sourceLabel: String - public var summary: String? - public var publishedAt: Date? - public var fetchedAt: Date - - public init( - id: String = UUID().uuidString, - readerID: String, - title: String, - sourceURL: String, - sourceLabel: String, - summary: String? = nil, - publishedAt: Date? = nil, - fetchedAt: Date = .now - ) { - self.id = id - self.readerID = readerID - self.title = title.trimmingCharacters(in: .whitespacesAndNewlines) - self.sourceURL = sourceURL.trimmingCharacters(in: .whitespacesAndNewlines) - self.sourceLabel = sourceLabel - self.summary = summary?.trimmingCharacters(in: .whitespacesAndNewlines) - self.publishedAt = publishedAt - self.fetchedAt = fetchedAt - } -} - -public enum NewsPresetTopic: String, Sendable, CaseIterable, Identifiable { - case financial - case tech - case international - case politics - case science - case sports - - public var id: String { rawValue } - - public var displayName: String { - switch self { - case .financial: return "Financial" - case .tech: return "Tech" - case .international: return "International" - case .politics: return "Politics" - case .science: return "Science" - case .sports: return "Sports" - } - } -} - -public enum NewsPresetSource: String, Sendable, CaseIterable, Identifiable { - case bbcWorld - case npr - case bbcTech - case hn - case googleNews - - public var id: String { rawValue } - - public var source: NewsSource { - switch self { - case .bbcWorld: - return NewsSource(id: rawValue, label: "BBC World", url: "https://feeds.bbci.co.uk/news/world/rss.xml") - case .npr: - return NewsSource(id: rawValue, label: "NPR", url: "https://feeds.npr.org/1001/rss.xml") - case .bbcTech: - return NewsSource(id: rawValue, label: "BBC Technology", url: "https://feeds.bbci.co.uk/news/technology/rss.xml") - case .hn: - return NewsSource(id: rawValue, label: "Hacker News", url: "https://hnrss.org/frontpage") - case .googleNews: - return NewsSource( - id: rawValue, - label: "Google News", - url: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" - ) - } - } -} - -public enum NewsReaderError: Error, Sendable, Equatable, LocalizedError { - case paywalled(url: String, detail: String) - case invalidURL(String) - case emptySources - case emptyName - case fetchFailed(url: String, detail: String) - case notReady - - public var errorDescription: String? { - switch self { - case .paywalled(let url, let detail): - return "This source is behind a paywall, which is not supported yet. \(detail) (\(url))" - case .invalidURL(let url): - return "That is not a usable web address: \(url)" - case .emptySources: - return "Add at least one source or URL." - case .emptyName: - return "Give this news list a name." - case .fetchFailed(let url, let detail): - return "Could not read \(url). \(detail)" - case .notReady: - return "News lists are not ready yet. Try again in a moment." - } - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift deleted file mode 100644 index 37c362f8..00000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift +++ /dev/null @@ -1,139 +0,0 @@ -import Foundation - -public protocol NewsHTTPClient: Sendable { - func get(url: URL) async throws -> NewsHTTPResponse -} - -public struct NewsHTTPResponse: Sendable { - public var status: Int - public var contentType: String? - public var body: Data - - public init(status: Int, contentType: String?, body: Data) { - self.status = status - self.contentType = contentType - self.body = body - } -} - -public struct URLSessionNewsHTTPClient: NewsHTTPClient { - public init() {} - - public func get(url: URL) async throws -> NewsHTTPResponse { - var request = URLRequest(url: url) - request.httpMethod = "GET" - request.timeoutInterval = 20 - request.setValue( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15", - forHTTPHeaderField: "User-Agent" - ) - request.setValue("application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.8", forHTTPHeaderField: "Accept") - let (data, response) = try await URLSession.shared.data(for: request) - let http = response as? HTTPURLResponse - return NewsHTTPResponse( - status: http?.statusCode ?? 0, - contentType: http?.value(forHTTPHeaderField: "Content-Type"), - body: data - ) - } -} - -public enum NewsReaderRefresh { - public static func validateAndFetch( - spec: NewsReaderSpec, - client: any NewsHTTPClient - ) async throws -> [NewsItem] { - let name = spec.name.trimmingCharacters(in: .whitespacesAndNewlines) - guard !name.isEmpty else { throw NewsReaderError.emptyName } - guard !spec.sources.isEmpty else { throw NewsReaderError.emptySources } - - var collected: [NewsItem] = [] - for source in spec.sources { - let parsed = try await fetchSource(source, readerID: spec.id, client: client) - collected.append(contentsOf: parsed) - } - - let filtered = filter(collected, topics: spec.topics) - let unique = uniqued(filtered) - let sorted = unique.sorted { lhs, rhs in - (lhs.publishedAt ?? lhs.fetchedAt) > (rhs.publishedAt ?? rhs.fetchedAt) - } - return Array(sorted.prefix(spec.maxCount)) - } - - public static func digest(from items: [NewsItem]) -> String { - let lines = items.prefix(8).map { item in - "• \(item.title) (\(item.sourceLabel))" - } - return lines.joined(separator: "\n") - } - - private static func fetchSource( - _ source: NewsSource, - readerID: String, - client: any NewsHTTPClient - ) async throws -> [NewsItem] { - guard let url = URL(string: source.url), url.scheme == "http" || url.scheme == "https" else { - throw NewsReaderError.invalidURL(source.url) - } - let fetchURL = NewsSourceURL.canonicalFetchURL(url) - if let reason = NewsPaywall.preflightRejection(url: fetchURL) { - throw NewsReaderError.paywalled(url: source.url, detail: reason) - } - let response: NewsHTTPResponse - do { - response = try await client.get(url: fetchURL) - } catch { - throw NewsReaderError.fetchFailed(url: source.url, detail: error.localizedDescription) - } - if let reason = NewsPaywall.rejectionReason( - url: fetchURL, - status: response.status, - contentType: response.contentType, - body: response.body - ) { - throw NewsReaderError.paywalled(url: source.url, detail: reason) - } - if response.status != 0, response.status < 200 || response.status >= 400 { - throw NewsReaderError.fetchFailed(url: source.url, detail: "HTTP \(response.status)") - } - let entries = NewsFeedParser.parse(data: response.body, sourceLabel: source.label, fallbackPageURL: fetchURL) - guard !entries.isEmpty else { - throw NewsReaderError.fetchFailed( - url: source.url, - detail: "No articles were found. For Google News, Derrick uses the public RSS feed." - ) - } - return entries.map { entry in - NewsItem( - readerID: readerID, - title: entry.title, - sourceURL: entry.sourceURL, - sourceLabel: source.label, - summary: entry.summary, - publishedAt: entry.publishedAt - ) - } - } - - private static func filter(_ items: [NewsItem], topics: [String]) -> [NewsItem] { - let needles = topics.map { $0.lowercased() }.filter { !$0.isEmpty } - guard !needles.isEmpty else { return items } - let matched = items.filter { item in - let hay = "\(item.title) \(item.summary ?? "")".lowercased() - return needles.contains { hay.contains($0) } - } - return matched.isEmpty ? items : matched - } - - private static func uniqued(_ items: [NewsItem]) -> [NewsItem] { - var seen = Set() - var result: [NewsItem] = [] - for item in items { - if seen.insert(item.sourceURL).inserted { - result.append(item) - } - } - return result - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift index a07e64af..454b7d6b 100644 --- a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift +++ b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift @@ -2,30 +2,86 @@ import Foundation /// Rewrites well-known news homepages to a public RSS/Atom endpoint the host can parse. public enum NewsSourceURL { - public static func canonicalFetchURL(_ url: URL) -> URL { + /// Rewrites Google News HTML/topic URLs to RSS feeds the host can fetch reliably. + /// `contextHint` may carry a crawl goal or user prompt (for example "tech news"). + public static func canonicalFetchURL(_ url: URL, contextHint: String? = nil) -> URL { let host = (url.host ?? "").lowercased() guard isGoogleNewsHost(host) else { return url } let path = url.path.lowercased() if path.contains("/rss") || path.hasSuffix(".xml") { + if !path.contains("/headlines/section/topic/"), + let section = googleNewsSection(from: contextHint) { + return googleNewsSectionRSS(section: section) + } return url } + if path.contains("/topics/") { + if let section = googleNewsSection(from: contextHint) { + return googleNewsSectionRSS(section: section) + } + return googleNewsGeneralRSS() + } var parts = URLComponents(url: url, resolvingAgainstBaseURL: false) ?? URLComponents() parts.scheme = "https" parts.host = "news.google.com" parts.path = "/rss" if parts.queryItems == nil || parts.queryItems?.isEmpty == true { - parts.queryItems = [ - URLQueryItem(name: "hl", value: "en-US"), - URLQueryItem(name: "gl", value: "US"), - URLQueryItem(name: "ceid", value: "US:en"), - ] + parts.queryItems = defaultLocaleQueryItems } parts.fragment = nil - return parts.url ?? URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + return parts.url ?? googleNewsGeneralRSS() } public static func isGoogleNewsHost(_ host: String) -> Bool { let value = host.lowercased() return value == "news.google.com" || value.hasSuffix(".news.google.com") } + + private static let defaultLocaleQueryItems = [ + URLQueryItem(name: "hl", value: "en-US"), + URLQueryItem(name: "gl", value: "US"), + URLQueryItem(name: "ceid", value: "US:en"), + ] + + private static func googleNewsGeneralRSS() -> URL { + URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + } + + private static func googleNewsSectionRSS(section: String) -> URL { + var parts = URLComponents() + parts.scheme = "https" + parts.host = "news.google.com" + parts.path = "/rss/headlines/section/topic/\(section)" + parts.queryItems = defaultLocaleQueryItems + return parts.url ?? googleNewsGeneralRSS() + } + + private static func googleNewsSection(from contextHint: String?) -> String? { + let hint = (contextHint ?? "") + .lowercased() + .replacingOccurrences(of: "-", with: " ") + .replacingOccurrences(of: "_", with: " ") + if hint.contains("tech") { + return "TECHNOLOGY" + } + if hint.contains("business") || hint.contains("finance") || hint.contains("market") { + return "BUSINESS" + } + if hint.contains("science") { + return "SCIENCE" + } + if hint.contains("sport") { + return "SPORTS" + } + if hint.contains("health") { + return "HEALTH" + } + if hint.contains("entertainment") { + return "ENTERTAINMENT" + } + if hint.contains("world") { + return "WORLD" + } + return nil + } } diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift new file mode 100644 index 00000000..f6b7d28d --- /dev/null +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift @@ -0,0 +1,352 @@ +import Foundation + +/// User-authored Agent Plugin intent before the factory materializes plugin.json, SKILL.md, and guest code. +public struct PluginSkillDraft: Sendable, Hashable { + public enum Trigger: String, Sendable, CaseIterable, Codable, Hashable { + case chat + case messaging + case schedule + case mention + + public var label: String { + switch self { + case .chat: return "When I ask in chat" + case .messaging: return "From Messaging" + case .schedule: return "On a schedule" + case .mention: return "When I type /plugin-name" + } + } + } + + public struct Example: Sendable, Hashable, Identifiable { + public var id: String + public var userSays: String + public var pluginDoes: String + + public init(id: String = UUID().uuidString, userSays: String, pluginDoes: String) { + self.id = id + self.userSays = userSays + self.pluginDoes = pluginDoes + } + } + + public enum PlannedKind: String, Sendable, Hashable { + case messagingConnector + case customCapability + } + + public var goal: String + public var purpose: String + public var triggers: Set + public var examples: [Example] + public var pluginName: String + + public init( + goal: String = "", + purpose: String = "", + triggers: Set = [.chat], + examples: [Example] = [], + pluginName: String = "" + ) { + self.goal = goal + self.purpose = purpose + self.triggers = triggers + self.examples = examples + self.pluginName = pluginName + } + + public var plannedKind: PlannedKind { + PluginSkillDraftPlanner.inferKind(from: self) + } + + public var inferredConnectorVendor: PluginFactoryCreateInput.ConnectorVendor? { + PluginSkillDraftPlanner.inferConnectorVendor(from: self) + } + + public var isBuildable: Bool { + switch plannedKind { + case .messagingConnector: + return inferredConnectorVendor?.isSelectableInWizard == true + case .customCapability: + return true + } + } + + public var buildBlockedReason: String? { + if plannedKind == .messagingConnector, + inferredConnectorVendor?.isSelectableInWizard != true { + let label = inferredConnectorVendor?.displayName ?? "That service" + return "\(label) messaging connectors are not available yet. Try Slack or describe a custom capability." + } + return nil + } + + public func isTriggerAvailable(_ trigger: Trigger) -> Bool { + PluginSkillDraftPlanner.availableTriggers(for: plannedKind).contains(trigger) + } + + public func skillMarkdown() -> String { + PluginSkillDraftPlanner.skillMarkdown(for: self) + } + + public func previewScenarios() -> [String] { + examples.map { example in + "When you say “\(example.userSays)”, the plugin will \(example.pluginDoes)." + } + } + + public func packageOutline() -> [String] { + switch plannedKind { + case .messagingConnector, .customCapability: + return [ + "plugin.json — name, permissions, and secrets", + "skills/\(normalizedPluginFolderName())/SKILL.md — purpose and examples", + "app.derrick/plugin.go — guest program (compiled in Docker)", + "app.derrick/plugin — compiled binary", + ] + } + } + + public func factoryDescription() -> String { + PluginSkillDraftPlanner.factoryDescription(for: self) + } + + public func normalizedPluginID() throws -> String { + try PluginID.normalized(pluginName).rawValue + } + + private func normalizedPluginFolderName() -> String { + let trimmed = pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return "plugin" } + return trimmed + .lowercased() + .replacingOccurrences(of: #"[^a-z0-9]+"#, with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + } +} + +public enum PluginSkillDraftPlanner { + public static func availableTriggers( + for kind: PluginSkillDraft.PlannedKind + ) -> Set { + switch kind { + case .messagingConnector: + return [.chat, .messaging, .mention] + case .customCapability: + return [.chat, .mention, .schedule] + } + } + + public static func sanitizeTriggers(in draft: inout PluginSkillDraft) { + let allowed = availableTriggers(for: draft.plannedKind) + draft.triggers = draft.triggers.intersection(allowed) + if draft.triggers.isEmpty { + draft.triggers = defaultTriggers(for: draft) + } + } + + public static func inferKind(from draft: PluginSkillDraft) -> PluginSkillDraft.PlannedKind { + let text = combinedText(draft) + if looksLikeMessaging(text) { return .messagingConnector } + return .customCapability + } + + public static func inferConnectorVendor( + from draft: PluginSkillDraft + ) -> PluginFactoryCreateInput.ConnectorVendor? { + let text = combinedText(draft) + if text.contains("slack") { return .slack } + if text.contains("telegram") { return .telegram } + if text.contains("whatsapp") { return .whatsapp } + if text.contains("discord") { return .discord } + if inferKind(from: draft) == .messagingConnector { return .slack } + return nil + } + + public static func applyGoal(_ goal: String, to draft: inout PluginSkillDraft, existingPluginIDs: [String]) { + let trimmed = goal.trimmingCharacters(in: .whitespacesAndNewlines) + draft.goal = trimmed + if draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + draft.purpose = trimmed + } + if draft.pluginName.isEmpty { + draft.pluginName = suggestPluginName(for: draft, existingIDs: existingPluginIDs) + } + if draft.examples.isEmpty { + draft.examples = defaultExamples(for: draft) + } + if draft.triggers.isEmpty { + draft.triggers = defaultTriggers(for: draft) + } + sanitizeTriggers(in: &draft) + } + + public static func skillMarkdown(for draft: PluginSkillDraft) -> String { + let name = draft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + let triggerLines = draft.triggers.sorted { $0.rawValue < $1.rawValue }.map(\.label) + let exampleBlock = draft.examples.map { example in + """ + ### User + \(example.userSays) + + ### Plugin + \(example.pluginDoes) + """ + }.joined(separator: "\n\n") + + return """ + # \(name.isEmpty ? "Plugin" : name) + + ## Purpose + \(draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines)) + + ## When to use + \(triggerLines.map { "- \($0)" }.joined(separator: "\n")) + + ## Examples + \(exampleBlock.isEmpty ? "_Add at least one example before building._" : exampleBlock) + """ + } + + public static func factoryDescription(for draft: PluginSkillDraft) -> String { + let purpose = draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines) + let examples = draft.examples + .map { "User: \($0.userSays) → Plugin: \($0.pluginDoes)" } + .joined(separator: "\n") + switch draft.plannedKind { + case .messagingConnector: + let vendor = inferConnectorVendor(from: draft)?.displayName ?? "messaging" + return """ + \(purpose) + + Messaging connector for \(vendor). List conversations as tabs, load messages including reply threads, and send messages. + + Confirmed behavior: + \(examples) + """ + case .customCapability: + return """ + \(purpose) + + Confirmed behavior: + \(examples) + """ + } + } + + public static func factoryGoal( + for draft: PluginSkillDraft, + crawlSummary: String?, + hostNotes: [String] + ) throws -> String { + let description = factoryDescription(for: draft) + switch draft.plannedKind { + case .messagingConnector: + guard let vendor = inferConnectorVendor(from: draft) else { + throw PluginSkillDraftError.missingConnectorVendor + } + var extra = hostNotes + extra.append("SKILL.md draft:\n\(skillMarkdown(for: draft))") + return try ConnectorContractPrompts.factoryGoal( + vendorLabel: vendor.displayName, + scope: .fullSync, + vendor: vendor, + crawlSummary: crawlSummary, + reference: extra.joined(separator: "\n"), + includeVendorBindings: true + ) + case .customCapability: + return """ + Create an Agent Plugin for this user goal. + + \(description) + + Host notes: + \(hostNotes.joined(separator: "\n")) + + SKILL.md draft (write this into skills/): + \(skillMarkdown(for: draft)) + + Return go_source, test_input_json, and skill_files. The host writes plugin.json when a host manifest is supplied; otherwise include a valid manifest in your output path via the builder contract. + """ + } + } + + private static func combinedText(_ draft: PluginSkillDraft) -> String { + [draft.goal, draft.purpose, draft.pluginName] + .joined(separator: " ") + .lowercased() + } + + private static func looksLikeMessaging(_ text: String) -> Bool { + ["slack", "telegram", "whatsapp", "discord", "messaging", "channel", "inbox", "dm", "chat app"] + .contains { text.contains($0) } + } + + private static func suggestPluginName(for draft: PluginSkillDraft, existingIDs: [String]) -> String { + switch inferKind(from: draft) { + case .messagingConnector: + if let vendor = inferConnectorVendor(from: draft) { + return ConnectorPluginNaming.defaultPluginID(vendor: vendor, existingIDs: existingIDs) + } + return ConnectorPluginNaming.defaultPluginID(vendor: .slack, existingIDs: existingIDs) + case .customCapability: + let words = draft.goal + .lowercased() + .split { !$0.isLetter && !$0.isNumber } + .filter { $0.count > 2 } + .prefix(3) + let stem = words.isEmpty ? "custom-plugin" : String(words.joined(separator: "-")) + if !existingIDs.contains(stem) { return stem } + return "\(stem)-2" + } + } + + private static func defaultTriggers(for draft: PluginSkillDraft) -> Set { + switch inferKind(from: draft) { + case .messagingConnector: + return [.messaging, .chat] + case .customCapability: + return [.chat] + } + } + + private static func defaultExamples(for draft: PluginSkillDraft) -> [PluginSkillDraft.Example] { + switch inferKind(from: draft) { + case .messagingConnector: + let vendor = inferConnectorVendor(from: draft)?.displayName ?? "Slack" + return [ + PluginSkillDraft.Example( + userSays: "Show my \(vendor) channels", + pluginDoes: "list conversations you can access as tabs in Messaging" + ), + PluginSkillDraft.Example( + userSays: "Send “hello” to #general", + pluginDoes: "post the message in that channel" + ), + ] + case .customCapability: + let snippet = draft.goal.trimmingCharacters(in: .whitespacesAndNewlines) + return [ + PluginSkillDraft.Example( + userSays: snippet.isEmpty ? "Do the thing I described" : snippet, + pluginDoes: "run the guest program and return a clear result" + ), + ] + } + } +} + +public enum PluginSkillDraftError: Error, LocalizedError { + case missingConnectorVendor + case invalidPluginName + + public var errorDescription: String? { + switch self { + case .missingConnectorVendor: + return "Could not determine which messaging service this plugin targets." + case .invalidPluginName: + return "Choose a valid plugin name using letters, numbers, and hyphens." + } + } +} diff --git a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift index 4a24d9bc..cf769fa3 100644 --- a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift +++ b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift @@ -31,10 +31,6 @@ public enum PromptResources { try load(named: "files_extract_skill", from: resourceRoot) } - public static func scriptReviewerInstructions(from resourceRoot: URL? = nil) throws -> String { - try DerrickBundledText.load("script_reviewer_instructions.md", from: resourceRoot) - } - public static func workerOverlay(from resourceRoot: URL? = nil) throws -> String { try DerrickBundledText.load("worker_overlay.md", from: resourceRoot) } @@ -43,22 +39,22 @@ public enum PromptResources { try DerrickBundledText.load("user_facing_spawn_overlay.md", from: resourceRoot) } - /// Python guest contract wrapped for a model prompt. + /// Go guest contract wrapped for a model prompt. public static func guestSDKForModel( from resourceRoot: URL? = nil, spec: PluginSpec? = nil ) throws -> String { _ = resourceRoot return DerrickBundledText.formatCodeForModel( - try DerrickGuestPython.source(for: spec), - heading: "standalone Python guest contract", - language: "python" + try DerrickGuestGo.source(for: spec), + heading: "standalone Go guest contract", + language: "go" ) } public static func guestSDKSource(from resourceRoot: URL? = nil) throws -> String { _ = resourceRoot - return try DerrickGuestPython.source() + return try DerrickGuestGo.source() } private static func load(named name: String, from resourceRoot: URL?, prefixTxt: String? = nil) throws -> String { diff --git a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift index f5f57d3b..1350aed8 100644 --- a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift +++ b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift @@ -13,7 +13,7 @@ public struct ExecutionContextSlots: Sendable { public var networkAccessPrompt: TurnProcessContextTypes.NetworkPrompt? public var policyDecisionPrompt: TurnProcessContextTypes.PolicyDecisionPrompt? public var policyNoticePublisher: TurnProcessContextTypes.PolicyNoticePublisher? - /// When true, `web.crawl` may run synchronously (plugin factory turns). + /// When true, plugin factory creation is active for this turn. public var pluginFactoryCreationActive: Bool public init( diff --git a/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift b/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift new file mode 100644 index 00000000..e64934fc --- /dev/null +++ b/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift @@ -0,0 +1,13 @@ +// Automatically generated by scripts/generate-script-exec-contract.swift. DO NOT EDIT. + +/// SHA-256 of script_exec protocol JSON and schemas. `swift test` fails when this is stale. +public enum ScriptExecContractFingerprint: Sendable { + public static let sha256 = "3a543a59f64b9c72a1fe94b8845af719740c17dfbc5ea81f5849c9554dcc8c89" + public static let sourceFiles: [String] = [ + "schemas/script-exec-contract.schema.json", + "schemas/guest-runtime.schema.json", + "schemas/hop-event.schema.json", + "schemas/envelope-list.schema.json", + "contracts/script-exec-contract.json", + ] +} diff --git a/packages/Structure/Sources/Contract/GuestContract.swift b/packages/Structure/Sources/Contract/GuestContract.swift index 0464da30..fdf034ad 100644 --- a/packages/Structure/Sources/Contract/GuestContract.swift +++ b/packages/Structure/Sources/Contract/GuestContract.swift @@ -11,6 +11,11 @@ public enum GuestContract: Sendable { case connectorParams = "connector-params.schema.json" case connectorResultEmit = "connector-result-emit.schema.json" case connectorVendor = "connector-vendor.schema.json" + case guestRuntime = "guest-runtime.schema.json" + case workerProduct = "worker-product.schema.json" + case webCrawlerResult = "web-crawler-result.schema.json" + case fileExtractorResult = "file-extractor-result.schema.json" + case scriptExecContract = "script-exec-contract.schema.json" } public static func loadSchemaText(_ schema: Schema) throws -> String { diff --git a/packages/Structure/Sources/Contract/GuestContractValidation.swift b/packages/Structure/Sources/Contract/GuestContractValidation.swift index 83fcb430..1da6e55f 100644 --- a/packages/Structure/Sources/Contract/GuestContractValidation.swift +++ b/packages/Structure/Sources/Contract/GuestContractValidation.swift @@ -9,4 +9,12 @@ public enum GuestContractValidation: Sendable { public static func validateHopEventJSON(_ data: Data) throws { try GuestContract.validate(json: data, against: .hopEvent) } + + public static func validateWebCrawlerResultJSON(_ data: Data) throws { + try GuestContract.validate(json: data, against: .webCrawlerResult) + } + + public static func validateFileExtractorResultJSON(_ data: Data) throws { + try GuestContract.validate(json: data, against: .fileExtractorResult) + } } diff --git a/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json b/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json new file mode 100644 index 00000000..99f30c60 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json @@ -0,0 +1,173 @@ +{ + "version": 1, + "runtime": { + "language": "go", + "package": "main", + "binary": "/tmp/guest", + "stdlib_only": true, + "forbidden_imports": ["net/http", "net", "os/exec"], + "allowed_os_usage": ["os.Stdin", "os.Stdout"], + "example_go": "package main\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n)\n\nfunc main() {\n\tvar event map[string]any\n\tif err := json.NewDecoder(os.Stdin).Decode(&event); err != nil {\n\t\treturn\n\t}\n\temit([]map[string]any{{\"verb\": \"result.emit\", \"title\": \"Result\", \"summary\": \"done\"}})\n}\n\nfunc emit(envelopes []map[string]any) {\n\tenc := json.NewEncoder(os.Stdout)\n\tenc.SetEscapeHTML(false)\n\t_ = enc.Encode(envelopes)\n}" + }, + "io": { + "stdin_schema": "hop-event.schema.json", + "stdout_schema": "envelope-list.schema.json", + "guest_runtime_schema": "guest-runtime.schema.json" + }, + "workflow": { + "first_hop_verbs": ["http.request"], + "http_results_event_kind": "http_results", + "terminal_verbs": ["result.emit", "message.post"], + "match_http_by": "request_id", + "http_results_accumulate": true, + "post_body_field": "json" + }, + "output": { + "fields": { + "content": { + "purpose": "parsed plain-text summaries from fetched HTML, XML, or RSS", + "host_strips_incidental_markup": true + }, + "summary": { + "purpose": "short plain-text headline or one-line result", + "host_strips_incidental_markup": true + }, + "html": { + "purpose": "only when the user explicitly requested HTML", + "host_allowlist_sanitizes": true + }, + "markdown": { + "purpose": "intentional Markdown formatting only", + "prefer_content_for_extracted_text": true + } + }, + "forbidden_patterns": ["repr(http_results)"], + "raw_body_in_content_requires_explicit_user_request": true + }, + "agent": { + "prefer_direct_urls_over_serp_html": true, + "retry_with_different_urls_on_empty_fetch": true, + "max_correction_attempts_after_block": 1 + }, + "review": { + "fail_fast": true, + "response_schema": { + "alignedWithRequest": "boolean", + "confidence": "number 0.0-1.0", + "suggestedAction": "allow|deny", + "concerns": "string[]", + "summary": "string" + }, + "checks": [ + { + "id": "intent_alignment", + "order": 1, + "description": "Script, description, reason, and user prompt are consistent.", + "notes": [ + "Derrick has a job scheduler (jobs_create, run_after_seconds, cron). Timing is applied by JobService before this script runs.", + "The script must do the work immediately when invoked.", + "Words like delayed, scheduled, in 7 seconds, later, or run_after refer to the scheduler, not sleep inside the script.", + "Do not deny a script that performs the requested work just because it has no delay." + ] + }, + { + "id": "no_secret_literals", + "order": 2, + "description": "No tokens, API keys, passwords, or other secret literals in the source." + }, + { + "id": "terminal_result_not_fetch_only", + "order": 3, + "description": "The script implements the requested terminal result, not just the fetch.", + "notes": [ + "For summarize, list, inspect, or extract requests, the http_results branch must parse the response body and emit the requested data.", + "Do not allow repr(http_results), a fetch-only confirmation, or an entire raw body copied to content unless the user explicitly requested the raw source.", + "If raw HTML is requested, html is allowed because the host sanitizes it before rendering." + ] + }, + { + "id": "untrusted_remote_content", + "order": 4, + "description": "Fetched HTML or XML is untrusted input; output must use envelope-list output fields correctly.", + "notes": [ + "Prefer content or summary for extracted plain-text lists and headlines.", + "Use html only when the user explicitly asked for HTML.", + "Use markdown only for intentional Markdown formatting.", + "Approve scripts that parse RSS, XML, or HTML and emit normalized plain text in content or summary.", + "Do not deny solely because the source was XML or HTML or because the guest does not re-validate http or https on plain-text lines." + ] + } + ], + "do_not_deny_for": [ + "go_style", + "envelope_construction", + "destination_urls", + "absence_of_dependencies" + ], + "enforced_elsewhere": [ + "static_go_verifier_enforces_forbidden_imports", + "guest_has_no_network_host_performs_http", + "host_applies_ssrf_on_http" + ] + }, + "rules": { + "guest_has_no_network": true, + "host_performs_http": true, + "host_applies_ssrf": true, + "static_verifier_enforces_imports": true, + "scheduler_timing_not_in_script": true, + "deterministic_output_required": true, + "stable_sort_and_dedupe_collections": true, + "no_time_random_uuid_for_visible_output": true, + "match_http_results_by_request_id": true + }, + "plugin_factory": { + "manifest": { + "host_creates_manifest": true, + "do_not_return_manifest_json": true, + "agent_plugin_schema": "Agent Plugin 1.0", + "entrypoint": "./app.derrick/plugin.go", + "plugin_id_allowed_chars": "lowercase letters, numbers, hyphens, dots", + "plugin_id_forbidden_chars": ["underscore"], + "roles": ["connector", "standard"], + "connector_messaging_ops": ["send_message", "poll_inbox", "sync_threads"], + "secret_kinds": ["username", "password", "token", "api_key"], + "never_embed_credentials_in_go_source": true, + "host_lists_connectors_under_messaging": true + }, + "builder": { + "response_keys": [ + "plugin_id", + "version", + "description", + "go_source", + "test_input_json", + "skill_files", + "secrets", + "role", + "messaging_ops" + ], + "skill_files_path_pattern": "skills//SKILL.md", + "empty_skill_files_when_unused": true, + "test_input_is_serialized_json_object": true, + "test_input_must_not_be_empty": true, + "test_input_exercises_terminal_result": true, + "connector_test_input_uses_hops_array": true, + "connector_parse_json_http_results_when_vendor_returns_json": true + }, + "review": { + "compilation_success_not_approval": true, + "response_schema": { + "decision": "approved|rejected", + "summary": "string", + "findings": "[{severity: info|warning|blocking, category: alignment|safety|correctness|privacy|supplyChain, message: string}]" + }, + "reject_non_go_source": true, + "reject_raw_network_outside_http_request_envelopes": true, + "reject_missing_stdin_read": true, + "source_derived_titles_may_be_fragments": true, + "reject_unsupported_direct_test_claims": true, + "connector_rules_document": "connector-contract.json" + } + } +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json new file mode 100644 index 00000000..c4e12bc4 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/file-extractor-result.json", + "title": "File extractor worker stdout", + "description": "JSON object written to stdout by derrick-file-extractor.", + "$ref": "worker-product.schema.json#/$defs/file_extractor_result" +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json new file mode 100644 index 00000000..03f4f622 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/guest-runtime.json", + "title": "Derrick offline guest runtime", + "description": "Canonical I/O contract for script_exec and plugin.invoke guests. Trusted worker stdout (crawler, extractor) is defined in worker-product.schema.json. Swift host and Go workers must match these schemas.", + "type": "object", + "required": ["language", "stdin", "stdout"], + "additionalProperties": false, + "properties": { + "language": { + "type": "string", + "const": "go", + "description": "Guest implementation language." + }, + "stdin": { + "description": "One hop event JSON object read from standard input.", + "$ref": "hop-event.schema.json" + }, + "stdout": { + "description": "Envelope list JSON array written to standard output.", + "$ref": "envelope-list.schema.json" + }, + "binary": { + "type": "string", + "const": "/tmp/guest", + "description": "Linux guest binary path inside the worker container." + } + } +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json new file mode 100644 index 00000000..2e369012 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/script-exec-contract.json", + "title": "Derrick script_exec protocol document", + "description": "Canonical guest runtime, workflow, output, agent, and reviewer rules for script_exec and offline Go guests.", + "type": "object", + "required": [ + "version", + "runtime", + "io", + "workflow", + "output", + "agent", + "review", + "rules", + "plugin_factory" + ], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "runtime": { "$ref": "#/$defs/runtime" }, + "io": { "$ref": "#/$defs/io" }, + "workflow": { "$ref": "#/$defs/workflow" }, + "output": { "$ref": "#/$defs/output" }, + "agent": { "$ref": "#/$defs/agent" }, + "review": { "$ref": "#/$defs/review" }, + "rules": { "$ref": "#/$defs/rules" }, + "plugin_factory": { "$ref": "#/$defs/plugin_factory" } + }, + "$defs": { + "runtime": { + "type": "object", + "required": [ + "language", + "package", + "binary", + "stdlib_only", + "forbidden_imports", + "allowed_os_usage", + "example_go" + ], + "additionalProperties": false, + "properties": { + "language": { "type": "string", "const": "go" }, + "package": { "type": "string", "const": "main" }, + "binary": { "type": "string", "const": "/tmp/guest" }, + "stdlib_only": { "type": "boolean" }, + "forbidden_imports": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "allowed_os_usage": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "example_go": { "type": "string", "minLength": 1 } + } + }, + "io": { + "type": "object", + "required": ["stdin_schema", "stdout_schema", "guest_runtime_schema"], + "additionalProperties": false, + "properties": { + "stdin_schema": { "type": "string", "const": "hop-event.schema.json" }, + "stdout_schema": { "type": "string", "const": "envelope-list.schema.json" }, + "guest_runtime_schema": { "type": "string", "const": "guest-runtime.schema.json" } + } + }, + "workflow": { + "type": "object", + "required": [ + "first_hop_verbs", + "http_results_event_kind", + "terminal_verbs", + "match_http_by", + "http_results_accumulate", + "post_body_field" + ], + "additionalProperties": false, + "properties": { + "first_hop_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "http_results_event_kind": { "type": "string" }, + "terminal_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "match_http_by": { "type": "string", "const": "request_id" }, + "http_results_accumulate": { "type": "boolean" }, + "post_body_field": { "type": "string", "const": "json" } + } + }, + "output": { + "type": "object", + "required": ["fields", "forbidden_patterns", "raw_body_in_content_requires_explicit_user_request"], + "additionalProperties": false, + "properties": { + "fields": { + "type": "object", + "required": ["content", "summary", "html", "markdown"], + "additionalProperties": false, + "properties": { + "content": { "$ref": "#/$defs/output_field" }, + "summary": { "$ref": "#/$defs/output_field" }, + "html": { "$ref": "#/$defs/output_field" }, + "markdown": { "$ref": "#/$defs/output_field" } + } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" } + }, + "raw_body_in_content_requires_explicit_user_request": { "type": "boolean" } + } + }, + "output_field": { + "type": "object", + "required": ["purpose"], + "additionalProperties": true, + "properties": { + "purpose": { "type": "string" }, + "host_strips_incidental_markup": { "type": "boolean" }, + "host_allowlist_sanitizes": { "type": "boolean" }, + "prefer_content_for_extracted_text": { "type": "boolean" } + } + }, + "agent": { + "type": "object", + "required": [ + "prefer_direct_urls_over_serp_html", + "retry_with_different_urls_on_empty_fetch", + "max_correction_attempts_after_block" + ], + "additionalProperties": false, + "properties": { + "prefer_direct_urls_over_serp_html": { "type": "boolean" }, + "retry_with_different_urls_on_empty_fetch": { "type": "boolean" }, + "max_correction_attempts_after_block": { "type": "integer", "minimum": 0 } + } + }, + "review": { + "type": "object", + "required": [ + "fail_fast", + "response_schema", + "checks", + "do_not_deny_for", + "enforced_elsewhere" + ], + "additionalProperties": false, + "properties": { + "fail_fast": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": [ + "alignedWithRequest", + "confidence", + "suggestedAction", + "concerns", + "summary" + ], + "additionalProperties": false, + "properties": { + "alignedWithRequest": { "type": "string" }, + "confidence": { "type": "string" }, + "suggestedAction": { "type": "string" }, + "concerns": { "type": "string" }, + "summary": { "type": "string" } + } + }, + "checks": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/review_check" } + }, + "do_not_deny_for": { + "type": "array", + "items": { "type": "string" } + }, + "enforced_elsewhere": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "review_check": { + "type": "object", + "required": ["id", "order", "description"], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "order": { "type": "integer", "minimum": 1 }, + "description": { "type": "string" }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "rules": { + "type": "object", + "required": [ + "guest_has_no_network", + "host_performs_http", + "host_applies_ssrf", + "static_verifier_enforces_imports", + "scheduler_timing_not_in_script", + "deterministic_output_required", + "stable_sort_and_dedupe_collections", + "no_time_random_uuid_for_visible_output", + "match_http_results_by_request_id" + ], + "additionalProperties": false, + "properties": { + "guest_has_no_network": { "type": "boolean" }, + "host_performs_http": { "type": "boolean" }, + "host_applies_ssrf": { "type": "boolean" }, + "static_verifier_enforces_imports": { "type": "boolean" }, + "scheduler_timing_not_in_script": { "type": "boolean" }, + "deterministic_output_required": { "type": "boolean" }, + "stable_sort_and_dedupe_collections": { "type": "boolean" }, + "no_time_random_uuid_for_visible_output": { "type": "boolean" }, + "match_http_results_by_request_id": { "type": "boolean" } + } + }, + "plugin_factory": { + "type": "object", + "required": ["manifest", "builder", "review"], + "additionalProperties": false, + "properties": { + "manifest": { "$ref": "#/$defs/plugin_factory_manifest" }, + "builder": { "$ref": "#/$defs/plugin_factory_builder" }, + "review": { "$ref": "#/$defs/plugin_factory_review" } + } + }, + "plugin_factory_manifest": { + "type": "object", + "required": [ + "host_creates_manifest", + "do_not_return_manifest_json", + "agent_plugin_schema", + "entrypoint", + "plugin_id_allowed_chars", + "plugin_id_forbidden_chars", + "roles", + "connector_messaging_ops", + "secret_kinds", + "never_embed_credentials_in_go_source", + "host_lists_connectors_under_messaging" + ], + "additionalProperties": false, + "properties": { + "host_creates_manifest": { "type": "boolean" }, + "do_not_return_manifest_json": { "type": "boolean" }, + "agent_plugin_schema": { "type": "string" }, + "entrypoint": { "type": "string" }, + "plugin_id_allowed_chars": { "type": "string" }, + "plugin_id_forbidden_chars": { + "type": "array", + "items": { "type": "string" } + }, + "roles": { + "type": "array", + "items": { "type": "string" } + }, + "connector_messaging_ops": { + "type": "array", + "items": { "type": "string" } + }, + "secret_kinds": { + "type": "array", + "items": { "type": "string" } + }, + "never_embed_credentials_in_go_source": { "type": "boolean" }, + "host_lists_connectors_under_messaging": { "type": "boolean" } + } + }, + "plugin_factory_builder": { + "type": "object", + "required": [ + "response_keys", + "skill_files_path_pattern", + "empty_skill_files_when_unused", + "test_input_is_serialized_json_object", + "test_input_must_not_be_empty", + "test_input_exercises_terminal_result", + "connector_test_input_uses_hops_array", + "connector_parse_json_http_results_when_vendor_returns_json" + ], + "additionalProperties": false, + "properties": { + "response_keys": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "skill_files_path_pattern": { "type": "string" }, + "empty_skill_files_when_unused": { "type": "boolean" }, + "test_input_is_serialized_json_object": { "type": "boolean" }, + "test_input_must_not_be_empty": { "type": "boolean" }, + "test_input_exercises_terminal_result": { "type": "boolean" }, + "connector_test_input_uses_hops_array": { "type": "boolean" }, + "connector_parse_json_http_results_when_vendor_returns_json": { "type": "boolean" } + } + }, + "plugin_factory_review": { + "type": "object", + "required": [ + "compilation_success_not_approval", + "response_schema", + "reject_non_go_source", + "reject_raw_network_outside_http_request_envelopes", + "reject_missing_stdin_read", + "source_derived_titles_may_be_fragments", + "reject_unsupported_direct_test_claims", + "connector_rules_document" + ], + "additionalProperties": false, + "properties": { + "compilation_success_not_approval": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": ["decision", "summary", "findings"], + "additionalProperties": false, + "properties": { + "decision": { "type": "string" }, + "summary": { "type": "string" }, + "findings": { "type": "string" } + } + }, + "reject_non_go_source": { "type": "boolean" }, + "reject_raw_network_outside_http_request_envelopes": { "type": "boolean" }, + "reject_missing_stdin_read": { "type": "boolean" }, + "source_derived_titles_may_be_fragments": { "type": "boolean" }, + "reject_unsupported_direct_test_claims": { "type": "boolean" }, + "connector_rules_document": { "type": "string" } + } + } + } +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json new file mode 100644 index 00000000..9af39755 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/web-crawler-result.json", + "title": "Web crawler worker stdout", + "description": "JSON object written to stdout by derrick-web-crawler.", + "$ref": "worker-product.schema.json#/$defs/web_crawler_result" +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json new file mode 100644 index 00000000..874f6bfb --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/worker-product.json", + "title": "Derrick trusted worker product contracts", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "$defs": { + "string_list": { + "type": "array", + "items": { "type": "string" } + }, + "web_crawler_stop_reason": { + "type": "string", + "enum": [ + "completed", + "max_pages", + "max_depth", + "timeout", + "total_bytes", + "queue_limit", + "cancelled", + "blocked" + ] + }, + "web_crawler_page": { + "type": "object", + "required": ["url", "depth", "status_code", "title", "text", "links_found"], + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "depth": { "type": "integer" }, + "status_code": { "type": "integer" }, + "content_type": { "type": "string" }, + "title": { "type": "string" }, + "text": { "type": "string" }, + "links_found": { "type": "integer" } + } + }, + "web_crawler_result": { + "type": "object", + "required": [ + "ok", + "start_url", + "pages", + "stop_reason", + "requests_made", + "bytes_read", + "truncated", + "diagnostics" + ], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "start_url": { "type": "string" }, + "pages": { + "type": "array", + "items": { "$ref": "#/$defs/web_crawler_page" } + }, + "stop_reason": { "$ref": "#/$defs/web_crawler_stop_reason" }, + "requests_made": { "type": "integer" }, + "bytes_read": { "type": "integer" }, + "truncated": { "type": "boolean" }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + }, + "file_extractor_operation": { + "type": "string", + "enum": ["extract", "convert"] + }, + "file_extractor_file_result": { + "type": "object", + "required": ["input_name", "kind", "byte_count"], + "additionalProperties": false, + "properties": { + "input_name": { "type": "string" }, + "output_name": { "type": "string" }, + "kind": { "type": "string" }, + "byte_count": { "type": "integer" }, + "preview": { "type": "string" }, + "error": { "type": "string" } + } + }, + "file_extractor_result": { + "type": "object", + "required": ["ok", "operation", "files", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "operation": { "$ref": "#/$defs/file_extractor_operation" }, + "files": { + "type": "array", + "items": { "$ref": "#/$defs/file_extractor_file_result" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift b/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift new file mode 100644 index 00000000..83a308d8 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift @@ -0,0 +1,250 @@ +import Foundation + +/// Canonical Derrick script_exec protocol decoded from `script-exec-contract.json`. +public struct ScriptExecContractDocument: Codable, Sendable, Hashable { + public var version: Int + public var runtime: ScriptExecRuntimeSpec + public var io: ScriptExecIOSpec + public var workflow: ScriptExecWorkflowSpec + public var output: ScriptExecOutputSpec + public var agent: ScriptExecAgentSpec + public var review: ScriptExecReviewSpec + public var rules: ScriptExecProtocolRules + public var pluginFactory: ScriptExecPluginFactorySpec + + enum CodingKeys: String, CodingKey { + case version, runtime, io, workflow, output, agent, review, rules + case pluginFactory = "plugin_factory" + } +} + +public struct ScriptExecRuntimeSpec: Codable, Sendable, Hashable { + public var language: String + public var package: String + public var binary: String + public var stdlibOnly: Bool + public var forbiddenImports: [String] + public var allowedOSUsage: [String] + public var exampleGo: String + + enum CodingKeys: String, CodingKey { + case language, package, binary + case stdlibOnly = "stdlib_only" + case forbiddenImports = "forbidden_imports" + case allowedOSUsage = "allowed_os_usage" + case exampleGo = "example_go" + } +} + +public struct ScriptExecIOSpec: Codable, Sendable, Hashable { + public var stdinSchema: String + public var stdoutSchema: String + public var guestRuntimeSchema: String + + enum CodingKeys: String, CodingKey { + case stdinSchema = "stdin_schema" + case stdoutSchema = "stdout_schema" + case guestRuntimeSchema = "guest_runtime_schema" + } +} + +public struct ScriptExecWorkflowSpec: Codable, Sendable, Hashable { + public var firstHopVerbs: [String] + public var httpResultsEventKind: String + public var terminalVerbs: [String] + public var matchHTTPBy: String + public var httpResultsAccumulate: Bool + public var postBodyField: String + + enum CodingKeys: String, CodingKey { + case firstHopVerbs = "first_hop_verbs" + case httpResultsEventKind = "http_results_event_kind" + case terminalVerbs = "terminal_verbs" + case matchHTTPBy = "match_http_by" + case httpResultsAccumulate = "http_results_accumulate" + case postBodyField = "post_body_field" + } +} + +public struct ScriptExecOutputSpec: Codable, Sendable, Hashable { + public var fields: [String: ScriptExecOutputFieldSpec] + public var forbiddenPatterns: [String] + public var rawBodyInContentRequiresExplicitUserRequest: Bool + + enum CodingKeys: String, CodingKey { + case fields + case forbiddenPatterns = "forbidden_patterns" + case rawBodyInContentRequiresExplicitUserRequest = + "raw_body_in_content_requires_explicit_user_request" + } +} + +public struct ScriptExecOutputFieldSpec: Codable, Sendable, Hashable { + public var purpose: String + public var hostStripsIncidentalMarkup: Bool? + public var hostAllowlistSanitizes: Bool? + public var preferContentForExtractedText: Bool? + + enum CodingKeys: String, CodingKey { + case purpose + case hostStripsIncidentalMarkup = "host_strips_incidental_markup" + case hostAllowlistSanitizes = "host_allowlist_sanitizes" + case preferContentForExtractedText = "prefer_content_for_extracted_text" + } +} + +public struct ScriptExecAgentSpec: Codable, Sendable, Hashable { + public var preferDirectURLsOverSERPHTML: Bool + public var retryWithDifferentURLsOnEmptyFetch: Bool + public var maxCorrectionAttemptsAfterBlock: Int + + enum CodingKeys: String, CodingKey { + case preferDirectURLsOverSERPHTML = "prefer_direct_urls_over_serp_html" + case retryWithDifferentURLsOnEmptyFetch = "retry_with_different_urls_on_empty_fetch" + case maxCorrectionAttemptsAfterBlock = "max_correction_attempts_after_block" + } +} + +public struct ScriptExecReviewSpec: Codable, Sendable, Hashable { + public var failFast: Bool + public var responseSchema: [String: String] + public var checks: [ScriptExecReviewCheck] + public var doNotDenyFor: [String] + public var enforcedElsewhere: [String] + + enum CodingKeys: String, CodingKey { + case failFast = "fail_fast" + case responseSchema = "response_schema" + case checks + case doNotDenyFor = "do_not_deny_for" + case enforcedElsewhere = "enforced_elsewhere" + } +} + +public struct ScriptExecReviewCheck: Codable, Sendable, Hashable { + public var id: String + public var order: Int + public var description: String + public var notes: [String]? +} + +public struct ScriptExecProtocolRules: Codable, Sendable, Hashable { + public var guestHasNoNetwork: Bool + public var hostPerformsHTTP: Bool + public var hostAppliesSSRF: Bool + public var staticVerifierEnforcesImports: Bool + public var schedulerTimingNotInScript: Bool + public var deterministicOutputRequired: Bool + public var stableSortAndDedupeCollections: Bool + public var noTimeRandomUUIDForVisibleOutput: Bool + public var matchHTTPResultsByRequestID: Bool + + enum CodingKeys: String, CodingKey { + case guestHasNoNetwork = "guest_has_no_network" + case hostPerformsHTTP = "host_performs_http" + case hostAppliesSSRF = "host_applies_ssrf" + case staticVerifierEnforcesImports = "static_verifier_enforces_imports" + case schedulerTimingNotInScript = "scheduler_timing_not_in_script" + case deterministicOutputRequired = "deterministic_output_required" + case stableSortAndDedupeCollections = "stable_sort_and_dedupe_collections" + case noTimeRandomUUIDForVisibleOutput = "no_time_random_uuid_for_visible_output" + case matchHTTPResultsByRequestID = "match_http_results_by_request_id" + } +} + +public struct ScriptExecPluginFactorySpec: Codable, Sendable, Hashable { + public var manifest: ScriptExecPluginFactoryManifestSpec + public var builder: ScriptExecPluginFactoryBuilderSpec + public var review: ScriptExecPluginFactoryReviewSpec +} + +public struct ScriptExecPluginFactoryManifestSpec: Codable, Sendable, Hashable { + public var hostCreatesManifest: Bool + public var doNotReturnManifestJSON: Bool + public var agentPluginSchema: String + public var entrypoint: String + public var pluginIDAllowedChars: String + public var pluginIDForbiddenChars: [String] + public var roles: [String] + public var connectorMessagingOps: [String] + public var secretKinds: [String] + public var neverEmbedCredentialsInGoSource: Bool + public var hostListsConnectorsUnderMessaging: Bool + + enum CodingKeys: String, CodingKey { + case hostCreatesManifest = "host_creates_manifest" + case doNotReturnManifestJSON = "do_not_return_manifest_json" + case agentPluginSchema = "agent_plugin_schema" + case entrypoint + case pluginIDAllowedChars = "plugin_id_allowed_chars" + case pluginIDForbiddenChars = "plugin_id_forbidden_chars" + case roles + case connectorMessagingOps = "connector_messaging_ops" + case secretKinds = "secret_kinds" + case neverEmbedCredentialsInGoSource = "never_embed_credentials_in_go_source" + case hostListsConnectorsUnderMessaging = "host_lists_connectors_under_messaging" + } +} + +public struct ScriptExecPluginFactoryBuilderSpec: Codable, Sendable, Hashable { + public var responseKeys: [String] + public var skillFilesPathPattern: String + public var emptySkillFilesWhenUnused: Bool + public var testInputIsSerializedJSONObject: Bool + public var testInputMustNotBeEmpty: Bool + public var testInputExercisesTerminalResult: Bool + public var connectorTestInputUsesHopsArray: Bool + public var connectorParseJSONHTTPResultsWhenVendorReturnsJSON: Bool + + enum CodingKeys: String, CodingKey { + case responseKeys = "response_keys" + case skillFilesPathPattern = "skill_files_path_pattern" + case emptySkillFilesWhenUnused = "empty_skill_files_when_unused" + case testInputIsSerializedJSONObject = "test_input_is_serialized_json_object" + case testInputMustNotBeEmpty = "test_input_must_not_be_empty" + case testInputExercisesTerminalResult = "test_input_exercises_terminal_result" + case connectorTestInputUsesHopsArray = "connector_test_input_uses_hops_array" + case connectorParseJSONHTTPResultsWhenVendorReturnsJSON = + "connector_parse_json_http_results_when_vendor_returns_json" + } +} + +public struct ScriptExecPluginFactoryReviewSpec: Codable, Sendable, Hashable { + public var compilationSuccessNotApproval: Bool + public var responseSchema: [String: String] + public var rejectNonGoSource: Bool + public var rejectRawNetworkOutsideHTTPRequestEnvelopes: Bool + public var rejectMissingStdinRead: Bool + public var sourceDerivedTitlesMayBeFragments: Bool + public var rejectUnsupportedDirectTestClaims: Bool + public var connectorRulesDocument: String + + enum CodingKeys: String, CodingKey { + case compilationSuccessNotApproval = "compilation_success_not_approval" + case responseSchema = "response_schema" + case rejectNonGoSource = "reject_non_go_source" + case rejectRawNetworkOutsideHTTPRequestEnvelopes = + "reject_raw_network_outside_http_request_envelopes" + case rejectMissingStdinRead = "reject_missing_stdin_read" + case sourceDerivedTitlesMayBeFragments = "source_derived_titles_may_be_fragments" + case rejectUnsupportedDirectTestClaims = "reject_unsupported_direct_test_claims" + case connectorRulesDocument = "connector_rules_document" + } +} + +public enum ScriptExecContractError: Error, Equatable, LocalizedError, Sendable { + case missingResource(String) + case invalidJSON(String) + case integrityFailed(String) + + public var errorDescription: String? { + switch self { + case .missingResource(let name): + return "Missing bundled script_exec contract resource \(name)." + case .invalidJSON(let name): + return "Script_exec contract resource \(name) is not valid JSON." + case .integrityFailed(let detail): + return "Script_exec contract JSON failed schema checks: \(detail)" + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift b/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift new file mode 100644 index 00000000..81face78 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift @@ -0,0 +1,93 @@ +import Foundation + +/// Host checks that the bundled script_exec JSON still matches its schema and wire schemas. +public enum ScriptExecContractIntegrity: Sendable { + public static func validateBundledGraph() throws { + let contractData = try ScriptExecContractStore.resourceData( + name: ScriptExecContractStore.protocolResource, + subdirectory: "contracts" + ) + let contractJSON: [String: Any] + do { + contractJSON = try JSONSchema.object( + from: contractData, + name: ScriptExecContractStore.protocolResource + ) + } catch { + throw ScriptExecContractError.invalidJSON(ScriptExecContractStore.protocolResource) + } + try validateAgainstSchema(contractJSON, schema: .scriptExecContract) + try validateWireSchemaRefs(contractJSON) + try validateOutputFields(contractJSON) + try validateReviewChecks(contractJSON) + } + + private static func validateAgainstSchema(_ instance: Any, schema: GuestContract.Schema) throws { + do { + try GuestContract.validate(instance, against: schema) + } catch let error as GuestContractError { + throw ScriptExecContractError.integrityFailed(error.localizedDescription) + } + } + + private static func validateWireSchemaRefs(_ contract: [String: Any]) throws { + guard let io = contract["io"] as? [String: Any] else { return } + for key in ["stdin_schema", "stdout_schema", "guest_runtime_schema"] { + guard let fileName = io[key] as? String, + GuestContract.Schema(rawValue: fileName) != nil else { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json io.\(key) must reference a bundled guest schema." + ) + } + } + } + + private static func validateOutputFields(_ contract: [String: Any]) throws { + let envelope = try GuestContract.loadSchemaObject(.envelopeList) + guard let properties = (envelope["items"] as? [String: Any])?["properties"] as? [String: Any] else { + throw ScriptExecContractError.integrityFailed( + "envelope-list.schema.json is missing items.properties." + ) + } + let envelopeKeys = Set(properties.keys) + guard let output = contract["output"] as? [String: Any], + let fields = output["fields"] as? [String: Any] else { + return + } + let missing = Set(fields.keys).subtracting(envelopeKeys) + if !missing.isEmpty { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json output.fields references unknown envelope fields: \(missing.sorted().joined(separator: ", "))." + ) + } + } + + private static func validateReviewChecks(_ contract: [String: Any]) throws { + guard let review = contract["review"] as? [String: Any], + let checks = review["checks"] as? [[String: Any]] else { + return + } + var seenIDs: Set = [] + var seenOrders: Set = [] + for check in checks { + guard let id = check["id"] as? String, + let order = check["order"] as? Int else { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks entries require id and order." + ) + } + if seenIDs.contains(id) { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks has duplicate id \(id)." + ) + } + if seenOrders.contains(order) { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks has duplicate order \(order)." + ) + } + seenIDs.insert(id) + seenOrders.insert(order) + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift b/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift new file mode 100644 index 00000000..82278b35 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift @@ -0,0 +1,56 @@ +import Foundation + +/// Renders the bundled script_exec protocol for agent, builder, and reviewer prompts. +public enum ScriptExecContractPrompts: Sendable { + public static func builderGuide() -> String { + dumpOrUnavailable(preamble: """ + Offline Go guest rules come only from this protocol JSON and the wire schemas below. \ + Do not invent requirements that are not in the JSON. + """) + } + + public static func reviewerGuide() -> String { + dumpOrUnavailable(preamble: """ + You are a reviewer for script_exec declarations. Review against script-exec-contract.json only. \ + If a rule is not in the JSON, do not require it. Apply review.checks in order; when review.fail_fast \ + is true, return on the first failing check. Return only valid JSON matching review.response_schema. + """) + } + + public static func pluginFactoryBuilderGuide() -> String { + dumpOrUnavailable(preamble: """ + Plugin factory builder rules come only from script-exec-contract.json (plugin_factory, runtime, \ + workflow, output, rules) and connector-contract.json when building a connector. Do not invent \ + requirements that are not in those JSON documents. + """) + } + + public static func pluginFactoryReviewerGuide() -> String { + dumpOrUnavailable(preamble: """ + You are Derrick's independent plugin alignment and safety reviewer. Review guest go_source against \ + script-exec-contract.json (runtime, workflow, output, rules, plugin_factory.review). If a guest rule \ + is not in the JSON, do not require it. For connector plugins also apply connector-contract.json below. \ + Return exactly one JSON object matching plugin_factory.review.response_schema. + """) + } + + public static func dump(preamble: String) throws -> String { + _ = try ScriptExecContractStore.loadProtocol() + var lines: [String] = [preamble, "", "--- script-exec-contract.json ---"] + lines.append(try ScriptExecContractStore.loadProtocolText()) + lines.append("") + lines.append("--- \(GuestContract.Schema.guestRuntime.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.guestRuntime)) + lines.append("") + lines.append("--- \(GuestContract.Schema.hopEvent.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.hopEvent)) + lines.append("") + lines.append("--- \(GuestContract.Schema.envelopeList.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.envelopeList)) + return lines.joined(separator: "\n") + } + + private static func dumpOrUnavailable(preamble: String) -> String { + (try? dump(preamble: preamble)) ?? "Script_exec contract JSON failed to load." + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractStore.swift b/packages/Structure/Sources/Contract/ScriptExecContractStore.swift new file mode 100644 index 00000000..063fe59f --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractStore.swift @@ -0,0 +1,64 @@ +import CryptoKit +import Foundation + +/// Loads the bundled script_exec protocol JSON. +public enum ScriptExecContractStore: Sendable { + public static let protocolResource = "script-exec-contract.json" + public static let fingerprintEntries: [(subdirectory: String, file: String)] = [ + ("schemas", "script-exec-contract.schema.json"), + ("schemas", "guest-runtime.schema.json"), + ("schemas", "hop-event.schema.json"), + ("schemas", "envelope-list.schema.json"), + ("contracts", "script-exec-contract.json"), + ] + + public static var fingerprintSources: [String] { + fingerprintEntries.map { "\($0.subdirectory)/\($0.file)" } + } + + public static func loadProtocol() throws -> ScriptExecContractDocument { + try ScriptExecContractIntegrity.validateBundledGraph() + let data = try resourceData(name: protocolResource, subdirectory: "contracts") + do { + return try JSONDecoder().decode(ScriptExecContractDocument.self, from: data) + } catch { + throw ScriptExecContractError.invalidJSON(protocolResource) + } + } + + public static func loadProtocolText() throws -> String { + try utf8Text(resourceData(name: protocolResource, subdirectory: "contracts")) + } + + public static func computeFingerprint() throws -> String { + var joined = Data() + for entry in fingerprintEntries { + let relative = "\(entry.subdirectory)/\(entry.file)" + let data = try resourceData(name: entry.file, subdirectory: entry.subdirectory) + joined.append(Data(relative.utf8)) + joined.append(0) + joined.append(data) + joined.append(0) + } + let digest = SHA256.hash(data: joined) + return digest.map { String(format: "%02x", $0) }.joined() + } + + static func resourceData(name: String, subdirectory: String) throws -> Data { + guard let url = Bundle.module.url( + forResource: name, + withExtension: nil, + subdirectory: subdirectory + ) else { + throw ScriptExecContractError.missingResource("\(subdirectory)/\(name)") + } + return try Data(contentsOf: url) + } + + private static func utf8Text(_ data: Data) throws -> String { + guard let text = String(data: data, encoding: .utf8) else { + throw ScriptExecContractError.invalidJSON(protocolResource) + } + return text + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift index 489d77ac..ca2f57e1 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift @@ -13,8 +13,8 @@ public enum DerrickDockerRuntimeIdentity: Sendable { /// Name prefixes for current and unlabeled leftover containers. public static let namePrefixes = [ - "derrick-web-crawler", - "derrick-guest-runtime", + "derrick-web-crawler", + "derrick-guest-runtime", "derrick-swift-runtime", "derrick-file-extractor", ] diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift new file mode 100644 index 00000000..7253423f --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Optional host Go toolchain probe (development diagnostics). Guest compile runs in Docker. +public enum DerrickGoToolchain: Sendable { + public static let minimumVersion = "1.27.1" + + public static func ensureInstalled() throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = ["go", "version"] + process.environment = [ + "PATH": "/opt/homebrew/bin:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin", + ] + let stdout = Pipe() + process.standardOutput = stdout + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw DerrickGoToolchainError.missing + } + let text = String(data: stdout.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" + guard let version = parseVersion(text) else { + throw DerrickGoToolchainError.unparseable(text.trimmingCharacters(in: .whitespacesAndNewlines)) + } + guard versionSatisfies(version, minimum: minimumVersion) else { + throw DerrickGoToolchainError.tooOld(found: version, required: minimumVersion) + } + } + + static func parseVersion(_ text: String) -> String? { + // go version go1.27.1 darwin/arm64 + for part in text.split(separator: " ") { + let token = String(part) + if token.hasPrefix("go"), token.count > 2 { + return String(token.dropFirst(2)) + } + } + return nil + } + + static func versionSatisfies(_ found: String, minimum: String) -> Bool { + compareVersions(found, minimum) != .orderedAscending + } + + private enum Ordering { + case orderedAscending, orderedSame, orderedDescending + } + + private static func compareVersions(_ lhs: String, _ rhs: String) -> Ordering { + let left = lhs.split(separator: ".").map { Int($0) ?? 0 } + let right = rhs.split(separator: ".").map { Int($0) ?? 0 } + let count = max(left.count, right.count) + for index in 0.. r { return .orderedDescending } + } + return .orderedSame + } +} + +public enum DerrickGoToolchainError: Error, LocalizedError, Sendable { + case missing + case unparseable(String) + case tooOld(found: String, required: String) + + public var errorDescription: String? { + switch self { + case .missing: + return "Go \(DerrickGoToolchain.minimumVersion) or later was expected for diagnostics. Guest compile runs in Docker." + case .unparseable(let detail): + return "Could not read the installed Go version (\(detail))." + case .tooOld(let found, let required): + return "Go \(required) or later is required to build plugins (found \(found))." + } + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift new file mode 100644 index 00000000..baa9c04e --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift @@ -0,0 +1,47 @@ +import Foundation + +/// SHA-256 image ID from `docker image inspect --format '{{.Id}}'`. +public struct DockerImageDigest: RawRepresentable, Sendable, Hashable, Codable { + public let rawValue: String + + public init(rawValue: String) { + self.rawValue = Self.normalize(rawValue) + } + + public init?(hexDigest: String) { + let normalized = Self.normalize(hexDigest) + guard Self.isValid(normalized) else { return nil } + rawValue = normalized + } + + public static func normalize(_ value: String) -> String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if trimmed.hasPrefix("sha256:") { + return trimmed + } + if trimmed.count == 64 { + return "sha256:\(trimmed)" + } + return trimmed + } + + public static func isValid(_ value: String) -> Bool { + let hex = value.hasPrefix("sha256:") ? String(value.dropFirst(7)) : value + guard hex.count == 64 else { return false } + return hex.unicodeScalars.allSatisfy { CharacterSet(charactersIn: "0123456789abcdef").contains($0) } + } +} + +public enum DockerImageDigestError: Error, LocalizedError, Sendable, Equatable { + case imageMissing(String) + case digestMismatch(tag: String, expected: DockerImageDigest, actual: DockerImageDigest) + + public var errorDescription: String? { + switch self { + case .imageMissing(let tag): + return "The worker image \(tag) is not installed." + case .digestMismatch(let tag, _, _): + return "The worker image \(tag) does not match the version shipped with Derrick. Rebuild or reinstall product images." + } + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift new file mode 100644 index 00000000..666ff6bf --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift @@ -0,0 +1,6 @@ +import Foundation + +/// Generated by scripts/record-docker-image-digests.sh — do not edit. +public enum DockerProductImageDigests: Sendable { + public static let worker = DockerImageDigest(rawValue: "sha256:686c54118b75056b7ca9af17697516fc242e65b7a8da0e43bb6c0a35f31e8052") +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift index bfbbfd37..9f323fea 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift @@ -1,16 +1,39 @@ import Foundation -/// Trusted product images built from in-repo Dockerfiles (not pulled from a registry). +/// Trusted product Docker images built from in-repo Dockerfiles (not pulled from a registry). public enum DockerProductImagePolicy: Sendable { + public static let workerImage = DockerWorkerRuntime.image + public static let workerDockerfileRelativePath = DockerWorkerRuntime.dockerfileRelativePath + public static let workerBuildContextRelativePath = DockerWorkerRuntime.buildContextRelativePath + + /// Legacy crawler tag — retained for orphan sweeps only. public static let webCrawlerImage = "derrick-web-crawler:swift-6.4-v1" public static let webCrawlerDockerfileRelativePath = "docker/web-crawler/Dockerfile" - /// Sibling Swift packages only — not the whole git checkout. public static let webCrawlerBuildContextRelativePath = "packages" public static let allowedBuildImageTags: Set = [ - webCrawlerImage, + workerImage, ] + public static func workerBuildContext(repoRoot: URL) -> URL { + repoRoot.standardizedFileURL + } + + public static func isAllowedWorkerBuild( + dockerfilePath: String, + imageTag: String, + contextPath: String + ) -> Bool { + guard imageTag == workerImage else { return false } + let dockerfileURL = URL(fileURLWithPath: dockerfilePath).standardizedFileURL + let contextURL = URL(fileURLWithPath: contextPath).standardizedFileURL + let repoRoot = contextURL.standardizedFileURL + let expectedDockerfile = repoRoot + .appendingPathComponent(workerDockerfileRelativePath) + .standardizedFileURL + return dockerfileURL.path == expectedDockerfile.path + } + public static func webCrawlerBuildContext(repoRoot: URL) -> URL { repoRoot.appendingPathComponent(webCrawlerBuildContextRelativePath).standardizedFileURL } diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift new file mode 100644 index 00000000..248376c2 --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift @@ -0,0 +1,32 @@ +import Foundation + +/// Unified Go worker image shared by crawl, extract, and plugin guest execution. +public enum DockerWorkerRuntime: Sendable { + public static let image = "derrick-worker:go-v1" + public static let dockerfileRelativePath = "docker/worker/Dockerfile" + public static let buildContextRelativePath = "." + + public static let crawlerBinary = "/usr/local/bin/derrick-web-crawler" + public static let extractorBinary = "/usr/local/bin/derrick-file-extractor" + /// Binaries that must exist in the unified worker image. + public static let requiredBinaries: [String] = [ + crawlerBinary, + extractorBinary, + ] + + public static let guestBinaryPath = "/tmp/guest" + public static let guestSourcePath = "/tmp/plugin.go" + public static let goBinaryPath = "/usr/local/go/bin/go" + + /// Allowed `docker exec … sh -c` payloads for guest source I/O and in-container compile. + public static let guestWriteSourceShell = "cat > /tmp/plugin.go" + public static let guestCompileShell = + "cd /tmp && /usr/local/go/bin/go build -trimpath -ldflags=\"-s -w\" -o guest plugin.go && chmod +x guest" + public static let guestReadBinaryShell = "cat /tmp/guest" + + public static let pinnedDigest = DockerProductImageDigests.worker + + /// OCI label written by `docker/worker/Dockerfile`; used to detect stale local images. + public static let binariesLabelKey = "derrick.worker.binaries" + public static let binariesLabelValue = "crawler,extractor" +} diff --git a/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift b/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift index 5fb77b5d..80d556d8 100644 --- a/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift +++ b/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift @@ -2,11 +2,11 @@ import Foundation import MCP public enum GuestScriptLanguage: String, Sendable, Equatable { - case python + case go - public var verifierID: String { "python-check-v1" } + public var verifierID: String { "go-check-v1" } - /// `language` is optional and must be Python when set. + /// `language` is optional and must be Go when set. public static func requestedLanguageIsUnsupported(_ arguments: [String: Value]) -> Bool { guard let raw = arguments["language"]?.stringValue? .trimmingCharacters(in: .whitespacesAndNewlines) @@ -15,7 +15,7 @@ public enum GuestScriptLanguage: String, Sendable, Equatable { else { return false } - return raw != "python" && raw != "py" + return raw != "go" && raw != "golang" } } diff --git a/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift b/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift index 97481bae..3e59478a 100644 --- a/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift +++ b/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift @@ -59,7 +59,7 @@ public enum ScriptFailureStage: String, Codable, Sendable, Equatable { case none /// Static verifier rejected the request before run. case staticValidation - /// Leftover Swift compiler stage. Guest scripts are Python; this case remains for stored outcomes. + /// Go compile stage before container execution. case typecheck /// LLM security reviewer rejected (or could not complete when required). case llmReview @@ -253,7 +253,7 @@ public struct ScriptExecutionResult: Sendable { stderr: String, durationMS: Int, phaseTiming: ScriptPhaseTiming?, - verifier: String = "python-check-v1" + verifier: String = "go-check-v1" ) -> ScriptExecutionResult { let combined = stdout + "\n" + stderr let looksLikeEgress = combined.localizedCaseInsensitiveContains("UNAUTHORIZED_EGRESS") diff --git a/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift b/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift index 2976abe7..a2ff70bd 100644 --- a/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift +++ b/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift @@ -43,7 +43,7 @@ public enum AllowedMCPTool: String, CaseIterable, Sendable, Codable, Hashable { public var defaultDescription: String { switch self { case .scriptExec: - return "Run declared standalone Python in a constrained Docker container after verification. Emit HTTP request envelopes; the host performs the request." + return "Run declared standalone Go in a constrained Docker container after verification. Emit HTTP request envelopes; the host performs the request." case .sessionMemorySearch: return "Search prior session memory entries with optional query and paging." case .agentsSpawn: @@ -69,7 +69,7 @@ public enum AllowedMCPTool: String, CaseIterable, Sendable, Codable, Hashable { case .pluginInvoke: return "Run one approved compiled Agent Plugin by id with a JSON input object." case .webCrawl: - return "Crawl a bounded same-origin website in an isolated Swift container and return structured page summaries." + return "Crawl a bounded same-origin website in an isolated container and return structured page summaries. Call directly in chat; use jobs_create only for crawls likely to exceed about one minute." case .filesExtract: return "Extract text or convert attached chat files (PDF, DOCX, XLSX, CSV, HTML) in an isolated Swift container. Call this tool directly; do not submit it through jobs_create." } diff --git a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift new file mode 100644 index 00000000..033bd9e7 --- /dev/null +++ b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift @@ -0,0 +1,50 @@ +import Foundation + +/// Go contract shown to models that generate Derrick plugin guest programs. +public enum DerrickGuestGo: Sendable { + public static func source(for spec: PluginSpec? = nil) throws -> String { + var sections = [ScriptExecContractPrompts.builderGuide()] + if let spec { + sections.append( + """ + Plugin parameters are delivered in the input object's `params` object. + The parameter contract is: + \(try spec.goParameterDeclaration()) + + --- \(GuestContract.Schema.connectorParams.rawValue) --- + \(try GuestContract.loadSchemaText(.connectorParams)) + """ + ) + } + return sections.joined(separator: "\n\n") + } +} + +private extension PluginSpec { + func goParameterDeclaration() throws -> String { + _ = try validated() + let fields = parameters.map { parameter in + " \(parameter.name) \(parameterType(parameter.type))" + } + return """ + type PluginParams struct { + \(fields.joined(separator: "\n")) + } + """ + } + + func parameterType(_ type: PluginParameterType) -> String { + switch type { + case .string: + return "string" + case .number: + return "float64" + case .boolean: + return "bool" + case .stringList: + return "[]string" + case .numberList: + return "[]float64" + } + } +} diff --git a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift deleted file mode 100644 index 3ef8d5e7..00000000 --- a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift +++ /dev/null @@ -1,81 +0,0 @@ -import Foundation - -/// Python contract shown to models that generate standalone Derrick guest programs. -public enum DerrickGuestPython: Sendable { - public static let modelContract = """ - Python guest contract: - - The program is a standalone Python script run as `python3 /tmp/guest.py`. - - Read one JSON object from standard input (`sys.stdin`). - - Write one JSON array of envelope objects to standard output. Every object needs `verb` from the envelope-list schema. - - POST bodies go in `json`. The host decodes `http.request` as HostHTTPRequest and sends `json` as the HTTP body. - - On the first event, emit `http.request` envelopes for host HTTP. - - On an `http_results` event, emit `result.emit` or `message.post`. - - The host, not the Python container, performs HTTP and supplies response bodies. - - Do not use socket, urllib, requests, httpx, subprocess, or filesystem access. - - Use only the Python standard library (no pip/uv dependencies in script_exec). - - For repeatable output, match HTTP responses by request_id, sort and de-duplicate collections - by stable keys, and never use current time, randomness, UUIDs, response arrival order, or - dict/set iteration order for user-visible output. - - Later http_results events include earlier responses plus the newest ones. Match by request_id. - - Minimal output pattern: - ```python - import json, sys - event = json.load(sys.stdin) - def emit(envelopes): - json.dump(envelopes, sys.stdout, separators=(",", ":")) - ``` - Inspect `event.get("kind")` and `event.get("http_results")` to choose the next envelopes. - - Example request envelope: - {"verb":"http.request","request_id":"news-1","method":"GET","url":"https://example.com/feed.xml"} - - POST bodies: put the JSON value in `json`. The host deserializes `http.request` into HostHTTPRequest and sends `json` as the HTTP body. - {"verb":"http.request","request_id":"send-1","method":"POST","url":"https://example.com/api","headers":{"Content-Type":"application/json"},"json":{"channel":"C1","text":"hello"}} - - Example result envelope: - {"verb":"result.emit","title":"Result","summary":"User-readable output"} - """ - - public static func source(for spec: PluginSpec? = nil) throws -> String { - var sections = [modelContract] - if let spec { - sections.append( - """ - Plugin parameters are delivered in the input object's `params` object. - The parameter contract is: - \(try spec.pythonParameterDeclaration()) - """ - ) - } - return sections.joined(separator: "\n\n") - } -} - -private extension PluginSpec { - func pythonParameterDeclaration() throws -> String { - _ = try validated() - let fields = parameters.map { parameter in - " \(parameter.name): \(parameterType(parameter.type))" - } - return """ - class PluginParams(TypedDict): - \(fields.joined(separator: "\n")) - """ - } - - func parameterType(_ type: PluginParameterType) -> String { - switch type { - case .string: - return "str" - case .number: - return "float" - case .boolean: - return "bool" - case .stringList: - return "list[str]" - case .numberList: - return "list[float]" - } - } -} diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift index 5548c6a3..20586073 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift @@ -1,7 +1,7 @@ import Foundation public enum PluginGuestLanguage: String, Sendable, Equatable, Codable { - case python + case go } /// Parsed `app.derrick/runtime.json` from an approved factory release. @@ -9,7 +9,7 @@ public struct PluginFactoryRuntime: Sendable, Equatable { public let language: PluginGuestLanguage public let entrypoint: String - public init(language: PluginGuestLanguage = .python, entrypoint: String) { + public init(language: PluginGuestLanguage = .go, entrypoint: String) { self.language = language self.entrypoint = entrypoint } @@ -22,6 +22,36 @@ public struct PluginFactoryRuntime: Sendable, Equatable { else { return nil } - return PluginFactoryRuntime(entrypoint: entrypoint) + let languageRaw = (object["language"] as? String) ?? PluginGuestLanguage.go.rawValue + let language = PluginGuestLanguage(rawValue: languageRaw) ?? .go + return PluginFactoryRuntime(language: language, entrypoint: entrypoint) + } + + /// Package-relative guest source path used when hashing and verifying releases. + public static func guestSourcePackagePath( + runtimeJSON: String, + manifestJSON: String, + defaultPath: String = "app.derrick/plugin.go" + ) -> String { + if let runtime = decode(from: runtimeJSON) { + return normalizePackageRelativePath(runtime.entrypoint) + } + if let data = manifestJSON.data(using: .utf8), + let manifest = try? AgentPluginManifest.decode(data), + let entrypoint = manifest.derrick?.entrypoint { + return normalizePackageRelativePath(entrypoint) + } + return defaultPath + } + + private static func normalizePackageRelativePath(_ entrypoint: String) -> String { + var path = entrypoint.trimmingCharacters(in: .whitespacesAndNewlines) + if path.hasPrefix("./") { + path = String(path.dropFirst(2)) + } + while path.hasPrefix("/") { + path = String(path.dropFirst()) + } + return path } } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift index aaed1995..74169873 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift @@ -77,7 +77,10 @@ public enum PluginFactoryHopTestRunner: Sendable { testInput: Data, executor: any PluginFactoryExecutor ) async throws -> PluginFactoryHopTestRun { - try await run(testInput: testInput) { input in + if let compiled = executor as? any PluginFactoryCompiledGuestExecutor { + return try await compiled.runGuestSourceHops(source: source, testInput: testInput) + } + return try await run(testInput: testInput) { input in try await executor.runGuestSource(source: source, input: input) } } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift index bea96cf2..fdecbf55 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift @@ -2,9 +2,9 @@ import Foundation public typealias PluginFactoryLogger = @Sendable (String) async -> Void -/// The factory creates Agent Plugin packages whose Derrick entrypoint is Python. -/// A draft is a standalone file: the container runs it with `python3 /tmp/guest.py`. -/// A released version stores UTF-8 source as the packaged artifact. +/// The factory creates Agent Plugin packages whose Derrick entrypoint is Go. +/// A draft is compiled to a Linux binary and run as `/tmp/guest` in the worker image. +/// A released version stores the compiled binary as the packaged artifact. public struct PluginFactoryDraft: Sendable, Hashable { public let manifestJSON: String public let guestSource: String @@ -146,7 +146,7 @@ public struct PluginFactoryManifestInput: Sendable, Hashable { throw PluginFactoryError.invalidManifest("Version is required.") } var derrick: [String: Any] = [ - "entrypoint": "./app.derrick/plugin.py", + "entrypoint": "./app.derrick/plugin.go", ] if !secrets.isEmpty { derrick["secrets"] = secrets.map(\.jsonObject) @@ -204,7 +204,7 @@ public struct PluginFactoryBuilderRequest: Sendable, Hashable { public let userGoal: String public let previousDraft: PluginFactoryDraft? public let feedback: String? - /// When set, the host writes `plugin.json`. The builder only supplies Python and tests. + /// When set, the host writes `plugin.json`. The builder only supplies Go source and tests. public let hostManifest: PluginFactoryManifestInput? public init( @@ -288,7 +288,7 @@ public struct PluginFactoryBuilderResponse: Codable, Sendable, Hashable { enum CodingKeys: String, CodingKey { case pluginID = "plugin_id" case version, description - case guestSource = "python_source" + case guestSource = "go_source" case legacySwiftSource = "swift_source" case testInputJSON = "test_input_json" case skillFiles = "skill_files" @@ -380,14 +380,19 @@ public struct PluginFactoryExecutionResult: Sendable, Hashable { } } -/// The host supplies this adapter. Its production implementation runs these -/// commands inside the restricted Linux Swift Docker container. +/// The host supplies this adapter. Its production implementation compiles and +/// runs guests inside the pinned Go worker Docker container. public protocol PluginFactoryExecutor: Sendable { func runGuestSource(source: String, input: Data) async throws -> PluginFactoryExecutionResult func packageGuestSource(source: String) async throws -> Data func runPackagedArtifact(_ artifact: Data, input: Data) async throws -> PluginFactoryExecutionResult } +/// Optional compile-once hop replay for factory direct tests. +public protocol PluginFactoryCompiledGuestExecutor: PluginFactoryExecutor { + func runGuestSourceHops(source: String, testInput: Data) async throws -> PluginFactoryHopTestRun +} + public enum PluginReviewDecision: String, Sendable, Hashable { case approved case rejected @@ -526,10 +531,14 @@ public struct PluginFactoryRelease: Sendable, Hashable { } public func packageFiles() -> [String: Data] { + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: manifestJSON + ) var files: [String: Data] = [ "plugin.json": Data(manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.py": Data(guestSource.utf8), + guestPath: Data(guestSource.utf8), "app.derrick/plugin": compiledArtifact, ] for (path, body) in skillFiles { @@ -571,15 +580,15 @@ public enum PluginFactoryError: Error, LocalizedError, Equatable, Sendable { case .invalidSkillPath(let path): return "Invalid skill path '\(path)'. Skill path must be skills//SKILL.md." case .reservedPluginID(let id): return "The plugin id '\(id)' is reserved by Derrick." - case .invalidSource(let message): return "Invalid Python guest source: \(message)" - case .directRunFailed(let message): return "Python draft test failed: \(message)" - case .invalidDirectOutput(let message): return "Python draft returned invalid plugin output: \(message)" + case .invalidSource(let message): return "Invalid Go guest source: \(message)" + case .directRunFailed(let message): return "Go draft test failed: \(message)" + case .invalidDirectOutput(let message): return "Go draft returned invalid plugin output: \(message)" case .reviewRejected(let summary, let findings): let detail = findings.isEmpty ? summary : "\(summary) \(findings.joined(separator: " "))" return "Plugin review rejected the draft: \(detail)" - case .packageFailed(let message): return "Python plugin packaging failed: \(message)" + case .packageFailed(let message): return "Go plugin packaging failed: \(message)" case .packagedRunFailed(let message): return "Packaged plugin test failed: \(message)" case .invalidPackagedOutput(let message): return "Packaged plugin returned invalid output: \(message)" case .draftValidationFailed(let findings): diff --git a/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift b/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift index bcfe23ef..9b95c902 100644 --- a/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift +++ b/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift @@ -1,6 +1,6 @@ import Foundation -/// Derrick runtime metadata (`app.derrick/runtime.json`) for a standalone Python entrypoint. +/// Derrick runtime metadata (`app.derrick/runtime.json`) for a standalone Go entrypoint. public struct DerrickRuntime: Codable, Sendable, Hashable { public var entrypoint: String public var dependencies: [String: String] @@ -66,7 +66,7 @@ public struct DerrickRuntime: Codable, Sendable, Hashable { if trimmed.hasPrefix("./") { return try PluginPath.validateRuntimeEntrypoint(trimmed) } - guard trimmed.hasSuffix(".py"), + guard trimmed.hasSuffix(".go"), !trimmed.contains("/"), !trimmed.contains("\\") else { throw PluginManifestError.invalidEntrypoint(raw) diff --git a/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift b/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift index 8a08c898..d575c57b 100644 --- a/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift +++ b/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift @@ -47,7 +47,7 @@ public enum PluginManifestError: Error, Equatable, LocalizedError { case .invalidFieldType(let f): return "plugin.json field has the wrong type: \(f)" case .invalidEntrypoint(let p): - return "Entrypoint must be a plugin-relative .py path: \(p)" + return "Entrypoint must be a plugin-relative .go path: \(p)" case .pathNotRelative(let p): return "Path must be plugin-relative and start with ./: \(p)" case .pathEscapesRoot(let p): diff --git a/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift b/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift index f5cf9141..fd5df8c3 100644 --- a/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift +++ b/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift @@ -19,18 +19,18 @@ public enum PluginPath { return trimmed } - /// Python factory entrypoints are standalone files run by `python3`. - public static func validatePythonEntrypoint(_ raw: String) throws -> String { + /// Go factory entrypoints are standalone `package main` files compiled to `/tmp/guest`. + public static func validateGoEntrypoint(_ raw: String) throws -> String { let path = try validateRelative(raw) - guard path.hasSuffix(".py") else { + guard path.hasSuffix(".go") else { throw PluginManifestError.invalidEntrypoint(raw) } return path } - /// Accepts the supported Derrick guest runtime source file (.py). + /// Accepts the supported Derrick guest runtime source file (.go). public static func validateRuntimeEntrypoint(_ raw: String) throws -> String { - try validatePythonEntrypoint(raw) + try validateGoEntrypoint(raw) } public static func resolve(root: URL, relative: String) throws -> URL { diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index 5d8b2588..40413446 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -84,11 +84,11 @@ import Testing #expect(decoded.executableFingerprint == nil) } - @Test func bundledScriptReviewerInstructionsLoadFromSourceTree() throws { - let scriptReviewer = try DerrickBundledText.load("script_reviewer_instructions.md") - #expect(scriptReviewer.contains("intent alignment")) - #expect(scriptReviewer.contains("secret literals")) - #expect(scriptReviewer.contains("Python verifier")) + @Test func scriptExecReviewerPromptLoadsFromBundledContract() { + let scriptReviewer = ScriptExecContractPrompts.reviewerGuide() + #expect(scriptReviewer.contains("script-exec-contract.json")) + #expect(scriptReviewer.contains("intent_alignment")) + #expect(scriptReviewer.contains("If a rule is not in the JSON")) } @Test func healthDecodesLegacyPayloadWithoutGuestRuntime() throws { @@ -369,7 +369,7 @@ import Testing @Test func slackConnectorFallsBackToBotTokenWhenManifestOmitsSecrets() { let json = """ {"$schema":"https://example.invalid/agent-plugin.json","name":"slack-connector","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads"]}}} """ let descriptors = PluginSecretField.resolvedDescriptors( pluginID: "slack-connector", @@ -1022,15 +1022,15 @@ import Testing reportedFingerprint: "a", expectedFingerprint: "a", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( !DerrickDaemonHygiene.shouldRetireConnectedDaemon( reportedFingerprint: "a", expectedFingerprint: "a", - reportedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + reportedGuestRuntime: DerrickGuestRuntime.guestDockerImage, + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1038,7 +1038,7 @@ import Testing reportedFingerprint: "old", expectedFingerprint: "new", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1046,7 +1046,7 @@ import Testing reportedFingerprint: "a", expectedFingerprint: "a", reportedGuestRuntime: "stale-guest:old", - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1054,7 +1054,7 @@ import Testing reportedFingerprint: nil, expectedFingerprint: "a", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1062,7 +1062,7 @@ import Testing reportedFingerprint: "a", expectedFingerprint: nil, reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) } @@ -1214,10 +1214,10 @@ import Testing #expect(!DerrickDockerRuntimeIdentity.isAllowedPsFilter("name=nginx")) #expect( DerrickDockerRuntimeIdentity.createHasRuntimeLabel( - ["create"] + DerrickDockerRuntimeIdentity.createLabelArguments + ["python:3.14.7"] + ["create"] + DerrickDockerRuntimeIdentity.createLabelArguments + [DockerWorkerRuntime.image] ) ) - #expect(!DerrickDockerRuntimeIdentity.createHasRuntimeLabel(["create", "--name", "x", "python:3.14.7"])) + #expect(!DerrickDockerRuntimeIdentity.createHasRuntimeLabel(["create", "--name", "x", DockerWorkerRuntime.image])) } @Test func webCrawlerProductImageBuildUsesPackagesContext() { @@ -1317,12 +1317,12 @@ import Testing ) } - @Test func effectorAdmissionDeniesLiveChatWithoutContext() { + @Test func effectorAdmissionAllowsLiveChatWithoutContext() { #expect( EffectorAdmissionPolicy.allowsSyncWebCrawl( context: nil, principal: .agent(sessionID: "s1", agentID: "a1") - ) == false + ) ) } @@ -1476,17 +1476,16 @@ import Testing #expect(!PluginFactoryValidationExpectations.isSendOnlyConnector(manifestJSON: sendAndReceive)) } - @Test func connectorFactoryFailureReturnsToVendorStep() { - #expect(PluginFactoryCreateInput.failureStep(forStage: "docs") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "factory") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "review") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "description") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "type") == .type) - #expect(PluginFactoryCreateInput.failureStep(forStage: "name") == .name) - #expect(PluginFactoryCreateInput.failureStep(forStage: "auth") == .auth) - #expect(PluginFactoryCreateInput.failureStep(forStage: "discover") == .auth) - #expect(PluginFactoryCreateInput.failureStep(forStage: "paywall") == .news) - #expect(PluginFactoryCreateInput.failureStep(forStage: "news") == .news) + @Test func pluginStudioFailureMapsToSkillFirstSteps() { + #expect(PluginFactoryCreateInput.failureStep(forStage: "docs") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "factory") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "review") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "description") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "type") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "name") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "auth") == .credentials) + #expect(PluginFactoryCreateInput.failureStep(forStage: "discover") == .credentials) + #expect(PluginFactoryCreateInput.failureStep(forStage: "goal") == .goal) } @Test func connectorWizardOffersFullSyncOnly() { diff --git a/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift b/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift index 8bde0814..6ab15dec 100644 --- a/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift +++ b/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift @@ -204,7 +204,7 @@ private func slackFullSyncGoal() -> String { private func slackFullSyncManifestJSON() -> String { """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ } diff --git a/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift new file mode 100644 index 00000000..b618f0f7 --- /dev/null +++ b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift @@ -0,0 +1,29 @@ +import Testing +@testable import Structure + +@Suite struct DerrickGoToolchainTests { + @Test func minimumVersionIsPinned() { + #expect(DerrickGoToolchain.minimumVersion == "1.27.1") + } + + @Test func parseVersionReadsGoVersionLine() { + #expect( + DerrickGoToolchain.parseVersion("go version go1.27.1 darwin/arm64") == "1.27.1" + ) + } + + @Test func parseVersionRejectsEmptyOutput() { + #expect(DerrickGoToolchain.parseVersion("") == nil) + #expect(DerrickGoToolchain.parseVersion("go version") == nil) + } + + @Test func versionSatisfiesAcceptsCurrentAndNewer() { + #expect(DerrickGoToolchain.versionSatisfies("1.27.1", minimum: "1.27.1")) + #expect(DerrickGoToolchain.versionSatisfies("1.28.0", minimum: "1.27.1")) + } + + @Test func versionSatisfiesRejectsOlder() { + #expect(!DerrickGoToolchain.versionSatisfies("1.27.0", minimum: "1.27.1")) + #expect(!DerrickGoToolchain.versionSatisfies("1.26.9", minimum: "1.27.1")) + } +} diff --git a/packages/Structure/Tests/StructureTests/GuestContractTests.swift b/packages/Structure/Tests/StructureTests/GuestContractTests.swift index d8c3d834..58d1a06d 100644 --- a/packages/Structure/Tests/StructureTests/GuestContractTests.swift +++ b/packages/Structure/Tests/StructureTests/GuestContractTests.swift @@ -10,6 +10,12 @@ import Testing } } + @Test func guestRuntimeSchemaRequiresGoLanguage() throws { + let schema = try GuestContract.loadSchemaObject(.guestRuntime) + let language = (schema["properties"] as? [String: Any])?["language"] as? [String: Any] + #expect(language?["const"] as? String == "go") + } + @Test func executionContextSchemaExposesWorkflowKinds() throws { let kinds = try GuestContract.officialWorkflowKinds() #expect(kinds.contains("plugin_factory_create")) @@ -104,4 +110,37 @@ import Testing try GuestContract.validate(json: Data(json.utf8), against: .envelopeList) } } + + @Test func webCrawlerResultValidationAcceptsMinimalSuccess() throws { + let json = """ + {"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":[]} + """ + try GuestContractValidation.validateWebCrawlerResultJSON(Data(json.utf8)) + } + + @Test func webCrawlerResultValidationRejectsNullDiagnostics() { + let json = """ + {"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":null} + """ + #expect(throws: GuestContractError.self) { + try GuestContractValidation.validateWebCrawlerResultJSON(Data(json.utf8)) + } + } + + @Test func fileExtractorResultValidationAcceptsMinimalSuccess() throws { + let json = """ + {"ok":true,"operation":"extract","files":[],"diagnostics":[]} + """ + try GuestContractValidation.validateFileExtractorResultJSON(Data(json.utf8)) + } + + @Test func fileExtractorResultValidationRejectsNullFiles() { + let json = """ + {"ok":false,"operation":"extract","files":null,"diagnostics":[]} + """ + #expect(throws: GuestContractError.self) { + try GuestContractValidation.validateFileExtractorResultJSON(Data(json.utf8)) + } + } + } diff --git a/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift b/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift new file mode 100644 index 00000000..0659531a --- /dev/null +++ b/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift @@ -0,0 +1,29 @@ +import Foundation +import Testing +import Structure + +@Suite struct NewsPaywallTests { + @Test func paywallWarningIsUserFacing() { + #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("paywall")) + #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("not supported")) + } + + @Test func preflightRejectsPaywalledArticleAllowsFeed() { + #expect( + NewsPaywall.preflightRejection( + url: URL(string: "https://www.nytimes.com/2024/01/01/world.html")! + ) != nil + ) + #expect( + NewsPaywall.preflightRejection( + url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")! + ) == nil + ) + } + + @Test func canonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint() { + let url = URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + let canonical = NewsSourceURL.canonicalFetchURL(url, contextHint: "tech-news Tech") + #expect(canonical.path.contains("/headlines/section/topic/TECHNOLOGY")) + } +} diff --git a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift deleted file mode 100644 index 7b0f12a5..00000000 --- a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift +++ /dev/null @@ -1,178 +0,0 @@ -import Foundation -import Testing -import Structure - -@Suite struct NewsReaderTests { - @Test func paywallWarningIsUserFacing() { - #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("paywall")) - #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("not supported")) - } - - @Test func preflightRejectsPaywalledArticleAllowsFeed() { - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://www.nytimes.com/2024/01/01/world.html")! - ) != nil - ) - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")! - ) == nil - ) - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://feeds.bbci.co.uk/news/rss.xml")! - ) == nil - ) - } - - @Test func knownPaywallHostIsDetected() { - #expect(NewsPaywall.hostLooksPaywalled(URL(string: "https://www.nytimes.com/2024/01/01/world.html")!)) - #expect(!NewsPaywall.hostLooksPaywalled(URL(string: "https://feeds.bbci.co.uk/news/rss.xml")!)) - } - - @Test func htmlPaywallIsRejected() { - let html = "Subscribe to continue reading this article" - let reason = NewsPaywall.rejectionReason( - url: URL(string: "https://www.nytimes.com/story")!, - status: 200, - contentType: "text/html", - body: Data(html.utf8) - ) - #expect(reason != nil) - #expect(reason?.localizedCaseInsensitiveContains("paywall") == true) - } - - @Test func rssFromPaywalledPublisherIsAllowed() { - let rss = """ - Feed - """ - let reason = NewsPaywall.rejectionReason( - url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")!, - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - ) - #expect(reason == nil) - } - - @Test func parserReadsRssItemsWithLinks() { - let rss = """ - - - Hello worldhttps://example.com/helloBody - - """ - let entries = NewsFeedParser.parse( - data: Data(rss.utf8), - sourceLabel: "Example", - fallbackPageURL: URL(string: "https://example.com/feed")! - ) - #expect(entries.count == 1) - #expect(entries[0].title == "Hello world") - #expect(entries[0].sourceURL == "https://example.com/hello") - } - - @Test func refreshFailsPaywalledURL() async throws { - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "text/html", - body: Data("This article is for subscribers only".utf8) - )) - let spec = NewsReaderSpec( - name: "Test", - topics: [], - sources: [NewsSource(label: "NYT", url: "https://www.nytimes.com/story")] - ) - do { - _ = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - Issue.record("expected paywall failure") - } catch let error as NewsReaderError { - guard case .paywalled = error else { - Issue.record("expected paywalled, got \(error)") - return - } - } - } - - @Test func refreshReturnsLinkedItemsCapped() async throws { - let rss = """ - - Onehttps://example.com/1 - Twohttps://example.com/2 - Threehttps://example.com/3 - - """ - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - )) - let spec = NewsReaderSpec( - name: "Cap", - topics: [], - sources: [NewsSource(label: "Ex", url: "https://example.com/rss.xml")], - maxCount: 2 - ) - let items = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - #expect(items.count == 2) - #expect(items.allSatisfy { !$0.sourceURL.isEmpty }) - } - - @Test func googleNewsHomepageMapsToRSS() { - let mapped = NewsSourceURL.canonicalFetchURL(URL(string: "https://news.google.com/")!) - #expect(mapped.host == "news.google.com") - #expect(mapped.path == "/rss") - let already = NewsSourceURL.canonicalFetchURL( - URL(string: "https://news.google.com/rss?hl=en-US")! - ) - #expect(already.path.contains("rss")) - } - - @Test func refreshParsesGoogleNewsHomepageViaRSS() async throws { - let rss = """ - - World headlinehttps://news.google.com/articles/abcSummary bit - - """ - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - )) - let spec = NewsReaderSpec( - name: "Google", - topics: [], - sources: [NewsSource(label: "Google News", url: "https://news.google.com")], - mode: .summaries, - maxCount: 10 - ) - let items = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - #expect(items.count == 1) - #expect(items[0].title == "World headline") - #expect(items[0].sourceURL.contains("news.google.com")) - #expect(NewsReaderRefresh.digest(from: items).contains("World headline")) - } - - @Test func liveGoogleNewsRSSHasLinkedSummaries() async throws { - let spec = NewsReaderSpec( - name: "Google live", - topics: [], - sources: [NewsPresetSource.googleNews.source], - mode: .summaries, - maxCount: 8 - ) - let items = try await NewsReaderRefresh.validateAndFetch( - spec: spec, - client: URLSessionNewsHTTPClient() - ) - #expect(!items.isEmpty) - #expect(items.allSatisfy { !$0.sourceURL.isEmpty && !$0.title.isEmpty }) - #expect(!NewsReaderRefresh.digest(from: items).isEmpty) - } - - private struct StubNewsClient: NewsHTTPClient { - let response: NewsHTTPResponse - func get(url: URL) async throws -> NewsHTTPResponse { response } - } -} diff --git a/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift new file mode 100644 index 00000000..a2bb3eb3 --- /dev/null +++ b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift @@ -0,0 +1,84 @@ +import Foundation +import Testing +@testable import Structure + +@Suite struct PluginSkillDraftTests { + @Test func newsGoalIsCustomCapabilityNotACoreProduct() { + var draft = PluginSkillDraft(goal: "Fetch tech news headlines daily") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .customCapability) + } + + @Test func infersSlackConnectorFromGoal() { + var draft = PluginSkillDraft(goal: "Send messages in Slack from Messaging") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .messagingConnector) + #expect(draft.inferredConnectorVendor == .slack) + #expect(draft.examples.count >= 1) + } + + @Test func infersCustomCapabilityFromGenericGoal() { + var draft = PluginSkillDraft(goal: "Summarize my clipboard when I ask") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .customCapability) + } + + @Test func skillMarkdownIncludesPurposeAndExamples() { + var draft = PluginSkillDraft( + goal: "Do something", + purpose: "Help with tasks", + triggers: [.chat], + examples: [ + PluginSkillDraft.Example(userSays: "run it", pluginDoes: "returns a result"), + ], + pluginName: "my-plugin" + ) + let markdown = draft.skillMarkdown() + #expect(markdown.contains("my-plugin")) + #expect(markdown.contains("Help with tasks")) + #expect(markdown.contains("run it")) + } + + @Test func makeFromSkillDraftBuildsConnectorInput() throws { + var draft = PluginSkillDraft( + goal: "Slack connector", + purpose: "Messaging", + examples: [ + PluginSkillDraft.Example(userSays: "hi", pluginDoes: "send"), + ], + pluginName: "slack-connector-1" + ) + let input = try PluginFactoryCreateInput.makeFromSkillDraft(draft) + #expect(input.pluginType == .connector) + #expect(input.vendor == .slack) + #expect(input.pluginID == "slack-connector-1") + #expect(input.skillMarkdown != nil) + } + + @Test func makeFromSkillDraftBuildsCustomInput() throws { + var draft = PluginSkillDraft( + goal: "Summarize text", + purpose: "Summarize", + examples: [ + PluginSkillDraft.Example(userSays: "summarize", pluginDoes: "returns summary"), + ], + pluginName: "summarizer" + ) + let input = try PluginFactoryCreateInput.makeFromSkillDraft(draft) + #expect(input.pluginType == .custom) + #expect(input.vendor == nil) + #expect(input.pluginID == "summarizer") + } + + @Test func messagingConnectorDisallowsScheduleTrigger() { + let allowed = PluginSkillDraftPlanner.availableTriggers(for: .messagingConnector) + #expect(allowed == Set([.chat, .messaging, .mention])) + var draft = PluginSkillDraft( + goal: "Slack connector", + triggers: [.schedule, .chat, .messaging], + pluginName: "slack-1" + ) + PluginSkillDraftPlanner.sanitizeTriggers(in: &draft) + #expect(draft.triggers == Set([.chat, .messaging])) + } +} diff --git a/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift b/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift new file mode 100644 index 00000000..c71db5c2 --- /dev/null +++ b/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift @@ -0,0 +1,50 @@ +import Foundation +import Structure +import Testing + +@Suite struct ScriptExecContractTests { + @Test func protocolJSONLoadsAndMatchesRules() throws { + let document = try ScriptExecContractStore.loadProtocol() + #expect(document.version == 1) + #expect(document.runtime.language == "go") + #expect(document.review.failFast) + #expect(document.review.checks.count == 4) + #expect(document.rules.guestHasNoNetwork) + #expect(document.output.fields["content"]?.purpose.contains("plain-text") == true) + #expect(document.pluginFactory.manifest.hostCreatesManifest) + #expect(document.pluginFactory.builder.connectorTestInputUsesHopsArray) + #expect(document.pluginFactory.review.compilationSuccessNotApproval) + } + + @Test func pluginFactoryPromptsReferenceContractJSON() { + let builder = ScriptExecContractPrompts.pluginFactoryBuilderGuide() + let reviewer = ScriptExecContractPrompts.pluginFactoryReviewerGuide() + #expect(builder.contains("plugin_factory")) + #expect(builder.contains("--- script-exec-contract.json ---")) + #expect(reviewer.contains("plugin_factory.review")) + #expect(reviewer.contains("If a guest rule is not in the JSON")) + } + + @Test func fingerprintMatchesGeneratedFile() throws { + #expect(try ScriptExecContractStore.computeFingerprint() == ScriptExecContractFingerprint.sha256) + #expect(ScriptExecContractFingerprint.sourceFiles == ScriptExecContractStore.fingerprintSources) + } + + @Test func bundledContractSatisfiesItsSchema() throws { + try ScriptExecContractIntegrity.validateBundledGraph() + } + + @Test func reviewerGuideDumpsCanonicalJSON() throws { + let guide = ScriptExecContractPrompts.reviewerGuide() + #expect(guide.contains("If a rule is not in the JSON")) + #expect(guide.contains("--- script-exec-contract.json ---")) + #expect(guide.contains(try ScriptExecContractStore.loadProtocolText())) + #expect(guide.contains("--- \(GuestContract.Schema.envelopeList.rawValue) ---")) + } + + @Test func builderGuideDumpsWireSchemas() throws { + let guide = ScriptExecContractPrompts.builderGuide() + #expect(guide.contains("--- \(GuestContract.Schema.hopEvent.rawValue) ---")) + #expect(guide.contains("--- \(GuestContract.Schema.guestRuntime.rawValue) ---")) + } +} diff --git a/readme.md b/readme.md index e02a8bc4..b8b12621 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ A native Swift macOS 27 desktop agent harness: chat with LLM providers, run isol |------|-------------| | **Chat** | Multi-tab conversations with OpenAI, Gemini, and other configured models | | **Tools (MCP)** | Model Context Protocol tool host inside the headless daemon | -| **Scripts** | Agent-generated Python executed in isolated Docker containers. Includes a secondary agent code reviewer and approvals flow. | +| **Scripts** | Agent-generated Go executed in isolated Docker containers. Includes a secondary agent code reviewer and approvals flow. | | **Plugin factory** | LLM-assisted creation of versioned, reviewed connector plugins | | **Jobs** | Scheduled and deferred tool/agent runs that survive app quit | | **Messaging** | Connector plugins (e.g. Slack) with threads, history, and live sync | @@ -35,14 +35,14 @@ A native Swift macOS 27 desktop agent harness: chat with LLM providers, run isol └────────────────┘ └───────────┬────────────┘ │ ┌───────────▼────────────┐ - │ Python guest containers │ + │ Go worker containers │ │ --network none │ └────────────────────────┘ ``` - **UI** is a client: it does not own agent turns or MCP when the daemon is up. - **Daemon** (`derrickd`) is the single owner of OS notifications and in-process Agent/Job/MCP modules. -- **Docker** runs untrusted Python for `script_exec`, plugin factory builds, and approved plugin invocations. +- **Docker** runs untrusted Go for `script_exec`, plugin factory builds, and approved plugin invocations. See [docs/adr-headless-backend.md](docs/adr-headless-backend.md) and [docs/services-plan.md](docs/services-plan.md). @@ -50,10 +50,12 @@ See [docs/adr-headless-backend.md](docs/adr-headless-backend.md) and [docs/servi Derrick treats model output and guest code as untrusted. -### Docker sandbox (Python guest runtime) +### Docker sandbox (Go worker runtime) -- Guest programs run in `python:3.14.7` containers with **`--network none`**. -- No sockets, urllib/requests, subprocess, or credentials inside the guest. +- **`script_exec`, plugins, web crawl, and file extract** share one Go worker image `derrick-worker:go-v1` (digest-pinned, `--network none` for guests). +- Plugin and script sources are compiled **inside the worker container** (Go 1.27.1 in the image). Users only need Docker Desktop. +- Canonical I/O types live in `packages/Structure/Sources/Contract/Resources/schemas/` and are mirrored to `workers/go/internal/contract/schemas/`. +- No net/http, subprocess, filesystem access, or credentials inside the guest. - The host dispatches `http.request` envelopes, attaches secrets, and enforces egress policy. - Historical Swift guest notes: [docs/adr-swift-script-runtime.md](docs/adr-swift-script-runtime.md). @@ -65,7 +67,7 @@ Before `script_exec` writes to disk, a **configured LLM reviewer** checks: - No secret literals in source - Safe handling of fetched content (no raw HTML leakage unless requested) -Instructions live in `ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md`. A static **Python verifier** also rejects forbidden APIs. +Rules live in `packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json` (same pattern as connector plugins). A static **Go verifier** also rejects forbidden APIs. ### Egress & network diff --git a/scripts/generate-script-exec-contract.swift b/scripts/generate-script-exec-contract.swift new file mode 100755 index 00000000..4c1995ce --- /dev/null +++ b/scripts/generate-script-exec-contract.swift @@ -0,0 +1,121 @@ +#!/usr/bin/env swift +import CryptoKit +import Foundation + +/// Regenerates `ScriptExecContract.generated.swift` from bundled script_exec JSON. +/// `--check` exits 1 when the committed fingerprint does not match the JSON files +/// or when the JSON no longer satisfies the bundled schemas. + +let repoRoot = URL(fileURLWithPath: CommandLine.arguments[0]) + .resolvingSymlinksInPath() + .deletingLastPathComponent() + .deletingLastPathComponent() + +let resources = repoRoot + .appendingPathComponent("packages/Structure/Sources/Contract/Resources") + +let sources: [(relative: String, url: URL)] = [ + "schemas/script-exec-contract.schema.json", + "schemas/guest-runtime.schema.json", + "schemas/hop-event.schema.json", + "schemas/envelope-list.schema.json", + "contracts/script-exec-contract.json", +].map { relative in + (relative, resources.appendingPathComponent(relative)) +} + +let generatedURL = repoRoot + .appendingPathComponent("packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift") + +func jsonObject(_ url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + guard let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ScriptError("\(url.lastPathComponent) is not a JSON object.") + } + return object +} + +func requireKeys(_ object: [String: Any], _ keys: [String], file: String) throws { + for key in keys where object[key] == nil { + throw ScriptError("\(file) is missing required key \(key).") + } +} + +func validateGraph() throws { + let contractSchema = try jsonObject(resources.appendingPathComponent("schemas/script-exec-contract.schema.json")) + let contract = try jsonObject(resources.appendingPathComponent("contracts/script-exec-contract.json")) + try requireKeys(contract, contractSchema["required"] as? [String] ?? [], file: "script-exec-contract.json") + let rulesSchema = (contractSchema["properties"] as? [String: Any])?["rules"] as? [String: Any] + try requireKeys( + contract["rules"] as? [String: Any] ?? [:], + rulesSchema?["required"] as? [String] ?? [], + file: "script-exec-contract.json rules" + ) + let checks = (contract["review"] as? [String: Any])?["checks"] as? [[String: Any]] ?? [] + if checks.isEmpty { + throw ScriptError("script-exec-contract.json review.checks must not be empty.") + } + let pluginFactorySchema = (contractSchema["properties"] as? [String: Any])?["plugin_factory"] as? [String: Any] + try requireKeys( + contract["plugin_factory"] as? [String: Any] ?? [:], + pluginFactorySchema?["required"] as? [String] ?? [], + file: "script-exec-contract.json plugin_factory" + ) +} + +func fingerprint() throws -> String { + var joined = Data() + for source in sources { + let data = try Data(contentsOf: source.url) + joined.append(Data(source.relative.utf8)) + joined.append(0) + joined.append(data) + joined.append(0) + } + return SHA256.hash(data: joined).map { String(format: "%02x", $0) }.joined() +} + +func generatedSource(hash: String) -> String { + """ + // Automatically generated by scripts/generate-script-exec-contract.swift. DO NOT EDIT. + + /// SHA-256 of script_exec protocol JSON and schemas. `swift test` fails when this is stale. + public enum ScriptExecContractFingerprint: Sendable { + public static let sha256 = "\(hash)" + public static let sourceFiles: [String] = [ + \(sources.map { " \"\($0.relative)\"," }.joined(separator: "\n")) + ] + } + + """ +} + +struct ScriptError: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +do { + try validateGraph() +} catch { + fputs("\(error)\n", stderr) + exit(1) +} + +let hash = try fingerprint() +let check = CommandLine.arguments.contains("--check") +if check { + let existing = try String(contentsOf: generatedURL, encoding: .utf8) + if !existing.contains("public static let sha256 = \"\(hash)\"") { + fputs("script_exec contract fingerprint is stale. Run scripts/generate-script-exec-contract.swift\n", stderr) + exit(1) + } + exit(0) +} + +try FileManager.default.createDirectory( + at: generatedURL.deletingLastPathComponent(), + withIntermediateDirectories: true +) +try generatedSource(hash: hash).write(to: generatedURL, atomically: true, encoding: .utf8) +print("Wrote \(generatedURL.path) sha256=\(hash)") diff --git a/scripts/record-docker-image-digests.sh b/scripts/record-docker-image-digests.sh new file mode 100755 index 00000000..4642aa41 --- /dev/null +++ b/scripts/record-docker-image-digests.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Builds the unified Go worker image and updates the pinned digest in Structure. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +IMAGE="derrick-worker:go-v1" +docker build -f docker/worker/Dockerfile -t "$IMAGE" . + +DIGEST="$(docker image inspect --format '{{.Id}}' "$IMAGE")" +OUT="packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift" + +cat >"$OUT" < Quit Derrick before resetting local state." +echo "==> This removes all local SQLite data (chats, plugins, messaging, credentials in DB)." +echo "==> Keychain plugin secrets are not removed." + +removed_dbs=0 +if [[ -d "$APP_GROUP" ]]; then + while IFS= read -r db; do + rm -f "$db" "${db}-wal" "${db}-shm" + echo "removed $(basename "$db") at ${db%/*}" + removed_dbs=$((removed_dbs + 1)) + done < <(find "$APP_GROUP" -name 'derrick.sqlite3' 2>/dev/null) +fi + +if [[ "$removed_dbs" -eq 0 ]]; then + echo "no derrick.sqlite3 files found under $APP_GROUP" +fi + +if command -v docker >/dev/null 2>&1; then + echo "==> Removing Derrick runtime containers" + for filter in 'name=derrick-guest-runtime' 'name=derrick-swift-runtime' 'label=app.derrick=runtime'; do + ids="$(docker ps -aq --filter "$filter" || true)" + if [[ -n "$ids" ]]; then + docker rm -f $ids >/dev/null 2>&1 || true + fi + done + + echo "==> Removing obsolete guest images (Python / legacy guest-runtime)" + for image in \ + 'derrick-guest-runtime:python-v1' \ + 'python:3.14.7'; do + if docker image inspect "$image" >/dev/null 2>&1; then + docker rmi -f "$image" >/dev/null + echo "removed image $image" + fi + done +else + echo "docker not available — skipped container/image cleanup" +fi + +echo +echo "Done. Reopen Derrick from $ROOT (go-workers) to recreate an empty database." +echo "Policy rules seed automatically on first UI launch." diff --git a/scripts/sync-guest-contract-schemas.sh b/scripts/sync-guest-contract-schemas.sh new file mode 100755 index 00000000..daa1d819 --- /dev/null +++ b/scripts/sync-guest-contract-schemas.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Copies canonical guest and worker-product JSON schemas from Structure into the Go worker module. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SRC="$ROOT/packages/Structure/Sources/Contract/Resources/schemas" +DST="$ROOT/workers/go/internal/contract/schemas" + +mkdir -p "$DST" +rsync -a --delete "$SRC/" "$DST/" +echo "Synced guest contract schemas to $DST" diff --git a/scripts/verify-guest-contract-schemas-in-sync.sh b/scripts/verify-guest-contract-schemas-in-sync.sh new file mode 100755 index 00000000..0dbce088 --- /dev/null +++ b/scripts/verify-guest-contract-schemas-in-sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Fails CI when canonical Structure schemas (guest + worker product) drift from the Go worker copy. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SRC="$ROOT/packages/Structure/Sources/Contract/Resources/schemas" +DST="$ROOT/workers/go/internal/contract/schemas" + +if ! diff -qr "$SRC" "$DST" >/dev/null; then + echo "Guest contract schemas are out of sync. Run: scripts/sync-guest-contract-schemas.sh" >&2 + diff -qr "$SRC" "$DST" >&2 || true + exit 1 +fi + +echo "Guest contract schemas are in sync." diff --git a/ui/AgentService/AgentServiceExportedObject.swift b/ui/AgentService/AgentServiceExportedObject.swift index 456bfa46..6336a224 100644 --- a/ui/AgentService/AgentServiceExportedObject.swift +++ b/ui/AgentService/AgentServiceExportedObject.swift @@ -67,7 +67,7 @@ final class AgentServiceExportedObject: NSObject, AgentServiceXPC { service: .agent, status: .ok, detail: "AgentService ready (DB+\(leaf))", - guestRuntimeImage: DerrickProcessRole.isDaemon ? DerrickGuestRuntime.pythonGuestDockerImage : nil + guestRuntimeImage: DerrickProcessRole.isDaemon ? DerrickGuestRuntime.guestDockerImage : nil ) let data = (try? AgentServiceXPCCodec.encodeHealth(report)) ?? Data("{}".utf8) reply(data as NSData) diff --git a/ui/AgentService/AgentServiceTurnHost.swift b/ui/AgentService/AgentServiceTurnHost.swift index 84e9efbc..e4578ea0 100644 --- a/ui/AgentService/AgentServiceTurnHost.swift +++ b/ui/AgentService/AgentServiceTurnHost.swift @@ -223,7 +223,6 @@ actor AgentServiceTurnHost { LLMModelSettings(repository: repo) } helperModelSettings = settings - await EgressAllowlistService.shared.configure(repository: repo) await ContentSensitivityGrantService.shared.configure(repository: repo) await UsageLimitsService.shared.configure(repository: repo) await ContainerLifecycleSettingsService.shared.configure(repository: repo) diff --git a/ui/JobKeepAlive/DaemonModuleBootstrap.swift b/ui/JobKeepAlive/DaemonModuleBootstrap.swift index dfae72f2..85956545 100644 --- a/ui/JobKeepAlive/DaemonModuleBootstrap.swift +++ b/ui/JobKeepAlive/DaemonModuleBootstrap.swift @@ -76,9 +76,6 @@ enum DaemonModuleBootstrap { Task { await prewarmGuestRuntimeImage() } - Task { - await startWebCrawlerImageInBackground() - } } catch { fputs("[derrickd] MCP module bootstrap failed: \(error.localizedDescription)\n", stderr) } @@ -102,7 +99,7 @@ enum DaemonModuleBootstrap { } /// Remove leftover runtime containers before the job scheduler starts. - /// Guest image pull stays in the background so a cold Python pull does not block jobs. + /// Guest image pull stays in the background so a cold worker image pull does not block jobs. private static func sweepEmbeddedDockerLeftovers() async { guard DerrickProcessRole.isDaemon else { return } do { @@ -127,14 +124,4 @@ enum DaemonModuleBootstrap { } } - /// Does not block jobs or chat. A crawl that arrives during this build waits on the same task. - private static func startWebCrawlerImageInBackground() async { - guard DerrickProcessRole.isDaemon else { return } - do { - try await MCPServiceDockerHelperRunner.shared.ensureWebCrawlerImage() - fputs("[derrickd] web crawler image ready\n", stderr) - } catch { - fputs("[derrickd] web crawler image background build skipped: \(error.localizedDescription)\n", stderr) - } - } } diff --git a/ui/JobService/MessagingAgentTurnClient.swift b/ui/JobService/MessagingAgentTurnClient.swift index 461a1e49..12e260b7 100644 --- a/ui/JobService/MessagingAgentTurnClient.swift +++ b/ui/JobService/MessagingAgentTurnClient.swift @@ -11,7 +11,7 @@ enum MessagingAgentTurnClient { let profile = try await resolveProfile(handle: route.profileHandle, repository: repository) let model = (try? JSONDecoder().decode(LLMModelChoice.self, from: profile.modelJSON)) ?? .defaultHelperModel - let apiKey = await resolveAPIKey(for: model) ?? "" + let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) ?? "" let profileContextJSON = try JSONEncoder().encode(AgentProfileTurnContext(profile: profile)) let sessionID = MessagingAgentSessionID.make( pluginID: route.pluginID, @@ -60,14 +60,6 @@ enum MessagingAgentTurnClient { throw MessagingAgentTurnClientError.profileUnavailable(handle) } - @MainActor - private static func resolveAPIKey(for model: LLMModelChoice) -> String? { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - } - private static func sendConnectorMessage( route: MessagingAgentRoute, text: String, diff --git a/ui/MCPService/MCPServiceDockerHelperRunner.swift b/ui/MCPService/MCPServiceDockerHelperRunner.swift index 5ed26f9c..b623facf 100644 --- a/ui/MCPService/MCPServiceDockerHelperRunner.swift +++ b/ui/MCPService/MCPServiceDockerHelperRunner.swift @@ -63,17 +63,14 @@ final class MCPServiceDockerHelperRunner: @unchecked Sendable { await DerrickDockerOrphanSweeper.sweep(executor: makeStdinCLIExecutor()) } - /// Prewarm the shared offline guest runtime image. + /// Prewarm the shared Go worker image used by script_exec and plugin.invoke. func prewarmGuestRuntime() async throws { - try await OneshotDockerContainer.ensurePulledImage( - DerrickGuestRuntime.pythonGuestDockerImage, - executor: makeStdinCLIExecutor() - ) + try await WorkerImageGate.shared.ensureReady(executor: makeStdinCLIExecutor()) } - /// Build the crawler image in the background. Joins an in-flight build if one exists. + /// Legacy alias — same worker image as `prewarmGuestRuntime`. func ensureWebCrawlerImage() async throws { - try await WebCrawlerImageGate.shared.ensureReady(executor: makeStdinCLIExecutor()) + try await prewarmGuestRuntime() } var hasPeerEndpoint: Bool { diff --git a/ui/MCPService/MCPServiceScriptReviewer.swift b/ui/MCPService/MCPServiceScriptReviewer.swift index 9a1e4986..7da7ba47 100644 --- a/ui/MCPService/MCPServiceScriptReviewer.swift +++ b/ui/MCPService/MCPServiceScriptReviewer.swift @@ -20,9 +20,8 @@ struct MCPServiceScriptReviewer: ScriptReviewer { } func review(_ args: ScriptExecutionArguments) async throws -> ScriptReviewOutcome { - guard let apiKey = MCPServiceCallContext.shared.helperAPIKey, - !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - else { + let selected = resolveSelectedModel() + guard let apiKey = await resolveAPIKey(for: selected) else { throw NSError( domain: "MCPService", code: 404, @@ -30,7 +29,6 @@ struct MCPServiceScriptReviewer: ScriptReviewer { ) } - let selected = resolveSelectedModel() do { return try await review(args, model: selected, apiKey: apiKey) } catch { @@ -38,7 +36,7 @@ struct MCPServiceScriptReviewer: ScriptReviewer { "[MCPService] reviewer model \(selected.label) failed: \(error.localizedDescription); trying defaults\n", stderr ) - if let fallback = await fallbackReview(args: args, apiKey: apiKey, excluding: selected) { + if let fallback = await fallbackReview(args: args, excluding: selected) { return fallback } throw error @@ -108,12 +106,12 @@ struct MCPServiceScriptReviewer: ScriptReviewer { private func fallbackReview( args: ScriptExecutionArguments, - apiKey: String, excluding: ReviewerModel ) async -> ScriptReviewOutcome? { var candidates: [ReviewerModel] = [Self.defaultModel, Self.secondaryDefault] candidates.removeAll { $0 == excluding } for candidate in candidates { + guard let apiKey = await resolveAPIKey(for: candidate) else { continue } do { let outcome = try await review(args, model: candidate, apiKey: apiKey) fputs("[MCPService] fallback reviewer succeeded model=\(candidate.label)\n", stderr) @@ -127,6 +125,19 @@ struct MCPServiceScriptReviewer: ScriptReviewer { } return nil } + + private func resolveAPIKey(for model: ReviewerModel) async -> String? { + await LLMProviderCredentialGate.resolveAPIKey(for: llmModelChoice(from: model)) + } + + private func llmModelChoice(from model: ReviewerModel) -> LLMModelChoice { + switch model { + case .openai(let openAIModel): + return .openai(openAIModel) + case .gemini(let geminiModel): + return .gemini(geminiModel) + } + } } /// Process-wide slots for the current MCPService tool call (handlers may not inherit TaskLocal). diff --git a/ui/MCPService/MCPServiceToolHost.swift b/ui/MCPService/MCPServiceToolHost.swift index b44967f9..d3aa0aff 100644 --- a/ui/MCPService/MCPServiceToolHost.swift +++ b/ui/MCPService/MCPServiceToolHost.swift @@ -42,7 +42,7 @@ actor MCPServiceToolHost { LLMModelThinkingSettings(repository: repo) } await factoryThinkingSettings.loadSettings() - let factoryExecutor = PythonPluginFactoryDockerExecutor( + let factoryExecutor = GoPluginFactoryDockerExecutor( executor: MCPServiceDockerHelperRunner.shared.makeStdinCLIExecutor() ) let webCrawlerExecutor = WebCrawlerDockerExecutor( @@ -67,10 +67,6 @@ actor MCPServiceToolHost { await WorkflowProgressPublisher.publish(stage: "factory", message: progress) } }, - apiKeyProvider: { - MCPServiceCallContext.shared.helperAPIKey - ?? TurnProcessContext.effectiveAPIKey - } ) let made = try await MCPLocalBridge.make { server in await server.registerScriptExecutionTool( @@ -233,32 +229,6 @@ actor MCPServiceToolHost { message: "Tool \(toolName) is owned by AgentService, not MCPService." ) } - if toolName == AllowedMCPTool.webCrawl.rawValue, - !EffectorAdmissionPolicy.allowsSyncWebCrawl( - context: EffectorAdmissionPolicy.parseContextJSON(request.executionContextJSON) - ?? legacyExecutionContext(from: request), - principal: request.principal - ) { - let outcome = ToolExecutionOutcome.failure( - status: .blocked, - stage: .validation, - diagnostics: [ - ToolExecutionOutcome.Diagnostic( - code: "web_crawl_requires_notification", - message: "web.crawl must be submitted through jobs_create so the result can arrive in a notification banner." - ) - ], - retry: ToolExecutionOutcome.Retry(allowed: false) - ) - return MCPToolCallResultDTO( - requestID: request.requestID, - ok: true, - isError: true, - text: (try? outcome.encodedJSON()) ?? "", - message: "Submit web.crawl through jobs_create for notification delivery." - ) - } - let sessionKey: MemorySessionKey switch request.principal { case .agent(let sessionID, let agentID): diff --git a/ui/SharedAgentRuntime/Conversation/ConversationModel.swift b/ui/SharedAgentRuntime/Conversation/ConversationModel.swift index 95e02e85..e2ed82c1 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationModel.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationModel.swift @@ -266,7 +266,7 @@ final class ConversationModel { let interceptor = makeContentPolicyInterceptor() let orchestrator = self.orchestrator let workerModel = helperModelSettings.workerAgentModel - let workerApiKey = resolveAPIKey(for: workerModel, turnFallback: apiKey) ?? apiKey + let workerApiKey = await LLMProviderCredentialGate.resolveAPIKey(for: workerModel) ?? apiKey let effectiveModel: LLMModelChoice let effectiveThinking: ModelThinkingOption? @@ -593,21 +593,6 @@ final class ConversationModel { return DefaultPolicyInterceptor(policy: policy) } - /// API key for a helper/worker model: keychain/env for its provider, else the active turn key. - private func resolveAPIKey(for model: LLMModelChoice, turnFallback: String) -> String? { - if let key = AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ), !key.isEmpty { - return key - } - if let turnKey = TurnProcessContext.effectiveAPIKey, !turnKey.isEmpty { - return turnKey - } - let trimmed = turnFallback.trimmingCharacters(in: .whitespacesAndNewlines) - return trimmed.isEmpty ? nil : trimmed - } - /// In-process host for orchestration tools (`agents_*`, `jobs_*`). Not used for MCP effectors. /// `nonisolated`: tool handlers must not hop to MainActor while the turn awaits the MCP local bridge /// (that pattern deadlocks when runTurn is MainActor-isolated). @@ -727,13 +712,14 @@ final class ConversationModel { wakePrompt: wakePrompt, description: description ) + let providerAPIKey = await LLMProviderCredentialGate.resolveAPIKey(for: .defaultHelperModel) let request = try JobOrderBuilder.createJobRequest( from: input, principal: principal, source: .agent, sessionID: sessionID, agentID: agentID, - helperAPIKey: TurnProcessContext.effectiveAPIKey, + helperAPIKey: providerAPIKey, helperReviewerModelJSON: reviewerJSON ) debugLog("[jobs_create] calling JobService createJob…") @@ -787,13 +773,14 @@ final class ConversationModel { wakePrompt: wakePrompt, enabled: true ) + let providerAPIKey = await LLMProviderCredentialGate.resolveAPIKey(for: .defaultHelperModel) let request = try JobOrderBuilder.createScheduleRequest( from: input, principal: principal, source: .agent, sessionID: sessionID, agentID: agentID, - helperAPIKey: TurnProcessContext.effectiveAPIKey, + helperAPIKey: providerAPIKey, helperReviewerModelJSON: reviewerJSON ) let schedule = try await placer.createSchedule(request) diff --git a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift index edd1d542..8a658c8e 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift @@ -122,11 +122,6 @@ struct ConversationPipeline: Sen if !toolInstructions.isEmpty { sections.append(toolInstructions) } - if !toolInstructions.contains("Python guest contract:") { - if let sdk = try? PromptResources.guestSDKForModel() { - sections.append(sdk) - } - } return sections.joined(separator: "\n\n") } diff --git a/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift b/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift index 8c8c64b3..406d88bf 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift @@ -877,7 +877,7 @@ extension ConversationPipeline { private static func longRunningToolProgressMessage(for toolName: String) -> String? { switch toolName { case AllowedMCPTool.webCrawl.rawValue: - return "Crawling vendor documentation. This may take a few minutes…" + return "Fetching web content. This may take a few minutes…" case AllowedMCPTool.pluginFactoryBuild.rawValue: return "Building the plugin (code generation, Docker tests, and safety review). This may take several minutes…" default: diff --git a/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift b/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift index 3a9f8666..38f040b8 100644 --- a/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift +++ b/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift @@ -40,7 +40,7 @@ enum ProfileDelegateRunner { let thinking = profileContext.thinkingJSON.flatMap { try? JSONDecoder().decode(ModelThinkingOption.self, from: $0) } - let apiKey = TurnProcessContext.effectiveAPIKey ?? "" + let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) ?? "" let delegateSessionKey = MemorySessionKey( sessionID: sessionKey.sessionID, diff --git a/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift b/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift index 540ea804..04547a72 100644 --- a/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift +++ b/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift @@ -5,9 +5,8 @@ import Plugin import PolicyUserInteraction import Structure -/// Before a scheduled network tool runs, ensure hosts are allowlisted. -/// Uncovered hosts use the HITL **banner** path (not live chat modals / schedule preflight). -/// +/// Before a scheduled network tool runs, prompt only on blacklist hits. +/// Public HTTPS is allowed by default; hard-blocked SSRF targets are denied without a prompt. public enum JobNetworkPreflight { public static func approveScriptNetworkIfNeeded( toolName: String, @@ -30,6 +29,13 @@ public enum JobNetworkPreflight { } guard !hosts.isEmpty else { return } + let hardBlockPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) + for host in hosts { + if hardBlockPolicy.isHardBlockedHostname(host) { + throw JobNetworkPreflightError.hardBlocked(host: host) + } + } + let blacklist = try await repository.listEgressBlacklist() let exceptions = try await repository.listEgressBlacklistExceptions() for host in hosts { @@ -40,13 +46,15 @@ public enum JobNetworkPreflight { ) else { continue } + let argumentsJSON = blacklistArgumentsJSON(host: host, entry: entry, toolName: toolName) let decision = await HITLOfflineNetworkService.awaitDecision( host: host, toolName: toolName, turnID: "job-\(jobID)", isJobContext: true, repository: repository, - timeoutNanoseconds: 300_000_000_000 + timeoutNanoseconds: 300_000_000_000, + argumentsJSON: argumentsJSON ) switch decision { case .approved, .approvedOnce: @@ -62,83 +70,28 @@ public enum JobNetworkPreflight { } } - let suffixes = try await loadEnabledSuffixes(repository: repository) - - let policy = DefaultDestinationPolicy(allowedDomainSuffixes: suffixes) - var uncovered: [String] = [] - for host in hosts { - if policy.isHardBlockedHostname(host) { - throw JobNetworkPreflightError.hardBlocked(host: host) - } - if policy.isHostCoveredByAllowlist(host) { - continue - } - uncovered.append(host) - } - guard !uncovered.isEmpty else { - fputs( - "[JobNetworkPreflight] job=\(jobID) hosts covered count=\(hosts.count)\n", - stderr - ) - return - } - fputs( - "[JobNetworkPreflight] job=\(jobID) banner approval needed hosts=\(uncovered.joined(separator: ","))\n", + "[JobNetworkPreflight] job=\(jobID) ok hosts=\(hosts.count)\n", stderr ) - - var sessionGrants: [String] = [] - var allowedSuffixes = suffixes - for host in uncovered { - let coverage = DefaultDestinationPolicy(allowedDomainSuffixes: allowedSuffixes) - if coverage.isHostCoveredByAllowlist(host) { - continue - } - // Session grants from earlier Allow Once in this preflight (suffix-scoped). - let sessionPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) - sessionPolicy.grantSessionHosts(sessionGrants) - if sessionPolicy.isHostCoveredByAllowlist(host) { - continue - } - let decision = await HITLOfflineNetworkService.awaitDecision( - host: host, - toolName: toolName, - turnID: "job-\(jobID)", - isJobContext: true, - repository: repository, - timeoutNanoseconds: 300_000_000_000 - ) - switch decision { - case .approvedPermanently(let actor): - let suffix = EgressHostExtractor.permanentSuffix(for: host) - try await repository.saveEgressAllowedDomainSuffix( - EgressAllowedDomainSuffix(suffix: suffix, source: actor ?? "job-banner", enabled: true) - ) - allowedSuffixes = try await loadEnabledSuffixes(repository: repository) - fputs( - "[JobNetworkPreflight] always host=\(host) suffix=\(suffix) actor=\(actor ?? "?")\n", - stderr - ) - case .approved(let actor), .approvedOnce(let actor): - sessionGrants.append(host) - fputs( - "[JobNetworkPreflight] once host=\(host) actor=\(actor ?? "?")\n", - stderr - ) - case .denied(let actor): - throw JobNetworkPreflightError.denied(host: host, actor: actor) - case .dismissed: - throw JobNetworkPreflightError.denied(host: host, actor: "system-dismissed") - case .timedOut: - throw JobNetworkPreflightError.denied(host: host, actor: "system-timeout") - } - } - } - private static func loadEnabledSuffixes(repository: DBRepository) async throws -> [String] { - let rows = try await repository.loadEgressAllowedDomainSuffixes(includeDisabled: false) - return rows.filter(\.enabled).map(\.suffix) + private static func blacklistArgumentsJSON( + host: String, + entry: BlacklistEntry, + toolName: String + ) -> String { + let payload: [String: String] = [ + "host": host, + "url": "https://\(host)", + "toolName": toolName, + "kind": "blacklist", + "pattern": entry.displayPattern, + ] + guard let data = try? JSONSerialization.data(withJSONObject: payload), + let json = String(data: data, encoding: .utf8) else { + return #"{"host":"\#(host)","toolName":"\#(toolName)","kind":"blacklist","pattern":"\#(entry.displayPattern)"}"# + } + return json } } diff --git a/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift b/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift index 4dfecd82..58cb31df 100644 --- a/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift +++ b/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift @@ -1,14 +1,40 @@ import Foundation import Structure -/// Whether an LLM provider has a usable API key (Keychain or `.env`, per `AppSecretResolver`). +/// Provider credential checks and API key resolution (Keychain or `.env`, per `AppSecretResolver`). @MainActor enum LLMProviderCredentialGate { - static func hasAPIKey(for provider: LLMProviderChoice, resolver: AppSecretResolver) -> Bool { + /// One key per provider — the single resolution path for chat, reviewers, workers, and jobs. + static func resolveAPIKey( + for provider: LLMProviderChoice, + resolver: AppSecretResolver = AppSecretResolver() + ) -> String? { resolver.resolve( account: provider.secretAccount, environmentKeys: provider.apiKeyEnvironmentKeys - ) != nil + ) + } + + static func resolveAPIKey( + for model: LLMModelChoice, + resolver: AppSecretResolver = AppSecretResolver() + ) -> String? { + resolveAPIKey(for: model.provider, resolver: resolver) + } + + /// Off-main callers (AgentService, MCPService, jobs). + static func resolveAPIKey(for provider: LLMProviderChoice) async -> String? { + await MainActor.run { + resolveAPIKey(for: provider) + } + } + + static func resolveAPIKey(for model: LLMModelChoice) async -> String? { + await resolveAPIKey(for: model.provider) + } + + static func hasAPIKey(for provider: LLMProviderChoice, resolver: AppSecretResolver) -> Bool { + resolveAPIKey(for: provider, resolver: resolver) != nil } static func configuredProviders(resolver: AppSecretResolver) -> [LLMProviderChoice] { diff --git a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md index 1c488a09..7b381d60 100644 --- a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md +++ b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md @@ -13,18 +13,11 @@ Use a tool when the user asks for any of: - search, look up, browse, fetch, scrape, or “from the web / online” - site-specific retail or catalog data (e.g. Amazon, “best sellers”, “top 10 … being sold”, prices, availability) -Use `script_exec` for scripting, automation, and live web access through the host HTTP bridge. - For those requests: 1. Prefer calling the tool **on the first turn** with reasonable defaults. 2. Do **not** answer with only clarifying questions when a sensible default exists (e.g. US site, general category, bestseller or top search results). State the default you used in the final answer after the tool runs. 3. Ask a clarifying question only when the request is impossible to execute without a critical missing fact (not for optional polish). 4. Never invent live rankings, prices, stock, market moves, or “what’s selling now” from training data. -5. For current events / market turmoil: fetch real articles from news or finance sites (not Google search results pages). If a scrape returns no usable content, retry with other sites before concluding data is unavailable. - -## Response format - -Always respond using the required JSON schema (`thinking` / `tool_call` / `tool_batch` / `complete`). Never reply as free-form plain text outside that schema. When presenting a list of choices, options, steps, items, or alternative paths to the user, ALWAYS format them as a clean Markdown bulleted list (using `-` or `*`) or a numbered list (using `1.`, `2.`), instead of writing them as plain paragraphs. diff --git a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md index 8e015391..11753240 100644 --- a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md +++ b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md @@ -7,31 +7,11 @@ - Set `status` to "tool_call" when you need to execute a single tool, and populate the `tool_call` object with your target `tool_name` and a stringified, JSON-formatted string of tool arguments under the `arguments` key. - Set `status` to "tool_batch" when you need to execute multiple tools in parallel, and populate the `tool_batch` object with your list of `invocations`. - Set `status` to "complete" when you have finished and are responding directly to the user, and populate the `assistant_response` field with your Markdown reply. - - Pass tool `arguments` as a **stringified JSON object** under the `arguments` key (schema requirement). Prefer short Python source and avoid embedding unescaped double quotes in the script body. + - Pass tool `arguments` as a **stringified JSON object** under the `arguments` key (schema requirement). Prefer short Go source and avoid embedding unescaped double quotes in the script body. 6. Users should not have to name tools. Choose tools autonomously from intent. -7. Use `files.extract` for attached PDFs, Office documents, HTML, CSV, and Excel. Use `script_exec` for other scripting/automation. Use `web.crawl` for live website access. - 1. For `script_exec`, use standalone **Python** only. Read one JSON event from standard input and write a JSON **array** of envelopes to standard output. Do not use sockets, urllib/requests, subprocess, shell commands, credentials, or package dependencies. - 2. The container has no network. Emit `http.request` envelopes; the host performs HTTP and invokes the guest program again with an `http_results` event. - 3. On the first hop emit `{"verb":"http.request","request_id":"…","method":"GET","url":"…"}`. On `http_results`, parse the supplied UTF-8 body and emit `result.emit` or `message.post`. - 4. The script must complete the user's requested extraction or summary, not only prove that a fetch happened. Never emit `repr(http_results)` or copy an entire fetched body into `content` unless the user explicitly requested the raw source. For HTML/XML, remove scripts and styles, extract the relevant visible fields, normalize the text, and cap the result. If raw HTML is explicitly requested, emit it in `html`; the host sanitizes that field. - 5. Prefer content sites. Do **not** scrape Google/Bing/Yahoo SERP HTML. - 6. Keep scripts short. Use `timeout_seconds` on the tool args if needed. - 7. If the first fetch is empty, try another `script_exec` with different URLs before answering. - 8. If `script_exec` returns `blocked` or `failed` with implementation findings, treat those - findings as correction feedback and make at most one corrected `script_exec` call before - answering. Do not repeat the same script unchanged. If the reviewer identifies a security - refusal or the corrected call also fails, report the exact finding instead of claiming success. -9. Use `web.crawl` for website crawling instead of generating a crawler script. Because a crawl - can run for a long time, submit it through `jobs_create` with `tool_name` set to `web.crawl`, - `wake_after` set to true, and a short `wake_prompt` that tells the agent to present the crawl - result to the user. The immediate response must say the crawl was submitted and that the - result will arrive in a notification banner. Never request more than 900 seconds. -10. A web crawl goal must describe the requested result. Never use it for DDoS, flooding, - load/stress testing, port scanning, brute force, or other high-volume behavior. Keep the - crawl same-origin and rely on the tool's page, depth, byte, rate, and timeout limits. -11. Use `files.extract` for attached files instead of generating an extractor script. Call it directly; do not submit it through `jobs_create`. Omit `filenames` to process every attached file in this chat. Never request more than 180 seconds. -12. After tool execution, respond with clean user-facing output only (Markdown/JSON/CSV as requested); do not include raw tool-call JSON, escaped script source, or internal control payloads. -13. Multi-agent tools (when listed in the catalog): +7. Use `files.extract` for attached files, `script_exec` for other scripting/automation, and `web.crawl` for live website access. Obey the bundled guest SDK (`script-exec-contract.json`) for `script_exec`; obey the web crawler and file extractor skills for those tools. +8. After tool execution, respond with clean user-facing output only (Markdown/JSON/CSV as requested); do not include raw tool-call JSON, escaped script source, or internal control payloads. +9. Multi-agent tools (when listed in the catalog): 1. If the user names a multi-agent tool or asks to spawn/list/send/cancel agents, issue that `tool_call` (or `tool_batch`) **before** any `complete` answer. Do not invent tool results. 2. `agents_spawn` — required args: `goal` (short), `task` (concrete). Blocks until the worker finishes; use the returned `result` in your next step. Optional `agent_id` slug. 3. Workers never talk to the user; you synthesize worker results into the final `assistant_response`. diff --git a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md b/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md deleted file mode 100644 index 1c7b9c8e..00000000 --- a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md +++ /dev/null @@ -1,37 +0,0 @@ -You are a reviewer for script_exec declarations. - -FAIL-FAST (mandatory): -- Apply checks in order. As soon as ANY single check fails, return with failure JSON immediately. -- Do NOT continue scanning for more issues after the first failure. -- On first failure: return suggestedAction "deny", alignedWithRequest false, concerns with only that one failing reason, and a short summary. No essays. -- Only if every check passes: return suggestedAction "allow" with a brief summary (1-2 sentences). concerns may be empty or at most one minor operational note. - -Checks (if any fail return failure JSON immediately): -1) Script, description, reason, and user prompt are consistent (intent alignment). - Derrick has a job scheduler (jobs_create / run_after_seconds / cron). Timing is applied by - JobService before this script runs. The script must do the work immediately when invoked. - Words like delayed, scheduled, in 7 seconds, later, or run_after in the reason or user_prompt - refer to that scheduler — not to sleep/setTimeout inside the script. Do not deny a script - that performs the requested work (e.g. netFetch the URL) just because it has no delay. -2) No tokens, API keys, passwords, or other secret literals in the source. - -3) The script implements the requested terminal result, not just the fetch. - For requests to summarize, list, inspect, or extract fetched content, the `http_results` branch - must parse the relevant response body and emit the requested data. Do not allow - `repr(http_results)`, a fetch-only confirmation, or an entire raw body copied to - `content` unless the user explicitly requested the raw source. If raw HTML is requested, `html` - is allowed because the host sanitizes it before rendering. -4) Fetched HTML/XML is treated as untrusted data. Text and Markdown results must remove unsafe - markup and validate generated links as http or https. Do not reject ordinary safe HTML in the - `html` field solely because the host performs the final sanitization. - -Do not deny for Python style, envelope construction, destination URLs, or the absence of dependencies. The static Python verifier enforces direct network and process restrictions. The guest has no network; the host performs HTTP and applies SSRF there. - -Return only valid JSON with this exact schema: -{ - "alignedWithRequest": true|false, - "confidence": 0.0-1.0, - "suggestedAction": "allow"|"deny", - "concerns": ["..."], - "summary": "short explanation" -} diff --git a/ui/SharedAgentRuntime/Resources/web_crawler_skill.md b/ui/SharedAgentRuntime/Resources/web_crawler_skill.md index f9d7be7e..4dc1eb69 100644 --- a/ui/SharedAgentRuntime/Resources/web_crawler_skill.md +++ b/ui/SharedAgentRuntime/Resources/web_crawler_skill.md @@ -3,8 +3,10 @@ Use the `web.crawl` MCP tool for website crawling. Do not generate a crawler script with `script_exec`. -Submit every crawl through `jobs_create` so the user receives the result in a -notification banner. +Call `web.crawl` directly in live chat for typical crawls. Submit through +`jobs_create` only when the crawl is likely to take more than about one minute +(large page budget, deep site, or long timeout). Background crawls should use +`wake_after: true` so the user gets a notification banner when they finish. Required `web.crawl` arguments: diff --git a/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift b/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift index 4b60200c..fd909ce4 100644 --- a/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift +++ b/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift @@ -22,7 +22,7 @@ public enum DaemonProcessHygiene { reportedFingerprint: health.executableFingerprint, expectedFingerprint: expectedFingerprint(), reportedGuestRuntime: health.guestRuntimeImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) } diff --git a/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift b/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift index e02829a9..dd097866 100644 --- a/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift +++ b/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift @@ -9,19 +9,16 @@ import Structure public struct XPCConversationToolClient: ConversationToolClient, Sendable { private let principal: ServicePrincipal private let agentsClient: MCPClient? - private let helperAPIKeyProvider: @Sendable () -> String? /// JSON `HelperModelWire` for MCP script security reviewer (from `LLMModelSettings`). private let helperReviewerModelJSONProvider: @Sendable () async -> String? public init( principal: ServicePrincipal, agentsClient: MCPClient? = nil, - helperAPIKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, helperReviewerModelJSONProvider: @escaping @Sendable () async -> String? = { nil } ) { self.principal = principal self.agentsClient = agentsClient - self.helperAPIKeyProvider = helperAPIKeyProvider self.helperReviewerModelJSONProvider = helperReviewerModelJSONProvider } @@ -85,7 +82,7 @@ public struct XPCConversationToolClient: ConversationToolClient, Sendable { principal: activePrincipal, toolName: name, argumentsJSON: argumentsJSON, - helperAPIKey: helperAPIKeyProvider(), + helperAPIKey: nil, helperReviewerModelJSON: reviewerModelJSON, pluginFactoryCreationActive: executionContextJSON != nil && TurnProcessContext.effectivePluginFactoryCreationActive, diff --git a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift index 543dcda0..e5016dab 100644 --- a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift +++ b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift @@ -18,8 +18,44 @@ final class AppBootstrapStatus: ObservableObject { case failed } + /// Parallel bootstrap steps shown in the init modal. Completed steps are removed from the list. + enum TaskID: String, Sendable, CaseIterable, Equatable { + case daemon + case database + case docker + case workerImage + + var sortOrder: Int { + switch self { + case .daemon: return 0 + case .database: return 1 + case .docker: return 2 + case .workerImage: return 3 + } + } + + var defaultMessage: String { + switch self { + case .daemon: + return "Connecting to Derrick daemon…" + case .database: + return "Opening local database…" + case .docker: + return "Checking Docker Desktop…" + case .workerImage: + return "Preparing worker image…" + } + } + } + + struct LoadingTask: Identifiable, Equatable, Sendable { + let id: TaskID + var message: String + } + @Published private(set) var phase: Phase = .idle @Published private(set) var statusMessage: String = "Starting…" + @Published private(set) var activeLoadingTasks: [LoadingTask] = [] @Published private(set) var failureTitle: String? @Published private(set) var failureMessage: String? /// Extra recovery control on the failure modal (for example Open Login Items). @@ -43,6 +79,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = false phase = .idle statusMessage = "Starting…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -108,6 +145,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = deferModal phase = .loadingSession statusMessage = "Loading session store…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -138,38 +176,86 @@ final class AppBootstrapStatus: ObservableObject { } } + func beginTask(_ id: TaskID, message: String? = nil) { + guard isInitializing else { return } + let label = message ?? id.defaultMessage + if let index = activeLoadingTasks.firstIndex(where: { $0.id == id }) { + activeLoadingTasks[index].message = label + } else { + activeLoadingTasks.append(LoadingTask(id: id, message: label)) + activeLoadingTasks.sort { $0.id.sortOrder < $1.id.sortOrder } + } + if !deferModalPresentation { + isModalPresented = true + } + debugLog("[bootstrap] task begin \(id.rawValue): \(label)") + } + + func updateTask(_ id: TaskID, message: String) { + guard isInitializing else { return } + if let index = activeLoadingTasks.firstIndex(where: { $0.id == id }) { + activeLoadingTasks[index].message = message + } else { + beginTask(id, message: message) + } + debugLog("[bootstrap] task update \(id.rawValue): \(message)") + } + + func completeTask(_ id: TaskID) { + guard activeLoadingTasks.contains(where: { $0.id == id }) else { return } + activeLoadingTasks.removeAll { $0.id == id } + debugLog("[bootstrap] task complete \(id.rawValue)") + } + func update(phase: Phase, message: String) { // Never re-open the modal after ready (parallel service ensure-up must not reflash it). if self.phase == .ready, phase != .failed, phase != .ready { debugLog("[bootstrap] ignore phase=\(phase.rawValue) (already ready): \(message)") return } - // Parallel bootstrap: once we move past Docker prep, do not let guest-image - // prewarm overwrite daemon/database status in the modal. - if phase == .checkingDocker || phase == .preparingImage || phase == .verifyingEnvironment { - switch self.phase { - case .connectingHelper, .loadingSession: - debugLog("[bootstrap] ignore docker phase=\(phase.rawValue) while \(self.phase.rawValue): \(message)") - return - default: - break - } + // Parallel bootstrap: keep the highest-priority in-flight step visible (daemon connect + // beats "Opening local database…" while XPC is still retrying). + if isInitializing, Self.phasePriority(phase) < Self.phasePriority(self.phase) { + debugLog("[bootstrap] ignore lower-priority phase=\(phase.rawValue) while \(self.phase.rawValue): \(message)") + } else { + // Don't let a cancelled re-entrant task demote ready via failed paths above. + self.phase = phase + self.statusMessage = message } - // Don't let a cancelled re-entrant task demote ready via failed paths above. - self.phase = phase - self.statusMessage = message + syncLoadingTask(for: phase, message: message) if !deferModalPresentation { isModalPresented = true } debugLog("[bootstrap] phase=\(phase.rawValue) \(message)") } + private func syncLoadingTask(for phase: Phase, message: String) { + guard isInitializing else { return } + switch phase { + case .connectingHelper: + beginTask(.daemon, message: message) + case .loadingSession: + if message.localizedCaseInsensitiveContains("database") { + beginTask(.database, message: message) + } + case .checkingDocker: + beginTask(.docker, message: message) + case .preparingImage: + beginTask(.workerImage, message: message) + case .verifyingEnvironment: + completeTask(.workerImage) + default: + break + } + } + func markReady() { deferredModalRevealTask?.cancel() deferredModalRevealTask = nil deferModalPresentation = false phase = .ready statusMessage = "Ready" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -213,6 +299,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = false phase = .idle statusMessage = "Starting…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -226,6 +313,16 @@ final class AppBootstrapStatus: ObservableObject { debugLog("[bootstrap] failure modal dismissed") } + private static func phasePriority(_ phase: Phase) -> Int { + switch phase { + case .connectingHelper: return 4 + case .checkingDocker: return 3 + case .preparingImage, .verifyingEnvironment: return 2 + case .loadingSession: return 1 + default: return 0 + } + } + enum FailureRecovery: Equatable, Sendable { case none case retryDaemon @@ -240,6 +337,9 @@ final class AppBootstrapStatus: ObservableObject { /// Maps prewarm / Docker errors into a short title and user-facing explanation. static func classifyError(_ error: Error) -> ClassifiedFailure { + if let goError = error as? DerrickGoToolchainError { + return classifyGoToolchainError(goError) + } if let agentError = error as? JobServiceLoginAgent.AgentError { return classifyDaemonAgentError(agentError) } @@ -338,6 +438,40 @@ final class AppBootstrapStatus: ObservableObject { ) } + private static func classifyGoToolchainError( + _ error: DerrickGoToolchainError + ) -> ClassifiedFailure { + switch error { + case .missing: + return ClassifiedFailure( + title: "Go Toolchain Required", + message: """ + Derrick could not find a Go toolchain for development diagnostics. Guest compile runs in Docker; this error is unexpected. + + Quit and reopen Derrick. If it persists, reinstall Docker Desktop. + """ + ) + case .unparseable: + return ClassifiedFailure( + title: "Go Toolchain Unreadable", + message: """ + Derrick could not read the installed Go version. Guest compile runs in Docker; this error is unexpected. + + \(error.localizedDescription) + """ + ) + case .tooOld(_, let required): + return ClassifiedFailure( + title: "Go Toolchain Too Old", + message: """ + Derrick found an older Go toolchain than \(required). Guest compile runs in Docker; this error is unexpected. + + Quit and reopen Derrick. + """ + ) + } + } + private static func classifyDaemonAgentError( _ error: JobServiceLoginAgent.AgentError ) -> ClassifiedFailure { diff --git a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift index f667eef2..b08dedf1 100644 --- a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift +++ b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift @@ -164,11 +164,13 @@ public final class XPCDockerRunner: @unchecked Sendable { public static let shared = XPCDockerRunner() private static let serviceName = "derrick.ui.DockerRunnerHelper" - private static let prewarmWaitCeilingSeconds: UInt64 = 1_200 + private static let dockerReachableWaitCeilingSeconds: UInt64 = 60 + private static let imagePrewarmWaitCeilingSeconds: UInt64 = 1_200 private let connection: NSXPCConnection private let appLogSink: XPCAppLogSink - private let prewarmState = PrewarmState() + private let dockerReachableState = PrewarmState() + private let imagePrewarmState = PrewarmState() public init() { let sink = XPCAppLogSink() @@ -206,11 +208,35 @@ public final class XPCDockerRunner: @unchecked Sendable { } } + /// Waits until Docker Desktop responds. Does not wait for the worker image build. + public func waitUntilDockerReachable() async throws { + if dockerReachableState.isCompleted() { + return + } + if let failure = dockerReachableState.failureIfCompleted() { + throw failure + } + let timeout = NSError( + domain: "XPCDockerRunner", + code: 504, + userInfo: [ + NSLocalizedDescriptionKey: + "Docker Desktop did not respond within \(Self.dockerReachableWaitCeilingSeconds)s." + ] + ) + try await dockerReachableState.wait( + timeoutNanoseconds: Self.dockerReachableWaitCeilingSeconds * 1_000_000_000, + timeoutError: timeout + ) + } + + /// Waits until the worker image is built or verified. Joins an in-flight background build. public func waitUntilPrewarmed() async throws { - if prewarmState.isCompleted() { + try await waitUntilDockerReachable() + if imagePrewarmState.isCompleted() { return } - if let failure = prewarmState.failureIfCompleted() { + if let failure = imagePrewarmState.failureIfCompleted() { throw failure } let timeout = NSError( @@ -218,11 +244,11 @@ public final class XPCDockerRunner: @unchecked Sendable { code: 504, userInfo: [ NSLocalizedDescriptionKey: - "Guest runtime setup timed out after \(Self.prewarmWaitCeilingSeconds)s." + "Worker image setup timed out after \(Self.imagePrewarmWaitCeilingSeconds)s." ] ) - try await prewarmState.wait( - timeoutNanoseconds: Self.prewarmWaitCeilingSeconds * 1_000_000_000, + try await imagePrewarmState.wait( + timeoutNanoseconds: Self.imagePrewarmWaitCeilingSeconds * 1_000_000_000, timeoutError: timeout ) } @@ -280,21 +306,28 @@ public final class XPCDockerRunner: @unchecked Sendable { ] ) } - await reportBootstrap(phase: .preparingImage, message: "Preparing guest runtime…") - let executor = makeDockerExecutor() - let image = DerrickGuestRuntime.pythonGuestDockerImage - let inspect = try await executor(["image", "inspect", image], Data(), 30) - if inspect.exitCode != 0 { - await MainActor.run { - AppBootstrapStatus.shared.revealModalIfStillInitializing() - } - try await OneshotDockerContainer.ensurePulledImage(image, executor: executor) + dockerReachableState.markCompleted() + await reportBootstrapTaskCompleted(.docker) + Task { + await prewarmWorkerImage() } - prewarmState.markCompleted() - await reportBootstrap(phase: .verifyingEnvironment, message: "Guest runtime ready.") } catch { - debugLog("Guest runtime prewarming failed: \(error.localizedDescription)") - prewarmState.markFailed(error) + debugLog("Docker reachability check failed: \(error.localizedDescription)") + dockerReachableState.markFailed(error) + imagePrewarmState.markFailed(error) + } + } + + private func prewarmWorkerImage() async { + do { + await reportBootstrap(phase: .preparingImage, message: "Preparing worker image…") + let executor = makeDockerExecutor() + try await WorkerImageGate.shared.ensureReady(executor: executor) + imagePrewarmState.markCompleted() + await reportBootstrap(phase: .verifyingEnvironment, message: "Worker image ready.") + } catch { + debugLog("Worker image prewarm failed: \(error.localizedDescription)") + imagePrewarmState.markFailed(error) } } @@ -381,6 +414,14 @@ public final class XPCDockerRunner: @unchecked Sendable { } } + private func reportBootstrapTaskCompleted(_ id: AppBootstrapStatus.TaskID) async { + await MainActor.run { + let status = AppBootstrapStatus.shared + guard status.isInitializing else { return } + status.completeTask(id) + } + } + deinit { connection.invalidate() } diff --git a/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift b/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift deleted file mode 100644 index 03c3ef13..00000000 --- a/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift +++ /dev/null @@ -1,273 +0,0 @@ -import Foundation -import Combine -import DBRepository -import EgressProxy -import AppEvents -import PolicyUserInteraction -import Structure - -/// App-owned egress allowlist: DB persistence + host HTTP preflight prompts. -/// Not exposed as MCP. Not part of tool/content policy rules. -@MainActor -final class EgressAllowlistService: ObservableObject { - static let shared = EgressAllowlistService() - - @Published private(set) var suffixes: [EgressAllowedDomainSuffix] = [] - - private var repository: DBRepository? - private let username = "ui" - private let password = "ui" - private let localPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) - - private init() {} - - func configure(repository: DBRepository) async { - self.repository = repository - do { - let inserted = try await repository.seedEgressAllowedDomainSuffixesIfNeeded( - EgressProxyConfiguration.defaultSeedDomainSuffixes, - source: "seed" - ) - if inserted > 0 { - debugLog("Egress allowlist seed inserted \(inserted) suffix(es).") - } else { - debugLog("Egress allowlist seed skipped (suffixes already present).") - } - try await reload() - } catch { - debugLog("Egress allowlist configure failed: \(error.localizedDescription)") - } - } - - func reload(clearSessionHosts: Bool = false) async throws { - guard let repository else { - suffixes = [] - return - } - let rows = try await repository.loadEgressAllowedDomainSuffixes(includeDisabled: true) - suffixes = rows - localPolicy.setAllowedDomainSuffixes(rows.filter(\.enabled).map(\.suffix)) - if clearSessionHosts { - // Settings edits must not be shadowed by prior Allow-once / mid-flight grants. - localPolicy.clearSessionHosts() - } - } - - private var isAgentServiceProcess: Bool { - let bid = Bundle.main.bundleIdentifier ?? "" - return bid == DerrickServiceID.agent.rawValue || bid.hasSuffix(".AgentService") - } - - func addSuffix(_ raw: String, source: String = "user") async throws { - let suffix = EgressHostExtractor.permanentSuffix(for: raw) - guard EgressHostExtractor.isPlausibleHostname(suffix) || suffix.contains(".") else { - throw NSError( - domain: "EgressAllowlist", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Invalid domain suffix: \(raw)"] - ) - } - guard let repository else { return } - try await repository.saveEgressAllowedDomainSuffix( - EgressAllowedDomainSuffix(suffix: suffix, source: source, enabled: true) - ) - try await reload() - } - - func removeSuffix(id: String) async throws { - guard let repository else { return } - try await repository.deleteEgressAllowedDomainSuffix(id: id) - try await reload(clearSessionHosts: true) - debugLog("Egress allowlist removed id=\(id)") - } - - /// Preflight network hosts before script execution. - /// - Returns: nil if allowed to proceed; blocked tool-result JSON if user denied or hard-blocked. - func preflightScriptNetwork( - script: String, - allowNetwork: Bool, - toolName: String = "script_exec" - ) async -> String? { - guard allowNetwork else { return nil } - - let preflightStarted = Date() - let hosts = EgressHostExtractor.extractHosts(from: script) - guard !hosts.isEmpty else { - PipelineTiming.log("egress_preflight hosts=0 total_ms=0 modal_ms=0") - return nil - } - - var sessionGrants: [String] = [] - var modalMS = 0 - var needsPrompt: [String] = [] - - for host in hosts { - if localPolicy.isHardBlockedHostname(host) { - let message = "Network access to “\(host)” is permanently blocked (private/metadata host)." - PipelineTiming.log( - "egress_preflight blocked_hard host=\(host) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - // Modal is published by the pipeline from the common network outcome. - return Self.blockedResultJSON(findings: [message]) - } - - if localPolicy.isHostCoveredByAllowlist(host) { - continue - } - needsPrompt.append(host) - } - - if !needsPrompt.isEmpty { - let modalStarted = Date() - let decision = await promptForHosts(needsPrompt, toolName: toolName) - modalMS += PipelineTiming.elapsedMS(from: modalStarted) - switch decision { - case .approved(let actor), .approvedOnce(let actor): - debugLog("Egress allow once for \(needsPrompt.count) host(s) by \(actor ?? "user")") - sessionGrants.append(contentsOf: needsPrompt) - localPolicy.grantSessionHosts(needsPrompt) - case .approvedPermanently(let actor): - debugLog("Egress allow always for \(needsPrompt.count) host(s) by \(actor ?? "user")") - for host in needsPrompt { - let suffix = EgressHostExtractor.permanentSuffix(for: host) - do { - try await addSuffix(suffix, source: "user") - sessionGrants.append(host) - localPolicy.grantSessionHosts([host]) - } catch { - debugLog("Failed to persist egress suffix \(suffix): \(error.localizedDescription)") - PipelineTiming.log( - "egress_preflight persist_failed host=\(host) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return Self.blockedResultJSON( - findings: ["Failed to save permanent allow for \(host): \(error.localizedDescription)"] - ) - } - } - case .denied(let actor): - debugLog("Egress deny for \(needsPrompt.count) host(s) by \(actor ?? "user") — aborting entire script run") - let listed = needsPrompt.joined(separator: ", ") - let message = "User denied network access to “\(listed)”. The script was not run." - PipelineTiming.log( - "egress_preflight user_denied hosts=\(needsPrompt.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS) prompted_hosts=\(needsPrompt.count)" - ) - return Self.blockedResultJSON(findings: [message]) - case .dismissed, .timedOut: - let listed = needsPrompt.joined(separator: ", ") - let message = "Network access to “\(listed)” was not approved. The script was not run." - PipelineTiming.log( - "egress_preflight dismissed_or_timeout hosts=\(needsPrompt.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return Self.blockedResultJSON(findings: [message]) - } - } - - PipelineTiming.log( - "egress_preflight ok hosts=\(hosts.count) prompted=\(needsPrompt.count) session_grants=\(sessionGrants.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return nil - } - - private func promptForHosts(_ hosts: [String], toolName: String) async -> PolicyUserDecision { - let unique = Self.uniqueHosts(hosts) - guard !unique.isEmpty else { - return .denied(actor: "system") - } - - if let remote = TurnProcessContext.effectiveNetworkAccessPrompt { - var last: PolicyUserDecision = .denied(actor: "system") - for host in unique { - debugLog("Egress prompt via AgentService path host=\(host)") - last = await remote(host, toolName) - switch last { - case .denied, .dismissed, .timedOut: - return last - case .approved, .approvedOnce, .approvedPermanently: - continue - } - } - return last - } - - if !isAgentServiceProcess { - debugLog("Egress prompt via UI modal hosts=\(unique.count)") - let event = PolicyUserEventFactory.egressAccessRequest( - hosts: unique, - toolName: toolName - ) - return await AppEventBus.shared.initDecision(event) - } - - guard let repository else { - return .denied(actor: "system-no-repository") - } - var last: PolicyUserDecision = .denied(actor: "system") - for host in unique { - debugLog("Egress prompt via notification path host=\(host)") - last = await HITLOfflineNetworkService.awaitDecision( - host: host, - toolName: toolName, - turnID: "egress-agent", - isJobContext: false, - repository: repository, - timeoutNanoseconds: 300_000_000_000 - ) - switch last { - case .denied, .dismissed, .timedOut: - return last - case .approved, .approvedOnce, .approvedPermanently: - continue - } - } - return last - } - - private static func uniqueHosts(_ hosts: [String]) -> [String] { - var seen = Set() - var ordered: [String] = [] - for raw in hosts { - let host = raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard !host.isEmpty, seen.insert(host).inserted else { continue } - ordered.append(host) - } - return ordered - } - - /// Apply a user egress decision in this process. - /// Call when the UI answers a network prompt so later requests do not re-prompt. - func applyUserNetworkDecision(host: String, decision: PolicyUserDecision) async { - let normalized = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard !normalized.isEmpty else { return } - - switch decision { - case .approved(let actor), .approvedOnce(let actor): - debugLog("Egress UI apply once host=\(normalized) actor=\(actor ?? "?")") - localPolicy.grantSessionHosts([normalized]) - case .approvedPermanently(let actor): - debugLog("Egress UI apply always host=\(normalized) actor=\(actor ?? "?")") - let suffix = EgressHostExtractor.permanentSuffix(for: normalized) - do { - try await addSuffix(suffix, source: "user") - } catch { - debugLog("Egress UI apply always persist failed: \(error.localizedDescription)") - } - localPolicy.grantSessionHosts([normalized]) - case .denied, .dismissed, .timedOut: - break - } - } - - private static func blockedResultJSON(findings: [String]) -> String { - let diagnostics = findings.isEmpty - ? [ToolExecutionOutcome.Diagnostic(code: "egress_denied", message: "Network access was denied.")] - : findings.map { - ToolExecutionOutcome.Diagnostic(code: "egress_denied", message: $0) - } - return (try? ToolExecutionOutcome.failure( - status: .blocked, - stage: .network, - diagnostics: diagnostics, - retry: ToolExecutionOutcome.Retry(allowed: false) - ).encodedJSON()) ?? #"{"status":"blocked","stage":"network","diagnostics":[]}"# - } -} diff --git a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift index 3b9552b6..035572df 100644 --- a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift +++ b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift @@ -21,7 +21,7 @@ actor ConfiguredScriptReviewer: ScriptReviewer { func review(_ args: ScriptExecutionArguments) async throws -> ScriptReviewOutcome { let selectedModel = await MainActor.run { settings.scriptReviewerModel } - guard let apiKey = await resolveAPIKey(for: selectedModel) else { + guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: selectedModel) else { await MainActor.run { debugLog( "Helper reviewer model \(selectedModel.helperDisplayName) unavailable; trying default helper reviewer." @@ -83,7 +83,7 @@ actor ConfiguredScriptReviewer: ScriptReviewer { if selectedModel == defaultModel { return nil } - guard let apiKey = await resolveAPIKey(for: defaultModel) else { + guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: defaultModel) else { return nil } @@ -111,17 +111,4 @@ actor ConfiguredScriptReviewer: ScriptReviewer { return try await reviewer.review(args) } } - - private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - // AgentService XPC process cannot read the UI keychain; use the turn-supplied key. - return TurnProcessContext.effectiveAPIKey - } } diff --git a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift index fe516fdb..8697e4d8 100644 --- a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift +++ b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift @@ -111,15 +111,7 @@ actor ConfiguredMemorySummarizer: MemorySummarizer { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return TurnProcessContext.effectiveAPIKey + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private static func makeSummary(text: String, keywords: [String], sourceTokenCount: Int) -> MemorySummary { diff --git a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift index 30ef4955..225f4294 100644 --- a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift +++ b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift @@ -13,22 +13,18 @@ actor ConfiguredPluginFactoryService { private let thinkingSettings: LLMModelThinkingSettings private let executor: any PluginFactoryExecutor private let logger: PluginFactoryLogger - private let apiKeyProvider: @Sendable () -> String? - init( repository: DBRepository, settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, executor: any PluginFactoryExecutor, - logger: @escaping PluginFactoryLogger = { _ in }, - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey } + logger: @escaping PluginFactoryLogger = { _ in } ) { self.repository = repository self.settings = settings self.thinkingSettings = thinkingSettings self.executor = executor self.logger = logger - self.apiKeyProvider = apiKeyProvider } func build( @@ -43,7 +39,6 @@ actor ConfiguredPluginFactoryService { settings: settings, thinkingSettings: thinkingSettings, existingReleases: existingReleases, - apiKeyProvider: apiKeyProvider, logger: logger ), executor: executor, @@ -51,7 +46,6 @@ actor ConfiguredPluginFactoryService { inner: ConfiguredPluginSafetyReviewer( settings: settings, thinkingSettings: thinkingSettings, - apiKeyProvider: apiKeyProvider, logger: logger ) ), @@ -68,20 +62,17 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { private let settings: LLMModelSettings private let thinkingSettings: LLMModelThinkingSettings private let existingReleases: [PluginFactoryReleaseSummary] - private let apiKeyProvider: @Sendable () -> String? private let logger: PluginFactoryLogger init( settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, existingReleases: [PluginFactoryReleaseSummary] = [], - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, logger: @escaping PluginFactoryLogger = { _ in } ) { self.settings = settings self.thinkingSettings = thinkingSettings self.existingReleases = existingReleases - self.apiKeyProvider = apiKeyProvider self.logger = logger } @@ -107,15 +98,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return apiKeyProvider() + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private func stream( @@ -148,49 +131,8 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { private static func builderSystemPrompt(for userGoal: String) -> String { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. - Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), python_source (string), - test_input_json (string containing valid JSON — a serialized object, not prose), - skill_files (array of objects with path and body), - secrets (array of objects with id, label, and kind; required for connector plugins), - role (string, optional: "connector" or "standard"). - plugin_id must use lowercase letters, numbers, hyphens, and dots only - (for example my-connector). Never use underscores in plugin_id. - If the plugin needs a username, password, token, or API key, declare them in secrets. - kind must be username, password, token, or api_key. id is a stable Keychain key - such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in python_source. - Set role to "connector" when the plugin sends and receives messages with an external - messaging service (any chat or mail connector). Omit role or use "standard" otherwise. - For role connector, include messaging_ops: an array of implemented ops - (send_message, poll_inbox, sync_threads). It must match the user goal scope and test_input_json. - The host writes messaging_ops into extensions.app.derrick in plugin.json. - The host lists connector plugins under Messaging. Do not guess this from the plugin_id. - Do not return manifest_json. The host creates the canonical Agent Plugin manifest, - including the exact `$schema` field for Agent Plugin 1.0 and the fixed - extensions.app.derrick.entrypoint ./app.derrick/plugin.py. - \(DerrickGuestPython.modelContract) + \(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) - Before returning the draft, self-check the implementation: - - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - - Match host responses by the emitted request_id. - - Use only the Python standard library (no pip, requests, urllib, socket, or subprocess). - - The direct test input must exercise the terminal result path with matching http_results fixtures. - If skill_files is not needed, return an empty array. Every skill file path must be exactly - skills//SKILL.md. - For messaging connector plugins (role connector) that call a vendor HTTP API: - - Declare secrets in the manifest only. Never hard-code credentials. - - Parse each http_results body as JSON when the vendor returns JSON. - - test_input_json http_results must exercise success paths for every messaging_op in scope. - - test_input_json must be a single JSON object serialized as a string (valid JSON.parse input). - - test_input_json must not be empty or "{}". - - For connector plugins, test_input_json must use a hops array: - {"hops":[{"kind":"message_in_room","params":{"messaging_op":"send_message",...}},\ - {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ - "params":{...}}]} - Repeat additional hop pairs for each messaging_op in scope. request_id values in fixtures must \ - match the http.request envelopes your python_source emits. - - Match http_results by request_id and de-duplicate with stable sorting; never depend on response order. When vendor documentation is supplied in the user prompt, use it only to fill may_call HTTP details. """ } @@ -201,7 +143,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { "plugin_id": AgentSchema(type: .string), "version": AgentSchema(type: .string), "description": AgentSchema(type: .string), - "python_source": AgentSchema(type: .string), + "go_source": AgentSchema(type: .string), "test_input_json": AgentSchema(type: .string), "skill_files": AgentSchema( type: .array, @@ -233,7 +175,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { ), ], required: [ - "plugin_id", "version", "description", "python_source", + "plugin_id", "version", "description", "go_source", "test_input_json", "skill_files", ] ) @@ -248,7 +190,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { """ The host already assigned plugin_id \(host.pluginID) and these secret ids: \ \(host.secrets.map(\.id).joined(separator: ", ")). \ - Return python_source and test_input_json. Do not pick a different plugin_id or secret ids. \ + Return go_source and test_input_json. Do not pick a different plugin_id or secret ids. \ Use {{secret:\(host.secrets.first?.id ?? "bot_token")}} in HTTP headers. """ ) @@ -315,18 +257,15 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private let settings: LLMModelSettings private let thinkingSettings: LLMModelThinkingSettings - private let apiKeyProvider: @Sendable () -> String? private let logger: PluginFactoryLogger init( settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, logger: @escaping PluginFactoryLogger = { _ in } ) { self.settings = settings self.thinkingSettings = thinkingSettings - self.apiKeyProvider = apiKeyProvider self.logger = logger } @@ -354,15 +293,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return apiKeyProvider() + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private func stream( @@ -394,22 +325,9 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ - You are Derrick's independent plugin alignment and safety reviewer. - Review the user's goal, manifest, test_input_json, exact Python source, and direct test output. - Return exactly one JSON object: - {"decision":"approved|rejected","summary":"...","findings":[ - {"severity":"info|warning|blocking","category":"alignment|safety|correctness|privacy|supplyChain","message":"..."} - ]} - Reject unsafe, misleading, unrelated, non-deterministic, credential-seeking, or policy-bypassing code. - Apply these checks from observable evidence: - - A deterministic result uses stable sorting and de-duplication and does not depend on response order, current time, randomness, or UUIDs. - - Source-derived headline titles may be fragments; only generated explanatory summaries must be complete sentences when the manifest requires prose. - - `result.emit.html` is an allowed output format. Derrick sanitizes it with an allowlist before rendering. Reject executable script behavior or a deliberate sanitizer bypass, not ordinary safe HTML tags. - - Reject missing source-grounded parsing or claims that the direct test output does not support. - - For connector plugins, obey the connector protocol JSON below. If a rule is not in that JSON, do not require it. + Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. + \(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) - Compilation success is not approval. Do not rewrite the code or approve a draft that fails these checks. - Reject Swift source, socket/urllib/requests usage, or missing stdin reads. """ } @@ -449,7 +367,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { test_input_json: \(String(decoding: draft.testInput, as: UTF8.self)) - Python source: + Go source: \(draft.guestSource) Direct test output: diff --git a/ui/SharedAgentRuntime/TurnProcessContext.swift b/ui/SharedAgentRuntime/TurnProcessContext.swift index ab0f2532..5e5759c7 100644 --- a/ui/SharedAgentRuntime/TurnProcessContext.swift +++ b/ui/SharedAgentRuntime/TurnProcessContext.swift @@ -31,7 +31,7 @@ public enum TurnProcessContext { /// Active agent profile handle for the current user-facing turn (orchestrator-only tools). @TaskLocal public static var activeProfileHandle: String? - /// Active `/create-plugin` or `/edit-plugin` factory turn (enables sync `web.crawl`). + /// Active `/create-plugin` or `/edit-plugin` factory turn. @TaskLocal public static var pluginFactoryCreationActive: Bool = false public static func install( diff --git a/ui/ui/Jobs/DerrickNotificationService.swift b/ui/ui/Jobs/DerrickNotificationService.swift index 74b1c9a2..e0c1d92e 100644 --- a/ui/ui/Jobs/DerrickNotificationService.swift +++ b/ui/ui/Jobs/DerrickNotificationService.swift @@ -170,15 +170,6 @@ final class DerrickNotificationService { fputs("[HumanDecision] resolve skip id=\(approvalID)\n", stderr) return } - if approved, HITLOfflineNetworkService.isNetworkToolName(row.toolName), - let host = HITLOfflineNetworkService.host(fromNetworkToolName: row.toolName) { - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: host, - decision: always - ? .approvedPermanently(actor: actor) - : .approvedOnce(actor: actor) - ) - } let status: PendingHITLApprovalStatus = approved ? .approved : .cancelled let edited = approved ? row.argumentsJSON : nil try? await repository.resolveHITLApproval( @@ -264,9 +255,6 @@ final class DerrickNotificationService { databaseDirectoryURL: directory ) repository = repo - if !JobResultPanelSession.isPanelOnlyLaunch { - await EgressAllowlistService.shared.configure(repository: repo) - } return repo } catch { fputs("[HumanDecision] ensureRepository failed: \(error.localizedDescription)\n", stderr) diff --git a/ui/ui/Jobs/HITLLiveApprovalHandlers.swift b/ui/ui/Jobs/HITLLiveApprovalHandlers.swift index cd52c2b2..8f70c351 100644 --- a/ui/ui/Jobs/HITLLiveApprovalHandlers.swift +++ b/ui/ui/Jobs/HITLLiveApprovalHandlers.swift @@ -166,51 +166,11 @@ enum HITLLiveApprovalHandlers { } private static func presentNetworkAccess(_ request: AgentNetworkAccessRequestDTO) async -> AgentNetworkAccessDecisionDTO { - let event = PolicyUserEventFactory.egressAccessRequest( - host: request.host, - toolName: request.toolName, - correlationId: request.requestID + // Blacklist-only egress: public hosts are allowed by default. Blacklist hits use PolicyEventBus. + AgentNetworkAccessDecisionDTO( + requestID: request.requestID, + decision: "once", + actor: "blacklist-only-auto" ) - let decision = await AppEventBus.shared.initDecision(event) - switch decision { - case .approvedOnce(let actor): - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: request.host, - decision: .approvedOnce(actor: actor) - ) - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "once", - actor: actor ?? "ui-modal-once" - ) - case .approvedPermanently(let actor): - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: request.host, - decision: .approvedPermanently(actor: actor) - ) - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "always", - actor: actor ?? "ui-modal-always" - ) - case .timedOut: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "timeout", - actor: "ui-modal-timeout" - ) - case .dismissed: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "dismissed", - actor: "ui-modal-dismissed" - ) - case .approved, .denied: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "deny", - actor: "ui-modal-deny" - ) - } } } diff --git a/ui/ui/Messaging/AppWorkspace.swift b/ui/ui/Messaging/AppWorkspace.swift index 08095538..872c89b9 100644 --- a/ui/ui/Messaging/AppWorkspace.swift +++ b/ui/ui/Messaging/AppWorkspace.swift @@ -4,6 +4,5 @@ enum AppWorkspace: Equatable { case chats case plugins case messaging - case news case debugLogs } diff --git a/ui/ui/News/NewsReaderStore.swift b/ui/ui/News/NewsReaderStore.swift deleted file mode 100644 index e5ea5ddc..00000000 --- a/ui/ui/News/NewsReaderStore.swift +++ /dev/null @@ -1,126 +0,0 @@ -import Combine -import DBRepository -import Foundation -import Structure - -@MainActor -final class NewsReaderStore: ObservableObject { - static let shared = NewsReaderStore() - - @Published private(set) var readers: [NewsReaderSpec] = [] - @Published private(set) var items: [NewsItem] = [] - @Published var selectedReaderID: String? - @Published private(set) var isRefreshing = false - @Published private(set) var lastError: String? - - private var repository: DBRepository? - private let client: any NewsHTTPClient - - init(client: any NewsHTTPClient = URLSessionNewsHTTPClient()) { - self.client = client - } - - var selectedReader: NewsReaderSpec? { - readers.first { $0.id == selectedReaderID } - } - - func configure(repository: DBRepository) async { - self.repository = repository - await reload() - } - - func reload() async { - guard let repository else { return } - do { - readers = try await repository.listNewsReaders() - if selectedReaderID == nil { - selectedReaderID = readers.first?.id - } - if let id = selectedReaderID { - items = try await repository.listNewsItems(readerID: id) - } else { - items = [] - } - lastError = nil - } catch { - lastError = error.localizedDescription - } - } - - func select(id: String) async { - selectedReaderID = id - await reloadItems() - if let reader = selectedReader, shouldRefreshForSchedule(reader) { - await refreshSelected() - } - } - - @discardableResult - func create(_ spec: NewsReaderSpec) async throws -> NewsReaderSpec { - guard let repository else { - throw NewsReaderError.notReady - } - var next = spec - next.updatedAt = .now - let items = try await NewsReaderRefresh.validateAndFetch(spec: next, client: client) - next.lastFetchedAt = .now - next.lastError = nil - try await repository.upsertNewsReader(next) - try await repository.replaceNewsItems(readerID: next.id, items: items) - await reload() - selectedReaderID = next.id - self.items = items - return next - } - - func refreshSelected() async { - guard let repository, var reader = selectedReader else { return } - isRefreshing = true - defer { isRefreshing = false } - do { - let fetched = try await NewsReaderRefresh.validateAndFetch(spec: reader, client: client) - reader.lastFetchedAt = .now - reader.lastError = nil - reader.updatedAt = .now - try await repository.upsertNewsReader(reader) - try await repository.replaceNewsItems(readerID: reader.id, items: fetched) - items = fetched - lastError = nil - await reload() - } catch { - reader.lastError = error.localizedDescription - reader.updatedAt = .now - try? await repository.upsertNewsReader(reader) - lastError = error.localizedDescription - await reload() - } - } - - func deleteSelected() async { - guard let repository, let id = selectedReaderID else { return } - try? await repository.deleteNewsReader(id: id) - selectedReaderID = nil - await reload() - } - - private func reloadItems() async { - guard let repository, let id = selectedReaderID else { - items = [] - return - } - items = (try? await repository.listNewsItems(readerID: id)) ?? [] - } - - private func shouldRefreshForSchedule(_ reader: NewsReaderSpec) -> Bool { - guard reader.schedule != .off else { return false } - guard let last = reader.lastFetchedAt else { return true } - switch reader.schedule { - case .off: - return false - case .hourly: - return Date().timeIntervalSince(last) >= 3_600 - case .daily: - return Date().timeIntervalSince(last) >= 86_400 - } - } -} diff --git a/ui/ui/News/NewsWorkspaceView.swift b/ui/ui/News/NewsWorkspaceView.swift deleted file mode 100644 index ff8a4ab5..00000000 --- a/ui/ui/News/NewsWorkspaceView.swift +++ /dev/null @@ -1,132 +0,0 @@ -import SwiftUI -import Structure - -struct NewsWorkspaceView: View { - @ObservedObject var store: NewsReaderStore - - var body: some View { - VStack(spacing: 0) { - header - Divider() - if store.selectedReader == nil { - emptyState - } else { - itemList - } - } - .background(Color(red: 252.0 / 255.0, green: 252.0 / 255.0, blue: 250.0 / 255.0)) - } - - private var header: some View { - HStack(spacing: 12) { - VStack(alignment: .leading, spacing: 2) { - Text(store.selectedReader?.name ?? "News") - .font(.headline) - if let reader = store.selectedReader { - Text(headerSubtitle(reader)) - .font(.caption) - .foregroundStyle(.secondary) - } - } - Spacer() - if store.selectedReader != nil { - Button { - Task { await store.refreshSelected() } - } label: { - if store.isRefreshing { - ProgressView() - .controlSize(.small) - } else { - Text("Refresh") - } - } - .disabled(store.isRefreshing) - .buttonStyle(.bordered) - } - } - .padding(.horizontal, 24) - .padding(.vertical, 14) - } - - private var emptyState: some View { - VStack(spacing: 10) { - Spacer() - Image(systemName: "newspaper") - .font(.system(size: 36)) - .foregroundStyle(.secondary) - Text("No news lists yet") - .font(.title3.weight(.semibold)) - Text("Create a News reader from Plugins. You can pick several topics and several sources.") - .font(.subheadline) - .foregroundStyle(.secondary) - .multilineTextAlignment(.center) - .frame(maxWidth: 420) - Spacer() - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - } - - private var itemList: some View { - ScrollView { - VStack(alignment: .leading, spacing: 16) { - if let error = store.selectedReader?.lastError ?? store.lastError, !error.isEmpty { - Text(error) - .font(.caption) - .foregroundStyle(.red) - .padding(12) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.red.opacity(0.08), in: RoundedRectangle(cornerRadius: 10)) - } - if store.selectedReader?.mode == .summaries, !store.items.isEmpty { - VStack(alignment: .leading, spacing: 8) { - Text("Summary") - .font(.subheadline.weight(.semibold)) - Text(NewsReaderRefresh.digest(from: store.items)) - .font(.body) - .textSelection(.enabled) - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) - } - ForEach(store.items) { item in - VStack(alignment: .leading, spacing: 6) { - Link(destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) { - Text(item.title) - .font(.body.weight(.semibold)) - .multilineTextAlignment(.leading) - } - HStack(spacing: 8) { - Text(item.sourceLabel) - .font(.caption) - .foregroundStyle(.secondary) - Link("Source", destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) - .font(.caption.weight(.semibold)) - } - if let summary = item.summary, !summary.isEmpty, store.selectedReader?.mode == .list { - Text(summary) - .font(.caption) - .foregroundStyle(.secondary) - .lineLimit(4) - } - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) - } - if store.items.isEmpty, store.lastError == nil, store.selectedReader?.lastError == nil { - Text("No articles yet. Refresh this list.") - .font(.subheadline) - .foregroundStyle(.secondary) - } - } - .padding(24) - } - } - - private func headerSubtitle(_ reader: NewsReaderSpec) -> String { - let topics = reader.topics.isEmpty ? "All topics" : reader.topics.joined(separator: ", ") - let sources = "\(reader.sources.count) source\(reader.sources.count == 1 ? "" : "s")" - return "\(topics) · \(sources) · \(reader.mode.displayName) · \(reader.schedule.displayName)" - } -} diff --git a/ui/ui/Plugins/PluginCreationController.swift b/ui/ui/Plugins/PluginCreationController.swift index 8888d399..c4228f2a 100644 --- a/ui/ui/Plugins/PluginCreationController.swift +++ b/ui/ui/Plugins/PluginCreationController.swift @@ -2,21 +2,24 @@ import Combine import DBRepository import Foundation import Structure +import SwiftUI @MainActor final class PluginCreationController: ObservableObject { enum Phase: Equatable { case intro - case chooseType - case chooseVendor - case chooseName - case chooseNews + case goal + case skill + case preview case discoveringAuth case creating case collectCredentials(pluginID: String) case failed(step: PluginFactoryCreateInput.FailureStep, message: String, technicalDetail: String? = nil) - case succeeded(pluginID: String) - case succeededNews(readerID: String) + case succeeded(pluginID: String, outcome: SuccessOutcome) + } + + enum SuccessOutcome: Equatable { + case plugin } struct ProgressStepState: Identifiable, Equatable { @@ -35,22 +38,7 @@ final class PluginCreationController: ObservableObject { @Published private(set) var phase: Phase = .intro @Published private(set) var statusMessage = "" @Published private(set) var progressSteps: [ProgressStepState] = [] - @Published var selectedType: PluginFactoryCreateInput.PluginType = .connector - @Published var selectedVendor: PluginFactoryCreateInput.ConnectorVendor = .slack - @Published var selectedScope: PluginFactoryCreateInput.ConnectorScope = .fullSync - @Published var customVendorName = "" - @Published var connectorName = "" - @Published private(set) var defaultNameReady = false - @Published var newsName = "" - @Published var selectedNewsTopics: Set = [] - @Published var extraNewsTopics: [String] = [] - @Published var newsTopicDraft = "" - @Published var selectedNewsSources: Set = [] - @Published var extraNewsURLs: [String] = [] - @Published var newsURLDraft = "" - @Published var newsMode: NewsReaderMode = .list - @Published var newsMaxCount = 20 - @Published var newsSchedule: NewsReaderSchedule = .off + @Published var skillDraft = PluginSkillDraft() @Published private(set) var credentialFields: [PluginCredentialFieldPresentation] = [] @Published var credentialDrafts: [String: String] = [:] @@ -59,8 +47,6 @@ final class PluginCreationController: ObservableObject { private var pollAfterSeq = 0 private var pollTask: Task? private var discoverTask: Task? - private var namePrepareTask: Task? - private var generatedConnectorName = "" private var pendingAuth: ConnectorAuthDiscovery? private var creationAPIKey: String? private var creationReviewerModelJSON: String? @@ -69,27 +55,27 @@ final class PluginCreationController: ObservableObject { deinit { pollTask?.cancel() discoverTask?.cancel() - namePrepareTask?.cancel() } - var canConfirmName: Bool { - guard defaultNameReady else { return false } - let trimmed = connectorName.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return false } - return (try? PluginID.normalized(trimmed)) != nil + var canContinueFromGoal: Bool { + !skillDraft.goal.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } - var canConfirmVendor: Bool { - guard selectedVendor.isSelectableInWizard else { return false } - if selectedVendor == .custom { - return !customVendorName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - } - return true + var canContinueFromSkill: Bool { + guard canConfirmPluginName else { return false } + guard !skillDraft.purpose.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return false } + guard !skillDraft.examples.isEmpty else { return false } + guard skillDraft.examples.allSatisfy({ + !$0.userSays.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + && !$0.pluginDoes.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + }) else { return false } + return skillDraft.buildBlockedReason == nil } - var canConfirmNews: Bool { - let nameOK = !newsName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - return nameOK && !builtNewsSources().isEmpty + var canConfirmPluginName: Bool { + let trimmed = skillDraft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return false } + return (try? PluginID.normalized(trimmed)) != nil } var canSaveCredentials: Bool { @@ -104,281 +90,121 @@ final class PluginCreationController: ObservableObject { self.repository = repository } + func skillDraftBinding(_ keyPath: WritableKeyPath) -> Binding { + Binding( + get: { self.skillDraft[keyPath: keyPath] }, + set: { newValue in + var draft = self.skillDraft + draft[keyPath: keyPath] = newValue + self.skillDraft = draft + } + ) + } + func showIntro() { cancelPolling() discoverTask?.cancel() - namePrepareTask?.cancel() pendingAuth = nil phase = .intro statusMessage = "" progressSteps = [] credentialFields = [] credentialDrafts = [:] - selectedScope = .fullSync - connectorName = "" - generatedConnectorName = "" - defaultNameReady = false + skillDraft = PluginSkillDraft() } func beginCreate() { cancelPolling() - phase = .chooseType + phase = .goal statusMessage = "" progressSteps = [] } - func selectType(_ type: PluginFactoryCreateInput.PluginType) { - selectedType = type - } - - func confirmTypeSelection() { - if selectedType == .newsReader { - resetNewsDraft() - phase = .chooseNews - return - } - guard selectedType == .connector else { - phase = .failed( - step: .type, - message: "Custom plugins are not available yet." + func continueFromGoal() { + guard canContinueFromGoal else { return } + Task { @MainActor in + await PluginFactoryListStore.shared.reload() + var draft = skillDraft + PluginSkillDraftPlanner.applyGoal( + draft.goal, + to: &draft, + existingPluginIDs: PluginFactoryListStore.shared.pluginIDs ) - return + skillDraft = draft + phase = .skill } - selectedVendor = .slack - refreshDefaultConnectorName() - phase = .chooseVendor } - func confirmVendor( - sessionID: String, - helperAPIKey: String?, - helperReviewerModelJSON: String? - ) { - selectedScope = .fullSync - creationSessionID = sessionID - creationAPIKey = helperAPIKey - creationReviewerModelJSON = helperReviewerModelJSON - defaultNameReady = false - phase = .chooseName - namePrepareTask?.cancel() - namePrepareTask = Task { @MainActor in - await PluginFactoryListStore.shared.reload() - guard !Task.isCancelled else { return } - refreshDefaultConnectorName() - defaultNameReady = true - startAuthDiscovery() - } + func continueToPreview() { + guard canContinueFromSkill else { return } + phase = .preview } - func confirmConnectorName() { - guard canConfirmName else { return } - if let pendingAuth { - presentCredentialsOrFail(auth: pendingAuth) - } else { - phase = .discoveringAuth - statusMessage = "Reading how this service authenticates…" - } - } - - func goBackToTypeSelection() { - namePrepareTask?.cancel() + func goBackToGoal() { discoverTask?.cancel() pendingAuth = nil - defaultNameReady = false - phase = .chooseType + phase = .goal } - func goBackToVendor() { - namePrepareTask?.cancel() + func goBackToSkill() { discoverTask?.cancel() pendingAuth = nil - defaultNameReady = false - phase = .chooseVendor + phase = .skill } - func addNewsTopic() { - let topic = newsTopicDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !topic.isEmpty else { return } - if !extraNewsTopics.contains(where: { $0.compare(topic, options: .caseInsensitive) == .orderedSame }) { - extraNewsTopics.append(topic) + func addExample() { + mutateSkillDraft { + $0.examples.append(PluginSkillDraft.Example(userSays: "", pluginDoes: "")) } - newsTopicDraft = "" - } - - func removeNewsTopic(_ topic: String) { - extraNewsTopics.removeAll { $0 == topic } - } - - func addNewsURL() { - let url = newsURLDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !url.isEmpty else { return } - extraNewsURLs.append(url) - newsURLDraft = "" } - func removeNewsURL(_ url: String) { - extraNewsURLs.removeAll { $0 == url } + func removeExample(id: String) { + mutateSkillDraft { $0.examples.removeAll { $0.id == id } } } - func startNewsCreation() { - let spec = NewsReaderSpec( - name: newsName, - topics: builtNewsTopics(), - sources: builtNewsSources(), - mode: newsMode, - maxCount: newsMaxCount, - schedule: newsSchedule - ) - if let blocked = spec.sources.compactMap({ source -> (NewsSource, String)? in - guard let url = URL(string: source.url), - let reason = NewsPaywall.preflightRejection(url: url) else { return nil } - return (source, reason) - }).first { - phase = .failed( - step: .news, - message: NewsReaderError.paywalled(url: blocked.0.url, detail: blocked.1).errorDescription - ?? "This source is behind a paywall, which is not supported yet." - ) - return - } - phase = .creating - statusMessage = "Checking sources…" - progressSteps = [ - ProgressStepState(id: "sources", title: "Check sources for paywalls", status: .active), - ProgressStepState(id: "fetch", title: "Fetch articles with source links", status: .pending), - ] - pollTask?.cancel() - pollTask = Task { @MainActor in - do { - let saved = try await NewsReaderStore.shared.create(spec) - setProgressStep("sources", status: .completed) - setProgressStep("fetch", status: .completed) - phase = .succeededNews(readerID: saved.id) - } catch let error as NewsReaderError { - setProgressStep("sources", status: .failed) - phase = .failed( - step: .news, - message: error.localizedDescription, - technicalDetail: String(describing: error) - ) - } catch { - setProgressStep("sources", status: .failed) - phase = .failed( - step: .news, - message: error.localizedDescription - ) - } + func updateExample(id: String, userSays: String? = nil, pluginDoes: String? = nil) { + mutateSkillDraft { draft in + guard let index = draft.examples.firstIndex(where: { $0.id == id }) else { return } + if let userSays { draft.examples[index].userSays = userSays } + if let pluginDoes { draft.examples[index].pluginDoes = pluginDoes } } } - func builtNewsTopics() -> [String] { - selectedNewsTopics.map(\.displayName) + extraNewsTopics - } - - func builtNewsSources() -> [NewsSource] { - var sources = selectedNewsSources.map(\.source) - for raw in extraNewsURLs + [newsURLDraft] { - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - let normalized = NewsSourceURL.canonicalFetchURL( - URL(string: trimmed.contains("://") ? trimmed : "https://\(trimmed)") - ?? URL(string: "https://news.google.com/rss")! - ).absoluteString - sources.append(NewsSource(label: hostLabel(normalized), url: normalized)) + func toggleTrigger(_ trigger: PluginSkillDraft.Trigger) { + mutateSkillDraft { draft in + guard draft.isTriggerAvailable(trigger) else { return } + if draft.triggers.contains(trigger) { + draft.triggers.remove(trigger) + } else { + draft.triggers.insert(trigger) + } } - var seen = Set() - return sources.filter { seen.insert($0.url).inserted } - } - - private func resetNewsDraft() { - newsName = "" - selectedNewsTopics = [] - extraNewsTopics = [] - newsTopicDraft = "" - selectedNewsSources = [] - extraNewsURLs = [] - newsURLDraft = "" - newsMode = .list - newsMaxCount = 20 - newsSchedule = .off } - private func hostLabel(_ urlString: String) -> String { - URL(string: urlString)?.host ?? urlString + private func mutateSkillDraft(_ transform: (inout PluginSkillDraft) -> Void) { + var draft = skillDraft + transform(&draft) + skillDraft = draft } - func startCreation( + func confirmPreview( sessionID: String, helperAPIKey: String?, helperReviewerModelJSON: String? ) { - guard selectedVendor.isSelectableInWizard else { - phase = .failed( - step: .vendor, - message: "Only Slack connectors can be created right now." - ) - return - } - guard let helperAPIKey, !helperAPIKey.isEmpty else { - phase = .failed( - step: .vendor, - message: "Add an API key in Settings before creating a plugin." - ) - return - } - guard let pluginID = normalizedConnectorName(), - let auth = pendingAuth - else { - phase = .failed( - step: .name, - message: "Name this connector and save its credentials before creating it." - ) - return - } - guard auth.authScheme.isSupportedInWizard else { - phase = .failed( - step: .auth, - message: "OAuth connectors are not available yet. Use a bot token or API key." - ) + creationSessionID = sessionID + creationAPIKey = helperAPIKey + creationReviewerModelJSON = helperReviewerModelJSON + + if skillDraft.plannedKind == .messagingConnector { + phase = .discoveringAuth + statusMessage = "Reading how this service authenticates…" + resetProgressSteps() + startAuthDiscovery() return } - selectedScope = .fullSync - cancelPolling() - phase = .creating - statusMessage = "Starting connector creation…" - resetProgressSteps() - pollAfterSeq = 0 - - let input = PluginFactoryCreateInput.makeConnector( - vendor: selectedVendor, - pluginID: pluginID, - auth: auth, - customVendorName: selectedVendor == .custom ? customVendorName : nil, - scope: selectedScope, - userDescription: "" - ) - - pollTask = Task { @MainActor in - do { - let inputJSON = try input.encodedJSON() - let handle = try await WorkflowRuntimeClient.shared.startWorkflow( - WorkflowStartRequest( - kind: .pluginFactoryCreate, - sessionID: sessionID, - agentID: "ui", - inputJSON: inputJSON, - principal: .agent(sessionID: sessionID, agentID: "ui"), - helperAPIKey: helperAPIKey, - helperReviewerModelJSON: helperReviewerModelJSON - ) - ) - workflowID = handle.workflowID - await pollUntilTerminal() - } catch { - phase = .failed(step: .creating, message: error.localizedDescription) - } - } + startFactoryCreation() } func saveCredentialsAndFinish() { @@ -390,14 +216,10 @@ final class PluginCreationController: ObservableObject { drafts: credentialDrafts ) markProgressCompleted("credentials") - startCreation( - sessionID: creationSessionID, - helperAPIKey: creationAPIKey, - helperReviewerModelJSON: creationReviewerModelJSON - ) + startFactoryCreation() } catch { phase = .failed( - step: .auth, + step: .credentials, message: "Could not save credentials: \(error.localizedDescription)" ) } @@ -407,13 +229,11 @@ final class PluginCreationController: ObservableObject { switch phase { case .failed(let step, _, _): switch step { - case .type: phase = .chooseType - case .news: phase = .chooseNews - case .name: phase = .chooseName - case .auth: phase = .chooseName - case .vendor, .description, .creating: - selectedVendor = .slack - phase = .chooseVendor + case .goal: phase = .goal + case .skill: phase = .skill + case .preview: phase = .preview + case .credentials: phase = .preview + case .build: phase = .preview } default: phase = .intro @@ -424,13 +244,59 @@ final class PluginCreationController: ObservableObject { showIntro() } + private func startFactoryCreation() { + guard let creationAPIKey, !creationAPIKey.isEmpty else { + phase = .failed( + step: .build, + message: "Add an API key in Settings before creating a plugin." + ) + return + } + do { + let input = try PluginFactoryCreateInput.makeFromSkillDraft(skillDraft, auth: pendingAuth) + cancelPolling() + phase = .creating + statusMessage = "Building your plugin…" + resetProgressSteps() + pollAfterSeq = 0 + + pollTask = Task { @MainActor in + do { + let inputJSON = try input.encodedJSON() + let handle = try await WorkflowRuntimeClient.shared.startWorkflow( + WorkflowStartRequest( + kind: .pluginFactoryCreate, + sessionID: creationSessionID, + agentID: "ui", + inputJSON: inputJSON, + principal: .agent(sessionID: creationSessionID, agentID: "ui"), + helperAPIKey: creationAPIKey, + helperReviewerModelJSON: creationReviewerModelJSON + ) + ) + workflowID = handle.workflowID + await pollUntilTerminal() + } catch { + phase = .failed(step: .build, message: error.localizedDescription) + } + } + } catch { + phase = .failed(step: .skill, message: error.localizedDescription) + } + } + private func resetProgressSteps() { progressSteps = [ - ProgressStepState(id: "docs", title: "Read vendor API docs", status: .pending), - ProgressStepState(id: "factory", title: "Build and test plugin", status: .pending), + ProgressStepState(id: "skill", title: "Write SKILL.md", status: .completed), + ProgressStepState(id: "docs", title: "Read API docs", status: .pending), + ProgressStepState(id: "factory", title: "Build guest program", status: .pending), ProgressStepState(id: "review", title: "Safety review", status: .pending), - ProgressStepState(id: "credentials", title: "Save credentials to Keychain", status: .pending), + ProgressStepState(id: "trial", title: "Trial run", status: .pending), + ProgressStepState(id: "credentials", title: "Save credentials", status: .pending), ] + if skillDraft.plannedKind == .customCapability { + setProgressStep("docs", status: .completed) + } } private func setProgressStep(_ id: String, status: ProgressStepState.Status) { @@ -450,16 +316,15 @@ final class PluginCreationController: ObservableObject { switch stage?.lowercased() { case "crawl", "docs": setProgressStep("docs", status: .failed) - case "factory", "build", "review": + case "factory", "build": markProgressCompleted("docs") setProgressStep("factory", status: .failed) + case "review": + markProgressCompleted("docs") + markProgressCompleted("factory") setProgressStep("review", status: .failed) default: - if progressSteps.first(where: { $0.id == "docs" })?.status == .completed { - setProgressStep("factory", status: .failed) - } else { - setProgressStep("docs", status: .failed) - } + setProgressStep("factory", status: .failed) } } @@ -476,6 +341,7 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("factory") markProgressCompleted("review") + markProgressCompleted("trial") default: break } @@ -488,10 +354,11 @@ final class PluginCreationController: ObservableObject { if message.contains("review decision=approved") { markProgressCompleted("factory") markProgressCompleted("review") + markProgressActive("trial") } if message.contains("review decision=rejected") || message.contains("review rejected=") { markProgressCompleted("factory") - markProgressActive("review") + setProgressStep("review", status: .failed) } default: break @@ -520,30 +387,27 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("factory") markProgressCompleted("review") + markProgressCompleted("trial") await PluginFactoryListStore.shared.reload() if let pluginID = parseSuccessPluginID(result.resultJSON) { markProgressCompleted("credentials") - phase = .succeeded(pluginID: pluginID) + phase = .succeeded(pluginID: pluginID, outcome: .plugin) + } else if let saved = PluginFactoryListStore.shared.releases.first { + markProgressCompleted("credentials") + phase = .succeeded(pluginID: saved.pluginID, outcome: .plugin) } else { - if let saved = PluginFactoryListStore.shared.releases.first { - markProgressCompleted("credentials") - phase = .succeeded(pluginID: saved.pluginID) - } else { - phase = .failed( - step: .creating, - message: """ - The connector was not saved. Creation reported success but no plugin release was found. - """, - technicalDetail: result.resultJSON - ) - } + phase = .failed( + step: .build, + message: "The plugin was not saved. Creation reported success but no release was found.", + technicalDetail: result.resultJSON + ) } cancelPolling() return case .failed: let stage = result.events.last(where: { $0.kind == "log" })?.stage markProgressFailed(fromStage: stage) - let raw = result.errorMessage ?? "Connector creation failed." + let raw = result.errorMessage ?? "Plugin creation failed." let presentation = PluginFactoryCreateFailureMessage.presentation(raw) phase = .failed( step: PluginFactoryCreateInput.failureStep(forStage: stage), @@ -555,8 +419,8 @@ final class PluginCreationController: ObservableObject { case .cancelled: markProgressFailed(fromStage: "factory") phase = .failed( - step: .creating, - message: "The connector was not saved. Creation was cancelled before it finished." + step: .build, + message: "Plugin creation was cancelled before it finished." ) cancelPolling() return @@ -564,7 +428,7 @@ final class PluginCreationController: ObservableObject { break } } catch { - phase = .failed(step: .creating, message: error.localizedDescription) + phase = .failed(step: .build, message: error.localizedDescription) cancelPolling() return } @@ -581,29 +445,13 @@ final class PluginCreationController: ObservableObject { return result.pluginID } - private func refreshDefaultConnectorName() { - let existing = PluginFactoryListStore.shared.pluginIDs - let generated = ConnectorPluginNaming.defaultPluginID( - vendor: selectedVendor, - existingIDs: existing - ) - if connectorName.isEmpty - || ConnectorPluginNaming.isGeneratedDefault(pluginID: connectorName, vendor: selectedVendor) - || connectorName == generatedConnectorName { - connectorName = generated - } - generatedConnectorName = generated - } - - private func normalizedConnectorName() -> String? { - let trimmed = connectorName.trimmingCharacters(in: .whitespacesAndNewlines) - return try? PluginID.normalized(trimmed).rawValue - } - private func startAuthDiscovery() { discoverTask?.cancel() pendingAuth = nil - let vendor = selectedVendor + guard let vendor = skillDraft.inferredConnectorVendor else { + phase = .failed(step: .skill, message: "Could not determine which messaging service this plugin targets.") + return + } let sessionID = creationSessionID let apiKey = creationAPIKey let reviewerJSON = creationReviewerModelJSON @@ -674,20 +522,20 @@ final class PluginCreationController: ObservableObject { private func presentCredentialsOrFail(auth: ConnectorAuthDiscovery) { guard auth.authScheme.isSupportedInWizard else { phase = .failed( - step: .auth, + step: .credentials, message: "OAuth connectors are not available yet. Use a bot token or API key." ) return } - guard let pluginID = normalizedConnectorName() else { - phase = .chooseName + guard let pluginID = try? skillDraft.normalizedPluginID() else { + phase = .skill return } let descriptors = auth.secrets.map(\.descriptor) guard !descriptors.isEmpty else { phase = .failed( - step: .auth, - message: "Could not determine which credentials this connector needs." + step: .credentials, + message: "Could not determine which credentials this plugin needs." ) return } @@ -704,7 +552,7 @@ final class PluginCreationController: ObservableObject { let env = PluginSecretDevelopmentSource.resolve(pluginID: pluginID, fieldID: field.id) ?? "" return (field.id, env) }) - statusMessage = auth.setupHint ?? "Enter the credentials this connector needs. They are stored in Keychain on your Mac." + statusMessage = auth.setupHint ?? "Enter the credentials this plugin needs. They are stored in Keychain on your Mac." phase = .collectCredentials(pluginID: pluginID) } diff --git a/ui/ui/Plugins/PluginsWorkspaceView.swift b/ui/ui/Plugins/PluginsWorkspaceView.swift index 5034315c..a3e8a3b3 100644 --- a/ui/ui/Plugins/PluginsWorkspaceView.swift +++ b/ui/ui/Plugins/PluginsWorkspaceView.swift @@ -8,7 +8,6 @@ struct PluginsWorkspaceView: View { let helperReviewerModelJSON: String? let sessionID: String let onOpenMessagingConnector: (String) -> Void - var onOpenNewsReader: (String) -> Void = { _ in } var body: some View { ZStack { @@ -33,8 +32,8 @@ struct PluginsWorkspaceView: View { isPresented: true, minWidth: 400, minHeight: 0, - maxWidth: 520, - maxHeight: controller.phase == .chooseNews ? 720 : 560, + maxWidth: 560, + maxHeight: modalMaxHeight, onBackdropDismiss: canDismiss ? { controller.showIntro() } : nil, onEscape: canDismiss ? { controller.showIntro() } : nil, header: { @@ -57,6 +56,13 @@ struct PluginsWorkspaceView: View { ) } + private var modalMaxHeight: CGFloat { + switch controller.phase { + case .skill, .preview: return 720 + default: return 560 + } + } + private var canDismiss: Bool { switch controller.phase { case .creating, .discoveringAuth: return false @@ -67,119 +73,59 @@ struct PluginsWorkspaceView: View { private var modalTitle: String { switch controller.phase { case .intro: return "Create a plugin" - case .chooseType: return "Create a plugin" - case .chooseVendor: return "Choose a vendor" - case .chooseName: return "Name this connector" - case .chooseNews: return "News list" + case .goal: return "What should it do?" + case .skill: return "Define the skill" + case .preview: return "Preview" case .discoveringAuth: return "Reading authentication docs" - case .creating: return controller.selectedType == .newsReader ? "Creating news list" : "Creating connector" - case .collectCredentials: return "Connector credentials" - case .failed: return controller.selectedType == .newsReader ? "Could not create news list" : "Could not create connector" - case .succeeded: return "Connector ready" - case .succeededNews: return "News list ready" + case .creating: return creatingTitle + case .collectCredentials: return "Plugin credentials" + case .failed: return failureTitle + case .succeeded(_, let outcome): + return "Plugin ready" } } + private var creatingTitle: String { + switch controller.skillDraft.plannedKind { + case .messagingConnector: return "Creating connector" + case .customCapability: return "Building plugin" + } + } + + private var failureTitle: String { + return "Could not create plugin" + } + @ViewBuilder private var modalBody: some View { switch controller.phase { case .intro: Text(""" - A plugin is a small program that extends the capabilities of Derrick. This form will guide you through the process of building your own unique and secure plugins. + Describe what you want Derrick to do. Derrick will draft a skill, show you a preview, and build a secure plugin package. """) .font(.body) .fixedSize(horizontal: false, vertical: true) - case .chooseType: + case .goal: VStack(alignment: .leading, spacing: 10) { - Text("What kind of plugin do you want?") + Text("What do you want Derrick to do?") .font(.subheadline) .foregroundStyle(.secondary) - typeButton( - title: "Connector", - subtitle: "Messaging integration (Slack)", - type: .connector, - enabled: true - ) - typeButton( - title: "News reader", - subtitle: "Saved lists from topics and sources", - type: .newsReader, - enabled: true - ) - typeButton( - title: "Custom", - subtitle: "Coming soon", - type: .custom, - enabled: false + TextField( + "e.g. Send Slack messages from Messaging, or fetch tech headlines", + text: controller.skillDraftBinding(\.goal), + axis: .vertical ) + .textFieldStyle(.roundedBorder) + .lineLimit(3...6) + .accessibilityIdentifier("plugin-goal-field") } - case .chooseVendor: - VStack(alignment: .leading, spacing: 10) { - Text("Which service should this connector use?") - .font(.subheadline) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 120), spacing: 8)], spacing: 8) { - ForEach(PluginFactoryCreateInput.ConnectorVendor.allCases, id: \.self) { vendor in - let enabled = vendor.isSelectableInWizard - Button { - guard enabled else { return } - controller.selectedVendor = vendor - } label: { - VStack(spacing: 4) { - Text(vendor.displayName) - .font(.subheadline.weight(.medium)) - if !enabled { - Text("Soon") - .font(.caption2) - .foregroundStyle(.secondary) - } - } - .frame(maxWidth: .infinity) - .padding(.vertical, 10) - .background( - enabled && controller.selectedVendor == vendor - ? Color.accentColor.opacity(0.15) - : Color.primary.opacity(enabled ? 0.05 : 0.03) - ) - .foregroundStyle(enabled ? .primary : .secondary) - .clipShape(RoundedRectangle(cornerRadius: 10)) - } - .buttonStyle(.plain) - .disabled(!enabled) - .accessibilityLabel(enabled ? vendor.displayName : "\(vendor.displayName), coming soon") - } - } - if controller.selectedVendor == .custom { - TextField("Vendor name", text: $controller.customVendorName) - .textFieldStyle(.roundedBorder) - } - Text("This connector lists conversations as tabs, including reply threads, then sends and receives new messages.") - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - .padding(.top, 4) - if controller.selectedVendor == .slack { - Text(ConnectorReplyThreadAccessMessage.slackSetupHint) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - } + case .skill: + skillBuilderForm - case .chooseName: - VStack(alignment: .leading, spacing: 10) { - Text("Give this connector a name. You can change the default.") - .font(.subheadline) - .foregroundStyle(.secondary) - TextField("Connector name", text: $controller.connectorName) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("connector-plugin-name") - } - - case .chooseNews: - newsReaderForm + case .preview: + previewForm case .discoveringAuth, .creating: VStack(alignment: .leading, spacing: 14) { @@ -198,7 +144,7 @@ struct PluginsWorkspaceView: View { .font(.subheadline) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) - if controller.selectedVendor == .slack { + if controller.skillDraft.inferredConnectorVendor == .slack { Text(ConnectorReplyThreadAccessMessage.slackSetupHint) .font(.caption) .foregroundStyle(.secondary) @@ -210,58 +156,10 @@ struct PluginsWorkspaceView: View { } case .failed(_, let message, let technicalDetail): - VStack(alignment: .leading, spacing: 10) { - if controller.selectedType == .newsReader { - Label( - message.localizedCaseInsensitiveContains("paywall") - ? "Blocked because of a paywall" - : "News list was not created", - systemImage: "exclamationmark.triangle.fill" - ) - .font(.subheadline.weight(.semibold)) - .foregroundStyle(message.localizedCaseInsensitiveContains("paywall") ? Color.orange : Color.secondary) - .accessibilityIdentifier( - message.localizedCaseInsensitiveContains("paywall") - ? "news-paywall-blocked" - : "news-create-failed" - ) - if message.localizedCaseInsensitiveContains("paywall") { - Text(NewsPaywall.userWarning) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - } else { - Label("Nothing was installed", systemImage: "minus.circle") - .font(.subheadline.weight(.semibold)) - .foregroundStyle(.secondary) - Text("Your sidebar and Messaging are unchanged.") - .font(.caption) - .foregroundStyle(.secondary) - } - Text(message) - .font(.body) - .fixedSize(horizontal: false, vertical: true) - .padding(.top, 4) - if let technicalDetail, !technicalDetail.isEmpty { - DisclosureGroup("Technical details") { - Text(technicalDetail) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - .textSelection(.enabled) - } - } - } + failureBody(message: message, technicalDetail: technicalDetail) - case .succeeded(let pluginID): - Text("Your connector /\(pluginID) is ready. Open it to start talking in Messaging.") - .font(.body) - .fixedSize(horizontal: false, vertical: true) - case .succeededNews: - Text("Your news list is ready. Every article includes a source link.") - .font(.body) - .fixedSize(horizontal: false, vertical: true) + case .succeeded(let pluginID, let outcome): + successBody(pluginID: pluginID, outcome: outcome) } } @@ -271,62 +169,48 @@ struct PluginsWorkspaceView: View { case .intro: HStack { Spacer() - Button("Create plugin") { controller.beginCreate() } + Button("Begin") { controller.beginCreate() } .buttonStyle(ModalPrimaryButtonStyle()) .keyboardShortcut(.defaultAction) } - case .chooseType: + case .goal: HStack { Button("Back") { controller.showIntro() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Continue") { controller.confirmTypeSelection() } + Button("Continue") { controller.continueFromGoal() } .buttonStyle(ModalPrimaryButtonStyle()) - .disabled( - controller.selectedType != .connector - && controller.selectedType != .newsReader - ) + .disabled(!controller.canContinueFromGoal) .keyboardShortcut(.defaultAction) } - case .chooseVendor: - HStack { - Button("Back") { controller.goBackToTypeSelection() } - .buttonStyle(ModalSecondaryButtonStyle()) - Spacer() - Button("Continue") { - controller.confirmVendor( - sessionID: sessionID, - helperAPIKey: helperAPIKey, - helperReviewerModelJSON: helperReviewerModelJSON - ) - } - .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!sessionReady || !controller.canConfirmVendor) - .keyboardShortcut(.defaultAction) - } - - case .chooseName: + case .skill: HStack { - Button("Back") { controller.goBackToVendor() } + Button("Back") { controller.goBackToGoal() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Continue") { controller.confirmConnectorName() } + Button("Preview") { controller.continueToPreview() } .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!controller.canConfirmName) + .disabled(!controller.canContinueFromSkill) .keyboardShortcut(.defaultAction) } - case .chooseNews: + case .preview: HStack { - Button("Back") { controller.goBackToTypeSelection() } + Button("Back") { controller.goBackToSkill() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Create") { controller.startNewsCreation() } - .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!controller.canConfirmNews) - .keyboardShortcut(.defaultAction) + Button(buildButtonTitle) { + controller.confirmPreview( + sessionID: sessionID, + helperAPIKey: helperAPIKey, + helperReviewerModelJSON: helperReviewerModelJSON + ) + } + .buttonStyle(ModalPrimaryButtonStyle()) + .disabled(!sessionReady || !controller.canContinueFromSkill) + .keyboardShortcut(.defaultAction) } case .discoveringAuth, .creating: @@ -352,28 +236,299 @@ struct PluginsWorkspaceView: View { .buttonStyle(ModalSecondaryButtonStyle()) } - case .succeeded(let pluginID): + case .succeeded(let pluginID, let outcome): HStack { - Button("Done") { controller.dismissSuccess() } - .buttonStyle(ModalSecondaryButtonStyle()) + if outcome != .plugin || controller.skillDraft.plannedKind == .messagingConnector { + Button("Done") { controller.dismissSuccess() } + .buttonStyle(ModalSecondaryButtonStyle()) + } Spacer() - Button("Open connector") { - onOpenMessagingConnector(pluginID) + switch outcome { + case .plugin where controller.skillDraft.plannedKind == .messagingConnector: + Button("Open connector") { + onOpenMessagingConnector(pluginID) + } + .buttonStyle(ModalPrimaryButtonStyle()) + .keyboardShortcut(.defaultAction) + case .plugin: + Button("Done") { controller.dismissSuccess() } + .buttonStyle(ModalPrimaryButtonStyle()) + .keyboardShortcut(.defaultAction) } - .buttonStyle(ModalPrimaryButtonStyle()) - .keyboardShortcut(.defaultAction) } - case .succeededNews(let readerID): + } + } + + private var buildButtonTitle: String { + "Build plugin" + } + + private var skillBuilderForm: some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + plannedKindBadge + + VStack(alignment: .leading, spacing: 6) { + Text("Purpose") + .font(.caption) + .foregroundStyle(.secondary) + TextField("What this plugin does", text: controller.skillDraftBinding(\.purpose), axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(2...4) + } + + VStack(alignment: .leading, spacing: 6) { + Text("When to use") + .font(.caption) + .foregroundStyle(.secondary) + FlowLayout(spacing: 8) { + ForEach(PluginSkillDraft.Trigger.allCases, id: \.self) { trigger in + triggerChip(trigger) + } + } + } + + examplesSection + + nameFieldSection + + if let blocked = controller.skillDraft.buildBlockedReason { + Text(blocked) + .font(.caption) + .foregroundStyle(.orange) + .fixedSize(horizontal: false, vertical: true) + } + } + } + } + + private var plannedKindBadge: some View { + let (label, icon) = plannedKindPresentation + return Label(label, systemImage: icon) + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(Color.primary.opacity(0.05), in: Capsule()) + } + + private var plannedKindPresentation: (String, String) { + switch controller.skillDraft.plannedKind { + case .messagingConnector: + let vendor = controller.skillDraft.inferredConnectorVendor?.displayName ?? "Messaging" + return ("\(vendor) connector", "bubble.left.and.bubble.right") + case .customCapability: + return ("Custom capability", "wand.and.stars") + } + } + + private var examplesSection: some View { + VStack(alignment: .leading, spacing: 8) { HStack { - Button("Done") { controller.dismissSuccess() } - .buttonStyle(ModalSecondaryButtonStyle()) + Text("Examples") + .font(.caption) + .foregroundStyle(.secondary) Spacer() - Button("Open news list") { - onOpenNewsReader(readerID) + Button("Add example") { controller.addExample() } + .font(.caption) + } + ForEach(controller.skillDraft.examples) { example in + VStack(alignment: .leading, spacing: 6) { + TextField("You say…", text: exampleBinding(example.id, field: .userSays)) + .textFieldStyle(.roundedBorder) + TextField("Plugin does…", text: exampleBinding(example.id, field: .pluginDoes)) + .textFieldStyle(.roundedBorder) + if controller.skillDraft.examples.count > 1 { + Button("Remove") { controller.removeExample(id: example.id) } + .font(.caption) + } + } + .padding(10) + .background(Color.primary.opacity(0.04), in: RoundedRectangle(cornerRadius: 8)) + } + } + } + + private enum ExampleField { case userSays, pluginDoes } + + private func exampleBinding(_ id: String, field: ExampleField) -> Binding { + Binding( + get: { + guard let index = controller.skillDraft.examples.firstIndex(where: { $0.id == id }) else { + return "" + } + switch field { + case .userSays: return controller.skillDraft.examples[index].userSays + case .pluginDoes: return controller.skillDraft.examples[index].pluginDoes + } + }, + set: { newValue in + switch field { + case .userSays: controller.updateExample(id: id, userSays: newValue) + case .pluginDoes: controller.updateExample(id: id, pluginDoes: newValue) + } + } + ) + } + + private func triggerChip(_ trigger: PluginSkillDraft.Trigger) -> some View { + let available = controller.skillDraft.isTriggerAvailable(trigger) + let on = available && controller.skillDraft.triggers.contains(trigger) + return Button { + controller.toggleTrigger(trigger) + } label: { + Text(trigger.label) + .font(.caption.weight(.medium)) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(chipBackground(on: on, available: available)) + .foregroundStyle(available ? .primary : .tertiary) + .clipShape(Capsule()) + } + .buttonStyle(.plain) + .disabled(!available) + .help(triggerHelp(trigger, available: available)) + } + + private func chipBackground(on: Bool, available: Bool) -> Color { + guard available else { return Color.primary.opacity(0.03) } + return on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05) + } + + private func triggerHelp( + _ trigger: PluginSkillDraft.Trigger, + available: Bool + ) -> String { + guard !available else { return "" } + switch controller.skillDraft.plannedKind { + case .messagingConnector: + switch trigger { + case .schedule: + return "Connectors respond in Messaging, not on a timer." + default: + return "Not available for messaging connectors." + } + case .customCapability: + switch trigger { + case .messaging: + return "Only messaging connectors use the Messaging tab." + default: + return "Not available for this plugin type." + } + } + } + + private var nameFieldSection: some View { + VStack(alignment: .leading, spacing: 6) { + Text("Plugin name") + .font(.caption) + .foregroundStyle(.secondary) + TextField("Plugin name", text: controller.skillDraftBinding(\.pluginName)) + .textFieldStyle(.roundedBorder) + .accessibilityIdentifier("connector-plugin-name") + if controller.canConfirmPluginName { + Text("Invoke this plugin in chat as /\(normalizedPluginID()).") + .font(.caption2) + .foregroundStyle(.tertiary) + } else { + Text("Use letters, numbers, and hyphens (for example tech-news).") + .font(.caption2) + .foregroundStyle(.orange) + } + } + } + + private func normalizedPluginID() -> String { + let trimmed = controller.skillDraft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + if let normalized = try? PluginID.normalized(trimmed) { + return normalized.rawValue + } + return trimmed.isEmpty ? "plugin" : trimmed + } + + private var previewForm: some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + plannedKindBadge + + nameFieldSection + + VStack(alignment: .leading, spacing: 6) { + Text("Scenarios") + .font(.caption) + .foregroundStyle(.secondary) + ForEach(controller.skillDraft.previewScenarios(), id: \.self) { scenario in + Text(scenario) + .font(.subheadline) + .fixedSize(horizontal: false, vertical: true) + } + } + + VStack(alignment: .leading, spacing: 6) { + Text("Package") + .font(.caption) + .foregroundStyle(.secondary) + ForEach(controller.skillDraft.packageOutline(), id: \.self) { line in + Label(line, systemImage: "doc") + .font(.caption) + .foregroundStyle(.secondary) + } + } + + VStack(alignment: .leading, spacing: 6) { + Text("SKILL.md") + .font(.caption) + .foregroundStyle(.secondary) + Text(controller.skillDraft.skillMarkdown()) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.primary.opacity(0.04), in: RoundedRectangle(cornerRadius: 8)) + .textSelection(.enabled) + } + } + } + } + + @ViewBuilder + private func failureBody(message: String, technicalDetail: String?) -> some View { + VStack(alignment: .leading, spacing: 10) { + Label("Nothing was installed", systemImage: "minus.circle") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + Text("Your sidebar and Messaging are unchanged.") + .font(.caption) + .foregroundStyle(.secondary) + Text(message) + .font(.body) + .fixedSize(horizontal: false, vertical: true) + .padding(.top, 4) + if let technicalDetail, !technicalDetail.isEmpty { + DisclosureGroup("Technical details") { + Text(technicalDetail) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) } - .buttonStyle(ModalPrimaryButtonStyle()) - .keyboardShortcut(.defaultAction) + } + } + } + + @ViewBuilder + private func successBody(pluginID: String, outcome: PluginCreationController.SuccessOutcome) -> some View { + switch outcome { + case .plugin: + if controller.skillDraft.plannedKind == .messagingConnector { + Text("Your connector /\(pluginID) is ready. Open it to start talking in Messaging.") + .font(.body) + .fixedSize(horizontal: false, vertical: true) + } else { + Text("Your plugin /\(pluginID) is ready.") + .font(.body) + .fixedSize(horizontal: false, vertical: true) } } } @@ -443,169 +598,47 @@ struct PluginsWorkspaceView: View { set: { controller.credentialDrafts[id] = $0 } ) } +} - private func typeButton( - title: String, - subtitle: String, - type: PluginFactoryCreateInput.PluginType, - enabled: Bool - ) -> some View { - Button { - guard enabled else { return } - controller.selectType(type) - } label: { - HStack(alignment: .top, spacing: 10) { - VStack(alignment: .leading, spacing: 4) { - Text(title) - .font(.body.weight(.semibold)) - .foregroundStyle(enabled ? .primary : .secondary) - Text(subtitle) - .font(.caption) - .foregroundStyle(.secondary) - } - Spacer() - if !enabled { - Text("Soon") - .font(.caption2) - .foregroundStyle(.secondary) - } else if controller.selectedType == type { - Image(systemName: "checkmark.circle.fill") - .foregroundStyle(Color.accentColor) - } - } - .padding(12) - .frame(maxWidth: .infinity, alignment: .leading) - .background( - controller.selectedType == type && enabled - ? Color.accentColor.opacity(0.12) - : Color.primary.opacity(enabled ? 0.05 : 0.03) +/// Simple horizontal flow for trigger chips when `Layout` is unavailable. +private struct FlowLayout: Layout { + var spacing: CGFloat = 8 + + func sizeThatFits(proposal: ProposedViewSize, subviews: Subviews, cache: inout ()) -> CGSize { + let result = arrange(proposal: proposal, subviews: subviews) + return result.size + } + + func placeSubviews(in bounds: CGRect, proposal: ProposedViewSize, subviews: Subviews, cache: inout ()) { + let result = arrange(proposal: proposal, subviews: subviews) + for (index, frame) in result.frames.enumerated() { + subviews[index].place( + at: CGPoint(x: bounds.minX + frame.minX, y: bounds.minY + frame.minY), + proposal: ProposedViewSize(frame.size) ) - .clipShape(RoundedRectangle(cornerRadius: 10)) } - .buttonStyle(.plain) - .disabled(!enabled) } - private var newsReaderForm: some View { - ScrollView { - VStack(alignment: .leading, spacing: 14) { - TextField("Name this list", text: $controller.newsName) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("news-list-name") - Text("Topics") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 110), spacing: 8)], spacing: 8) { - ForEach(NewsPresetTopic.allCases) { topic in - let on = controller.selectedNewsTopics.contains(topic) - Button { - if on { - controller.selectedNewsTopics.remove(topic) - } else { - controller.selectedNewsTopics.insert(topic) - } - } label: { - Text(topic.displayName) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("Add a custom topic", text: $controller.newsTopicDraft) - .textFieldStyle(.roundedBorder) - .onSubmit { controller.addNewsTopic() } - Button("Add") { controller.addNewsTopic() } - } - if !controller.extraNewsTopics.isEmpty { - Text(controller.extraNewsTopics.joined(separator: ", ")) - .font(.caption) - .foregroundStyle(.secondary) - } - Text("Sources") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 130), spacing: 8)], spacing: 8) { - ForEach(NewsPresetSource.allCases) { source in - let on = controller.selectedNewsSources.contains(source) - Button { - if on { - controller.selectedNewsSources.remove(source) - } else { - controller.selectedNewsSources.insert(source) - } - } label: { - Text(source.source.label) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("https://…", text: $controller.newsURLDraft) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("news-url-field") - .onSubmit { controller.addNewsURL() } - Button("Add URL") { controller.addNewsURL() } - } - ForEach(controller.extraNewsURLs, id: \.self) { url in - VStack(alignment: .leading, spacing: 2) { - HStack { - Text(url) - .font(.caption) - .lineLimit(1) - Spacer() - Button("Remove") { controller.removeNewsURL(url) } - .font(.caption) - } - if let reason = newsURLPaywallReason(url) { - Text(reason) - .font(.caption2) - .foregroundStyle(.orange) - } - } - } - HStack(alignment: .top, spacing: 8) { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundStyle(.orange) - Text(NewsPaywall.userWarning) - .fixedSize(horizontal: false, vertical: true) - } - .font(.caption) - .padding(10) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.orange.opacity(0.12), in: RoundedRectangle(cornerRadius: 8)) - .accessibilityElement(children: .combine) - .accessibilityIdentifier("news-paywall-warning") - Picker("Mode", selection: $controller.newsMode) { - ForEach(NewsReaderMode.allCases, id: \.self) { mode in - Text(mode.displayName).tag(mode) - } - } - Stepper("Up to \(controller.newsMaxCount) items", value: $controller.newsMaxCount, in: 5...50, step: 5) - Picker("Schedule", selection: $controller.newsSchedule) { - ForEach(NewsReaderSchedule.allCases, id: \.self) { schedule in - Text(schedule.displayName).tag(schedule) - } - } + private func arrange(proposal: ProposedViewSize, subviews: Subviews) -> (size: CGSize, frames: [CGRect]) { + let maxWidth = proposal.width ?? .infinity + var x: CGFloat = 0 + var y: CGFloat = 0 + var rowHeight: CGFloat = 0 + var frames: [CGRect] = [] + + for subview in subviews { + let size = subview.sizeThatFits(.unspecified) + if x + size.width > maxWidth, x > 0 { + x = 0 + y += rowHeight + spacing + rowHeight = 0 } + frames.append(CGRect(x: x, y: y, width: size.width, height: size.height)) + rowHeight = max(rowHeight, size.height) + x += size.width + spacing } - } - private func newsURLPaywallReason(_ raw: String) -> String? { - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return nil } - let normalized = trimmed.contains("://") ? trimmed : "https://\(trimmed)" - guard let url = URL(string: normalized) else { return nil } - return NewsPaywall.preflightRejection(url: url) + return (CGSize(width: maxWidth, height: y + rowHeight), frames) } } diff --git a/ui/ui/Services/AgentServiceClient.swift b/ui/ui/Services/AgentServiceClient.swift index 339fb52d..73b1afd8 100644 --- a/ui/ui/Services/AgentServiceClient.swift +++ b/ui/ui/Services/AgentServiceClient.swift @@ -74,7 +74,10 @@ public final class AgentServiceClient: @unchecked Sendable { /// Connect (launch-on-demand), bootstrap DB/logs, return health. Retries a few times. /// Use at app startup (and when `ensureReadyForTurn` finds the link dead). - public func ensureUpAndHealth(retries: Int = 3) async throws -> ServiceHealthReport { + public func ensureUpAndHealth( + retries: Int = 3, + verifyHealth: Bool = true + ) async throws -> ServiceHealthReport { var lastError: Error? for attempt in 0.. NSXPCListenerEndpoint? { - bootstrapStatus.update(phase: .checkingDocker, message: "Starting Docker runtime…") + bootstrapStatus.updateTask(.docker, message: "Starting Docker runtime…") _ = XPCDockerRunner.shared - try await XPCDockerRunner.shared.waitUntilPrewarmed() + try await XPCDockerRunner.shared.waitUntilDockerReachable() + bootstrapStatus.completeTask(.docker) do { return try await XPCDockerRunner.shared.fetchPeerListenerEndpoint() } catch { @@ -1489,10 +1480,7 @@ struct ContentView: View { } private func resolveAPIKey() -> String? { - secretResolver.resolve( - account: selectedModel.provider.secretAccount, - environmentKeys: selectedModel.provider.apiKeyEnvironmentKeys - ) + LLMProviderCredentialGate.resolveAPIKey(for: selectedModel.provider, resolver: secretResolver) } @ViewBuilder diff --git a/ui/ui/Views/LLMModelSettingsView.swift b/ui/ui/Views/LLMModelSettingsView.swift index 2321a609..d3209fc9 100644 --- a/ui/ui/Views/LLMModelSettingsView.swift +++ b/ui/ui/Views/LLMModelSettingsView.swift @@ -349,7 +349,7 @@ struct LLMModelSettingsView: View { Text("Multi-agent") .font(.system(size: 26, weight: .semibold, design: .rounded)) - Text("Caps for agents_spawn and worker turns in a chat session. New tabs use saved values; open tabs keep the limits they started with. Parallel script_exec runs wait in line (one Python guest container at a time). Crawls and file conversions have their own lines.") + Text("Caps for agents_spawn and worker turns in a chat session. New tabs use saved values; open tabs keep the limits they started with. Parallel script_exec runs wait in line (one Go guest container at a time). Crawls and file conversions have their own lines.") .font(.subheadline) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) diff --git a/ui/ui/Views/MarkdownView.swift b/ui/ui/Views/MarkdownView.swift index d39b0670..c5b9f787 100644 --- a/ui/ui/Views/MarkdownView.swift +++ b/ui/ui/Views/MarkdownView.swift @@ -2,6 +2,7 @@ import AppKit import SwiftUI enum MarkdownBlock: Identifiable { + case heading(level: Int, text: String) case paragraph(String) case bullet(String) case numbered(number: Int, text: String) @@ -10,6 +11,8 @@ enum MarkdownBlock: Identifiable { var id: String { switch self { + case .heading(let level, let text): + return "h\(level)-\(text.hashValue)" case .paragraph(let text): return "p-\(text.hashValue)" case .bullet(let text): @@ -66,6 +69,22 @@ enum MarkdownBlock: Identifiable { continue } + if trimmed.hasPrefix("#") { + var level = 0 + var index = trimmed.startIndex + while index < trimmed.endIndex, trimmed[index] == "#", level < 6 { + level += 1 + index = trimmed.index(after: index) + } + if level > 0, index < trimmed.endIndex, trimmed[index] == " " { + flushParagraph() + let headingText = String(trimmed[trimmed.index(after: index)...]) + .trimmingCharacters(in: .whitespaces) + blocks.append(.heading(level: level, text: headingText)) + continue + } + } + // Check bullet if trimmed.hasPrefix("- ") || trimmed.hasPrefix("* ") { flushParagraph() @@ -290,6 +309,13 @@ struct MarkdownResponseView: View { @ViewBuilder private func blockView(for block: MarkdownBlock) -> some View { switch block { + case .heading(let level, let text): + Text((try? AttributedString(markdown: text)) ?? AttributedString(text)) + .font(headingFont(level: level)) + .fontWeight(.semibold) + .padding(.horizontal, 2) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) case .paragraph(let text): Text((try? AttributedString(markdown: text)) ?? AttributedString(text)) .lineSpacing(2) @@ -352,6 +378,16 @@ struct MarkdownResponseView: View { } } + private func headingFont(level: Int) -> Font { + switch level { + case 1: return .title + case 2: return .title2 + case 3: return .title3 + case 4: return .headline + default: return .subheadline + } + } + @ViewBuilder private func csvTableView(table: CSVTable, source: String) -> some View { VStack(alignment: .leading, spacing: 10) { diff --git a/ui/ui/Views/PluginHTMLResultExtractor.swift b/ui/ui/Views/PluginHTMLResultExtractor.swift index 77e7d828..cbe041a7 100644 --- a/ui/ui/Views/PluginHTMLResultExtractor.swift +++ b/ui/ui/Views/PluginHTMLResultExtractor.swift @@ -57,6 +57,7 @@ enum PluginResultExtractor { let format = envelope.payload["format"]?.stringValue?.lowercased() let candidate = envelope.payload["content"]?.stringValue ?? envelope.payload["summary"]?.stringValue + ?? envelope.payload["markdown"]?.stringValue ?? envelope.payload["text"]?.stringValue ?? "" guard !candidate.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { @@ -320,6 +321,7 @@ enum PluginResultExtractor { let candidate = item["content"] as? String ?? item["summary"] as? String + ?? item["markdown"] as? String ?? item["text"] as? String ?? "" guard !candidate.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { diff --git a/ui/ui/Views/SidebarView.swift b/ui/ui/Views/SidebarView.swift index 20b340f4..b719f13d 100644 --- a/ui/ui/Views/SidebarView.swift +++ b/ui/ui/Views/SidebarView.swift @@ -10,7 +10,6 @@ struct SidebarView: View { @ObservedObject var modelThinkingSettings: LLMModelThinkingSettings @ObservedObject var chatSessions: ChatSessionStore @ObservedObject var messaging: MessagingStore - @ObservedObject var news: NewsReaderStore @Binding var workspace: AppWorkspace var isDebugEnabled: Bool = false /// Reference type must not be recreated every `View` value; hold via `@State`. @@ -81,17 +80,6 @@ struct SidebarView: View { workspace = .messaging Task { await messaging.syncConnectorsFromFactory() } } - SidebarActionRow( - row: SidebarRow( - id: "news", - icon: "newspaper.fill", - title: "News", - isProminent: workspace == .news - ) - ) { - workspace = .news - Task { await news.reload() } - } if isDebugEnabled { SidebarActionRow( row: SidebarRow( @@ -110,8 +98,6 @@ struct SidebarView: View { pluginsList } else if workspace == .messaging { messagingList - } else if workspace == .news { - newsList } else if workspace == .debugLogs { debugLogsHint } else { @@ -289,46 +275,6 @@ struct SidebarView: View { } } - private var newsList: some View { - VStack(alignment: .leading, spacing: 8) { - HStack { - Text("Saved lists") - .font(.caption) - .foregroundStyle(.secondary) - Spacer() - } - .padding(.top, 4) - ScrollView { - LazyVStack(alignment: .leading, spacing: 10) { - if news.readers.isEmpty { - Text("No news lists yet") - .font(.system(size: sideMenuRecentsFontSize)) - .foregroundStyle(.secondary) - } else { - ForEach(news.readers) { reader in - Button { - workspace = .news - Task { await news.select(id: reader.id) } - } label: { - Text(reader.name) - .font(.system(size: sideMenuRecentsFontSize)) - .lineLimit(1) - .frame(maxWidth: .infinity, alignment: .leading) - .foregroundStyle( - news.selectedReaderID == reader.id - ? Color.primary - : Color.primary.opacity(0.9) - ) - } - .buttonStyle(.plain) - } - } - } - .frame(maxWidth: .infinity, alignment: .leading) - } - } - } - private var pluginsList: some View { VStack(alignment: .leading, spacing: 8) { ScrollView { @@ -439,7 +385,6 @@ struct SidebarView: View { modelThinkingSettings: LLMModelThinkingSettings(repository: repo), chatSessions: store, messaging: MessagingStore(), - news: NewsReaderStore.shared, workspace: .constant(.chats) ) } diff --git a/ui/uiTests/AppBootstrapStatusTests.swift b/ui/uiTests/AppBootstrapStatusTests.swift index 7fd7994e..007d27a1 100644 --- a/ui/uiTests/AppBootstrapStatusTests.swift +++ b/ui/uiTests/AppBootstrapStatusTests.swift @@ -174,6 +174,45 @@ import Testing #expect(status.phase == .ready) } + @MainActor + @Test func loadingSessionDoesNotOverwriteConnectingHelper() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.update(phase: .connectingHelper, message: "Connecting to Derrick daemon…") + status.update(phase: .loadingSession, message: "Opening local database…") + #expect(status.phase == .connectingHelper) + #expect(status.statusMessage == "Connecting to Derrick daemon…") + } + + @MainActor + @Test func parallelLoadingTasksTrackIndependently() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.beginTask(.daemon) + status.beginTask(.database) + status.beginTask(.docker) + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + status.completeTask(.database) + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .docker]) + status.completeTask(.daemon) + status.completeTask(.docker) + #expect(status.activeLoadingTasks.isEmpty) + } + + @MainActor + @Test func phaseUpdateAddsMatchingLoadingTasks() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.update(phase: .connectingHelper, message: "Connecting to Derrick daemon…") + status.update(phase: .loadingSession, message: "Opening local database…") + status.update(phase: .checkingDocker, message: "Checking Docker Desktop…") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + status.update(phase: .preparingImage, message: "Preparing worker image…") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker, .workerImage]) + status.update(phase: .verifyingEnvironment, message: "Worker image ready.") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + } + @MainActor @Test func cancelClearsInProgressModal() { let status = freshStatus() diff --git a/ui/uiTests/MessagingNavigationTests.swift b/ui/uiTests/MessagingNavigationTests.swift index 426c103e..88476aea 100644 --- a/ui/uiTests/MessagingNavigationTests.swift +++ b/ui/uiTests/MessagingNavigationTests.swift @@ -343,9 +343,9 @@ import Testing _ = try await repository.createEmptyDatabaseIfNeeded(username: "ui", password: "ui") let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-bot","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ - let runtimeJSON = #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + let runtimeJSON = #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# let guestSource = "print([])" var release = PluginFactoryRelease( pluginID: "slack-bot", diff --git a/ui/uiTests/PromptResourcesTests.swift b/ui/uiTests/PromptResourcesTests.swift index 66950af9..0c9990a0 100644 --- a/ui/uiTests/PromptResourcesTests.swift +++ b/ui/uiTests/PromptResourcesTests.swift @@ -74,10 +74,10 @@ import Testing @Test func loadsGuestSDKFromResourcesDirectory() throws { let source = try PromptResources.guestSDKSource() - #expect(source.contains("standalone Python script")) + #expect(source.contains("package main")) let wrapped = try PromptResources.guestSDKForModel() - #expect(wrapped.contains("```python")) - #expect(wrapped.contains("Python guest contract")) + #expect(wrapped.contains("```go")) + #expect(wrapped.contains("script-exec-contract.json")) } @Test func throwsWhenConversationRAGInstructionsAreMissing() throws { diff --git a/ui/uiTests/uiTests.swift b/ui/uiTests/uiTests.swift index c2687c8d..a3f64c31 100644 --- a/ui/uiTests/uiTests.swift +++ b/ui/uiTests/uiTests.swift @@ -205,6 +205,8 @@ import DBRepository #expect(LLMProviderCredentialGate.hasAPIKey(for: .openai, resolver: resolver)) #expect(!LLMProviderCredentialGate.hasAPIKey(for: .google, resolver: resolver)) #expect(LLMProviderCredentialGate.configuredProviders(resolver: resolver) == [.openai]) + #expect(LLMProviderCredentialGate.resolveAPIKey(for: .openai, resolver: resolver) == "test") + #expect(LLMProviderCredentialGate.resolveAPIKey(for: .google, resolver: resolver) == nil) } @Test func llmFailureClassifierDetectsCreditErrors() { diff --git a/workers/go/cmd/derrick-crawler/main.go b/workers/go/cmd/derrick-crawler/main.go new file mode 100644 index 00000000..4ea3160b --- /dev/null +++ b/workers/go/cmd/derrick-crawler/main.go @@ -0,0 +1,103 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "strconv" + + "github.com/jsoneaday/derrick/workers/internal/contract" + "github.com/jsoneaday/derrick/workers/internal/crawler" +) + +func main() { + input, err := io.ReadAll(os.Stdin) + if err != nil { + writeResult(blockedResult("", "Crawler input must be a valid JSON object.")) + return + } + + var req crawler.Request + if err := json.Unmarshal(input, &req); err != nil { + writeResult(blockedResult("", "Crawler input must be a valid JSON object.")) + return + } + + validated, err := crawler.Validate(req) + if err != nil { + writeResult(blockedResult(req.StartURL, err.Error())) + return + } + + proxy := readProxy() + ctx := context.Background() + result := crawler.Run(ctx, validated, proxy) + if len(result.Pages) == 0 && result.StopReason == crawler.StopCompleted { + result.OK = false + } + if len(result.Pages) > 0 { + result.OK = true + } + writeResult(result) +} + +func readProxy() *crawler.ProxyConfig { + host := os.Getenv("DERRICK_EGRESS_PROXY_HOST") + portStr := os.Getenv("DERRICK_EGRESS_PROXY_PORT") + token := os.Getenv("DERRICK_EGRESS_PROXY_TOKEN") + if host == "" && portStr == "" && token == "" { + return nil + } + port, _ := strconv.Atoi(portStr) + return &crawler.ProxyConfig{Host: host, Port: port, Token: token} +} + +func blockedResult(startURL string, message string) crawler.Result { + return crawler.Result{ + OK: false, + StartURL: startURL, + Pages: []crawler.Page{}, + StopReason: crawler.StopBlocked, + Diagnostics: []string{message}, + } +} + +func writeResult(result crawler.Result) { + payload, err := encodedCrawlerResult(result) + if err != nil { + writeEncodedResult(blockedResult(result.StartURL, "Crawler output violated worker contract.")) + return + } + _, _ = os.Stdout.Write(payload) +} + +func encodedCrawlerResult(result crawler.Result) ([]byte, error) { + if result.Pages == nil { + result.Pages = []crawler.Page{} + } + if result.Diagnostics == nil { + result.Diagnostics = []string{} + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(result); err != nil { + return nil, err + } + data := bytes.TrimSpace(buf.Bytes()) + if err := contract.ValidateWebCrawlerResultJSON(data); err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +func writeEncodedResult(result crawler.Result) { + payload, err := encodedCrawlerResult(result) + if err != nil { + fallback := blockedResult("", "Crawler failed to produce schema-compliant output.") + payload, _ = encodedCrawlerResult(fallback) + } + _, _ = os.Stdout.Write(payload) +} diff --git a/workers/go/cmd/derrick-file-extractor/main.go b/workers/go/cmd/derrick-file-extractor/main.go new file mode 100644 index 00000000..8a93b092 --- /dev/null +++ b/workers/go/cmd/derrick-file-extractor/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + + "github.com/jsoneaday/derrick/workers/internal/contract" + "github.com/jsoneaday/derrick/workers/internal/extractor" +) + +func main() { + input, err := io.ReadAll(os.Stdin) + if err != nil { + writeResult(extractor.Result{ + OK: false, + Operation: extractor.OperationExtract, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor input must be a valid JSON object."}, + }, 1) + return + } + + var req extractor.Request + if err := json.Unmarshal(input, &req); err != nil { + writeResult(extractor.Result{ + OK: false, + Operation: extractor.OperationExtract, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor input must be a valid JSON object."}, + }, 1) + return + } + + result := extractor.Run(req, extractor.InputDirectory, extractor.OutputDirectory) + code := 0 + if !result.OK { + code = 1 + } + writeResult(result, code) +} + +func writeResult(result extractor.Result, code int) { + payload, err := encodedExtractorResult(result) + if err != nil { + payload, _ = encodedExtractorResult(extractor.Result{ + OK: false, + Operation: result.Operation, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor output violated worker contract."}, + }) + code = 1 + } + _, _ = os.Stdout.Write(payload) + os.Exit(code) +} + +func encodedExtractorResult(result extractor.Result) ([]byte, error) { + if result.Files == nil { + result.Files = []extractor.FileResult{} + } + if result.Diagnostics == nil { + result.Diagnostics = []string{} + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(result); err != nil { + return nil, err + } + data := bytes.TrimSpace(buf.Bytes()) + if err := contract.ValidateFileExtractorResultJSON(data); err != nil { + return nil, err + } + return append(data, '\n'), nil +} diff --git a/workers/go/go.mod b/workers/go/go.mod new file mode 100644 index 00000000..df29702d --- /dev/null +++ b/workers/go/go.mod @@ -0,0 +1,21 @@ +module github.com/jsoneaday/derrick/workers + +go 1.27.1 + +require ( + github.com/PuerkitoBio/goquery v1.10.3 + github.com/xuri/excelize/v2 v2.9.1 +) + +require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect + github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.1 // indirect + golang.org/x/crypto v0.38.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/text v0.25.0 // indirect +) diff --git a/workers/go/go.sum b/workers/go/go.sum new file mode 100644 index 00000000..80c00844 --- /dev/null +++ b/workers/go/go.sum @@ -0,0 +1,100 @@ +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= +github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= +github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= +github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= +github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= +golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/workers/go/internal/contract/contract.go b/workers/go/internal/contract/contract.go new file mode 100644 index 00000000..ba341c0b --- /dev/null +++ b/workers/go/internal/contract/contract.go @@ -0,0 +1,120 @@ +package contract + +import ( + "bytes" + "embed" + "encoding/json" + "fmt" + "io/fs" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +//go:embed schemas/*.json +var schemaFS embed.FS + +// ValidateHopEventJSON checks stdin against hop-event.schema.json. +func ValidateHopEventJSON(data []byte) error { + return validate("hop-event.schema.json", data) +} + +// ValidateEnvelopeListJSON checks stdout against envelope-list.schema.json. +func ValidateEnvelopeListJSON(data []byte) error { + return validate("envelope-list.schema.json", data) +} + +// ValidateWebCrawlerResultJSON checks crawler stdout against web-crawler-result.schema.json. +func ValidateWebCrawlerResultJSON(data []byte) error { + return validate("web-crawler-result.schema.json", data) +} + +// ValidateFileExtractorResultJSON checks extractor stdout against file-extractor-result.schema.json. +func ValidateFileExtractorResultJSON(data []byte) error { + return validate("file-extractor-result.schema.json", data) +} + +func validate(schemaName string, data []byte) error { + compiler := jsonschema.NewCompiler() + if err := loadSchemas(compiler); err != nil { + return err + } + schema, err := compiler.Compile(schemaCompileURL(schemaName)) + if err != nil { + return fmt.Errorf("compile schema %s: %w", schemaName, err) + } + var value any + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&value); err != nil { + return fmt.Errorf("invalid json: %w", err) + } + if err := schema.Validate(value); err != nil { + return fmt.Errorf("schema validation failed: %w", err) + } + return nil +} + +const schemaBase = "https://derrick.local/schemas/" + +func loadSchemas(compiler *jsonschema.Compiler) error { + return fs.WalkDir(schemaFS, "schemas", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + raw, err := schemaFS.ReadFile(path) + if err != nil { + return err + } + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + name := strings.TrimPrefix(path, "schemas/") + ids := schemaResourceIDs(name, doc) + for _, id := range ids { + if err := compiler.AddResource(id, doc); err != nil { + return err + } + } + return nil + }) +} + +func schemaResourceIDs(filename string, doc any) []string { + seen := map[string]bool{} + add := func(id string) { + if id == "" || seen[id] { + return + } + seen[id] = true + } + add(schemaBase + filename) + if m, ok := doc.(map[string]any); ok { + if id, ok := m["$id"].(string); ok { + add(id) + } + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + return ids +} + +func schemaCompileURL(name string) string { + raw, err := schemaFS.ReadFile("schemas/" + name) + if err != nil { + return schemaBase + name + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err == nil { + if id, ok := doc["$id"].(string); ok && id != "" { + return id + } + } + return schemaBase + name +} diff --git a/workers/go/internal/contract/contract_test.go b/workers/go/internal/contract/contract_test.go new file mode 100644 index 00000000..bd5d2291 --- /dev/null +++ b/workers/go/internal/contract/contract_test.go @@ -0,0 +1,49 @@ +package contract + +import "testing" + +func TestValidateHopEventAndEnvelopeList(t *testing.T) { + hop := []byte(`{"kind":"manual"}`) + if err := ValidateHopEventJSON(hop); err != nil { + t.Fatalf("hop event: %v", err) + } + env := []byte(`[{"verb":"result.emit","summary":"ok"}]`) + if err := ValidateEnvelopeListJSON(env); err != nil { + t.Fatalf("envelope list: %v", err) + } +} + +func TestValidateEnvelopeListRejectsNestedAlias(t *testing.T) { + env := []byte(`[{"verb":"result.emit","result":{"emit":{"content":"x"}}}]`) + if err := ValidateEnvelopeListJSON(env); err == nil { + t.Fatal("expected rejection") + } +} + +func TestValidateWebCrawlerResultAcceptsEmptyDiagnosticsArray(t *testing.T) { + payload := []byte(`{"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":[]}`) + if err := ValidateWebCrawlerResultJSON(payload); err != nil { + t.Fatalf("web crawler result: %v", err) + } +} + +func TestValidateWebCrawlerResultRejectsNullDiagnostics(t *testing.T) { + payload := []byte(`{"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":null}`) + if err := ValidateWebCrawlerResultJSON(payload); err == nil { + t.Fatal("expected rejection for null diagnostics") + } +} + +func TestValidateFileExtractorResultAcceptsEmptyFilesArray(t *testing.T) { + payload := []byte(`{"ok":true,"operation":"extract","files":[],"diagnostics":[]}`) + if err := ValidateFileExtractorResultJSON(payload); err != nil { + t.Fatalf("file extractor result: %v", err) + } +} + +func TestValidateFileExtractorResultRejectsNullFiles(t *testing.T) { + payload := []byte(`{"ok":false,"operation":"extract","files":null,"diagnostics":[]}`) + if err := ValidateFileExtractorResultJSON(payload); err == nil { + t.Fatal("expected rejection for null files") + } +} diff --git a/workers/go/internal/contract/schemas/connector-contract.schema.json b/workers/go/internal/contract/schemas/connector-contract.schema.json new file mode 100644 index 00000000..5ca68299 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-contract.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-contract.json", + "title": "Derrick connector protocol document", + "description": "Canonical host ops, emit shapes, allowed vendor-call ids, and success rules. Vendor HTTP bindings live in contracts/vendors/*.json.", + "type": "object", + "required": ["version", "ops", "scopes", "rules"], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "ops": { + "type": "object", + "required": ["sync_threads", "poll_inbox", "send_message"], + "additionalProperties": { "$ref": "#/$defs/op" } + }, + "scopes": { + "type": "object", + "required": ["full_sync"], + "additionalProperties": { "$ref": "#/$defs/scope" } + }, + "rules": { + "type": "object", + "required": [ + "sync_threads_lists_tabs_only", + "poll_loads_one_conversation", + "paginate_within_op", + "live_http_results_accumulate", + "direct_test_poll_requires_non_empty_messages", + "direct_test_threads_requires_non_empty", + "runtime_empty_messages_ok_if_vendor_ok" + ], + "properties": { + "sync_threads_lists_tabs_only": { "type": "boolean" }, + "poll_loads_one_conversation": { "type": "boolean" }, + "paginate_within_op": { "type": "boolean" }, + "live_http_results_accumulate": { "type": "boolean" }, + "direct_test_poll_requires_non_empty_messages": { "type": "boolean" }, + "direct_test_threads_requires_non_empty": { "type": "boolean" }, + "runtime_empty_messages_ok_if_vendor_ok": { "type": "boolean" } + }, + "additionalProperties": false + } + }, + "$defs": { + "op": { + "type": "object", + "required": ["kind", "emit"], + "properties": { + "kind": { "type": "string", "enum": ["manual", "message_in_room"] }, + "emit": { "type": "string", "enum": ["threads", "messages", "sent_message"] }, + "params": { "type": "array", "items": { "type": "string" } }, + "may_call": { "type": "array", "items": { "type": "string" } }, + "must_not_call": { "type": "array", "items": { "type": "string" } }, + "when_no_parent": { "$ref": "#/$defs/call_set" }, + "when_parent": { "$ref": "#/$defs/call_set" }, + "success": { "$ref": "#/$defs/success" }, + "failure": { "$ref": "#/$defs/failure" } + }, + "additionalProperties": false + }, + "call_set": { + "type": "object", + "properties": { + "may_call": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + }, + "success": { + "type": "object", + "properties": { + "empty_collection_ok": { "type": "boolean" }, + "empty_array_ok_if_vendor_ok": { "type": "boolean" }, + "requires_sent_message": { "type": "boolean" } + }, + "additionalProperties": false + }, + "failure": { + "type": "object", + "properties": { + "vendor_ok_false_empty_is_not_success": { "type": "boolean" }, + "codes": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + }, + "scope": { + "type": "object", + "required": ["ops", "include_reply_poll", "test_pagination"], + "properties": { + "ops": { "type": "array", "items": { "type": "string" } }, + "include_reply_poll": { "type": "boolean" }, + "test_pagination": { "type": "string", "enum": ["single_page", "follow_cursor"] }, + "poll_must_not_call": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + } + } +} diff --git a/workers/go/internal/contract/schemas/connector-params.schema.json b/workers/go/internal/contract/schemas/connector-params.schema.json new file mode 100644 index 00000000..32ba1a50 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-params.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-params.json", + "title": "Connector hop params", + "description": "params object on a connector plugin.invoke hop event.", + "type": "object", + "properties": { + "messaging_op": { + "type": "string", + "enum": ["sync_threads", "poll_inbox", "send_message"] + }, + "vendor_thread_id": { "type": "string" }, + "text": { "type": "string" }, + "since": { "type": "string" }, + "oldest": { "type": "string" }, + "parent_vendor_message_id": { "type": "string" }, + "thread_ts": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/connector-result-emit.schema.json b/workers/go/internal/contract/schemas/connector-result-emit.schema.json new file mode 100644 index 00000000..b67fcd14 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-result-emit.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-result-emit.json", + "title": "Connector result.emit payload", + "description": "Structured fields the host persists from a connector terminal envelope.", + "type": "object", + "additionalProperties": false, + "properties": { + "threads": { + "type": "array", + "items": { + "type": "object", + "required": ["vendor_thread_id", "title"], + "additionalProperties": false, + "properties": { + "vendor_thread_id": { "type": "string" }, + "title": { "type": "string" }, + "is_member": { "type": "boolean" }, + "accessible": { "type": "boolean" } + } + } + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "required": ["vendor_thread_id", "vendor_message_id", "direction", "sender", "body", "created_at"], + "additionalProperties": false, + "properties": { + "vendor_thread_id": { "type": "string" }, + "vendor_message_id": { "type": "string" }, + "direction": { "type": "string", "enum": ["inbound", "outbound"] }, + "sender": { "type": "string" }, + "body": { "type": "string" }, + "created_at": { "type": ["string", "number"] }, + "parent_vendor_message_id": { "type": "string" }, + "thread_ts": { "type": "string" }, + "reply_count": { "type": "integer", "minimum": 0 } + } + } + }, + "sent_message": { + "type": "object", + "required": ["vendor_message_id", "created_at"], + "additionalProperties": false, + "properties": { + "vendor_message_id": { "type": ["string", "number"] }, + "created_at": { "type": ["string", "number"] } + } + }, + "title": { "type": "string" }, + "summary": { "type": "string" } + } +} diff --git a/workers/go/internal/contract/schemas/connector-vendor.schema.json b/workers/go/internal/contract/schemas/connector-vendor.schema.json new file mode 100644 index 00000000..5959d2a2 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-vendor.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-vendor.schema.json", + "title": "Connector vendor profile", + "description": "HTTP bindings for one vendor. Call ids must match may_call / must_not_call in connector-contract.json.", + "type": "object", + "required": ["vendor", "vendor_ok_field", "calls"], + "additionalProperties": false, + "properties": { + "vendor": { "type": "string" }, + "vendor_ok_field": { "type": "string" }, + "membership_flag": { "type": "string" }, + "pagination_cursor": { "type": "string" }, + "calls": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "required": ["method", "url"], + "additionalProperties": false, + "properties": { + "method": { "type": "string", "minLength": 1 }, + "url": { "type": "string", "minLength": 1 } + } + } + } + } +} diff --git a/workers/go/internal/contract/schemas/envelope-list.schema.json b/workers/go/internal/contract/schemas/envelope-list.schema.json new file mode 100644 index 00000000..5fbef6d8 --- /dev/null +++ b/workers/go/internal/contract/schemas/envelope-list.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/envelope-list.json", + "title": "Guest stdout envelope list", + "description": "JSON array written to stdout by an offline guest (plugin or script). Canonical contract for all guest languages.", + "type": "array", + "minItems": 0, + "items": { + "type": "object", + "required": ["verb"], + "additionalProperties": false, + "properties": { + "verb": { + "type": "string", + "enum": [ + "http.request", + "result.emit", + "message.post", + "ui.present", + "secret.request", + "storage.read", + "storage.write", + "job.schedule", + "log" + ] + }, + "request_id": { "type": "string" }, + "method": { "type": "string" }, + "url": { "type": "string" }, + "title": { "type": "string" }, + "summary": { "type": "string" }, + "text": { "type": "string" }, + "content": { "type": "string" }, + "html": { "type": "string" }, + "markdown": { "type": "string" }, + "message": { "type": "string" }, + "schema_version": { "type": "integer" }, + "auth_ref": { "type": ["string", "null"] }, + "json": { + "description": "HTTP request body as a JSON value. HostHTTPRequest.json. The host sends this as the wire body." + }, + "headers": { "type": "object", "additionalProperties": { "type": "string" } }, + "widgets": { "type": "array", "items": { "type": "object" } }, + "secret_ref": { "type": "string" }, + "reason": { "type": "string" }, + "key": { "type": "string" }, + "value": true, + "interval_seconds": { "type": "integer" }, + "timezone": { "type": "string" }, + "threads": { "$ref": "connector-result-emit.schema.json#/properties/threads" }, + "messages": { "$ref": "connector-result-emit.schema.json#/properties/messages" }, + "sent_message": { "$ref": "connector-result-emit.schema.json#/properties/sent_message" } + } + } +} diff --git a/workers/go/internal/contract/schemas/execution-context-wire.schema.json b/workers/go/internal/contract/schemas/execution-context-wire.schema.json new file mode 100644 index 00000000..b1529b66 --- /dev/null +++ b/workers/go/internal/contract/schemas/execution-context-wire.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ExecutionContextWire", + "type": "object", + "required": ["schema_version", "session_id", "principal"], + "properties": { + "schema_version": { "const": 1 }, + "session_id": { "type": "string", "minLength": 1 }, + "turn_id": { "type": "string" }, + "agent_id": { "type": "string" }, + "principal": { "type": "string", "minLength": 1 }, + "workflow": { + "type": "object", + "properties": { + "workflow_id": { "type": "string" }, + "kind": { + "type": "string", + "enum": [ + "plugin_factory_create", + "plugin_factory_edit", + "connector_auth_discover", + "job_step", + "interactive_tool", + "none" + ] + }, + "step_id": { "type": "string" }, + "step_kind": { "type": "string" } + }, + "additionalProperties": false + }, + "delivery": { + "type": "string", + "enum": ["live_chat", "notification", "silent"] + }, + "capabilities": { + "type": "array", + "items": { + "type": "string", + "enum": ["sync_web_crawl", "host_review_retry"] + }, + "uniqueItems": true + } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/file-extractor-result.schema.json b/workers/go/internal/contract/schemas/file-extractor-result.schema.json new file mode 100644 index 00000000..c4e12bc4 --- /dev/null +++ b/workers/go/internal/contract/schemas/file-extractor-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/file-extractor-result.json", + "title": "File extractor worker stdout", + "description": "JSON object written to stdout by derrick-file-extractor.", + "$ref": "worker-product.schema.json#/$defs/file_extractor_result" +} diff --git a/workers/go/internal/contract/schemas/guest-runtime.schema.json b/workers/go/internal/contract/schemas/guest-runtime.schema.json new file mode 100644 index 00000000..03f4f622 --- /dev/null +++ b/workers/go/internal/contract/schemas/guest-runtime.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/guest-runtime.json", + "title": "Derrick offline guest runtime", + "description": "Canonical I/O contract for script_exec and plugin.invoke guests. Trusted worker stdout (crawler, extractor) is defined in worker-product.schema.json. Swift host and Go workers must match these schemas.", + "type": "object", + "required": ["language", "stdin", "stdout"], + "additionalProperties": false, + "properties": { + "language": { + "type": "string", + "const": "go", + "description": "Guest implementation language." + }, + "stdin": { + "description": "One hop event JSON object read from standard input.", + "$ref": "hop-event.schema.json" + }, + "stdout": { + "description": "Envelope list JSON array written to standard output.", + "$ref": "envelope-list.schema.json" + }, + "binary": { + "type": "string", + "const": "/tmp/guest", + "description": "Linux guest binary path inside the worker container." + } + } +} diff --git a/workers/go/internal/contract/schemas/hop-event.schema.json b/workers/go/internal/contract/schemas/hop-event.schema.json new file mode 100644 index 00000000..a183e284 --- /dev/null +++ b/workers/go/internal/contract/schemas/hop-event.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/hop-event.json", + "title": "Guest stdin hop event", + "description": "JSON object read from stdin when the host invokes an offline guest. Canonical contract for all guest languages.", + "type": "object", + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "enum": [ + "manual", + "schedule", + "message_in_room", + "http_results", + "ui_action", + "grant_ready", + "harness", + "script" + ] + }, + "http_results": { + "type": "array", + "items": { + "type": "object", + "required": ["request_id", "status"], + "additionalProperties": false, + "properties": { + "request_id": { "type": "string" }, + "status": { "type": "integer" }, + "headers": { "type": "object", "additionalProperties": { "type": "string" } }, + "body": { "type": "string" }, + "error": { "type": ["string", "null"] } + } + } + }, + "params": { + "$ref": "connector-params.schema.json" + } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/script-exec-contract.schema.json b/workers/go/internal/contract/schemas/script-exec-contract.schema.json new file mode 100644 index 00000000..2e369012 --- /dev/null +++ b/workers/go/internal/contract/schemas/script-exec-contract.schema.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/script-exec-contract.json", + "title": "Derrick script_exec protocol document", + "description": "Canonical guest runtime, workflow, output, agent, and reviewer rules for script_exec and offline Go guests.", + "type": "object", + "required": [ + "version", + "runtime", + "io", + "workflow", + "output", + "agent", + "review", + "rules", + "plugin_factory" + ], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "runtime": { "$ref": "#/$defs/runtime" }, + "io": { "$ref": "#/$defs/io" }, + "workflow": { "$ref": "#/$defs/workflow" }, + "output": { "$ref": "#/$defs/output" }, + "agent": { "$ref": "#/$defs/agent" }, + "review": { "$ref": "#/$defs/review" }, + "rules": { "$ref": "#/$defs/rules" }, + "plugin_factory": { "$ref": "#/$defs/plugin_factory" } + }, + "$defs": { + "runtime": { + "type": "object", + "required": [ + "language", + "package", + "binary", + "stdlib_only", + "forbidden_imports", + "allowed_os_usage", + "example_go" + ], + "additionalProperties": false, + "properties": { + "language": { "type": "string", "const": "go" }, + "package": { "type": "string", "const": "main" }, + "binary": { "type": "string", "const": "/tmp/guest" }, + "stdlib_only": { "type": "boolean" }, + "forbidden_imports": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "allowed_os_usage": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "example_go": { "type": "string", "minLength": 1 } + } + }, + "io": { + "type": "object", + "required": ["stdin_schema", "stdout_schema", "guest_runtime_schema"], + "additionalProperties": false, + "properties": { + "stdin_schema": { "type": "string", "const": "hop-event.schema.json" }, + "stdout_schema": { "type": "string", "const": "envelope-list.schema.json" }, + "guest_runtime_schema": { "type": "string", "const": "guest-runtime.schema.json" } + } + }, + "workflow": { + "type": "object", + "required": [ + "first_hop_verbs", + "http_results_event_kind", + "terminal_verbs", + "match_http_by", + "http_results_accumulate", + "post_body_field" + ], + "additionalProperties": false, + "properties": { + "first_hop_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "http_results_event_kind": { "type": "string" }, + "terminal_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "match_http_by": { "type": "string", "const": "request_id" }, + "http_results_accumulate": { "type": "boolean" }, + "post_body_field": { "type": "string", "const": "json" } + } + }, + "output": { + "type": "object", + "required": ["fields", "forbidden_patterns", "raw_body_in_content_requires_explicit_user_request"], + "additionalProperties": false, + "properties": { + "fields": { + "type": "object", + "required": ["content", "summary", "html", "markdown"], + "additionalProperties": false, + "properties": { + "content": { "$ref": "#/$defs/output_field" }, + "summary": { "$ref": "#/$defs/output_field" }, + "html": { "$ref": "#/$defs/output_field" }, + "markdown": { "$ref": "#/$defs/output_field" } + } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" } + }, + "raw_body_in_content_requires_explicit_user_request": { "type": "boolean" } + } + }, + "output_field": { + "type": "object", + "required": ["purpose"], + "additionalProperties": true, + "properties": { + "purpose": { "type": "string" }, + "host_strips_incidental_markup": { "type": "boolean" }, + "host_allowlist_sanitizes": { "type": "boolean" }, + "prefer_content_for_extracted_text": { "type": "boolean" } + } + }, + "agent": { + "type": "object", + "required": [ + "prefer_direct_urls_over_serp_html", + "retry_with_different_urls_on_empty_fetch", + "max_correction_attempts_after_block" + ], + "additionalProperties": false, + "properties": { + "prefer_direct_urls_over_serp_html": { "type": "boolean" }, + "retry_with_different_urls_on_empty_fetch": { "type": "boolean" }, + "max_correction_attempts_after_block": { "type": "integer", "minimum": 0 } + } + }, + "review": { + "type": "object", + "required": [ + "fail_fast", + "response_schema", + "checks", + "do_not_deny_for", + "enforced_elsewhere" + ], + "additionalProperties": false, + "properties": { + "fail_fast": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": [ + "alignedWithRequest", + "confidence", + "suggestedAction", + "concerns", + "summary" + ], + "additionalProperties": false, + "properties": { + "alignedWithRequest": { "type": "string" }, + "confidence": { "type": "string" }, + "suggestedAction": { "type": "string" }, + "concerns": { "type": "string" }, + "summary": { "type": "string" } + } + }, + "checks": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/review_check" } + }, + "do_not_deny_for": { + "type": "array", + "items": { "type": "string" } + }, + "enforced_elsewhere": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "review_check": { + "type": "object", + "required": ["id", "order", "description"], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "order": { "type": "integer", "minimum": 1 }, + "description": { "type": "string" }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "rules": { + "type": "object", + "required": [ + "guest_has_no_network", + "host_performs_http", + "host_applies_ssrf", + "static_verifier_enforces_imports", + "scheduler_timing_not_in_script", + "deterministic_output_required", + "stable_sort_and_dedupe_collections", + "no_time_random_uuid_for_visible_output", + "match_http_results_by_request_id" + ], + "additionalProperties": false, + "properties": { + "guest_has_no_network": { "type": "boolean" }, + "host_performs_http": { "type": "boolean" }, + "host_applies_ssrf": { "type": "boolean" }, + "static_verifier_enforces_imports": { "type": "boolean" }, + "scheduler_timing_not_in_script": { "type": "boolean" }, + "deterministic_output_required": { "type": "boolean" }, + "stable_sort_and_dedupe_collections": { "type": "boolean" }, + "no_time_random_uuid_for_visible_output": { "type": "boolean" }, + "match_http_results_by_request_id": { "type": "boolean" } + } + }, + "plugin_factory": { + "type": "object", + "required": ["manifest", "builder", "review"], + "additionalProperties": false, + "properties": { + "manifest": { "$ref": "#/$defs/plugin_factory_manifest" }, + "builder": { "$ref": "#/$defs/plugin_factory_builder" }, + "review": { "$ref": "#/$defs/plugin_factory_review" } + } + }, + "plugin_factory_manifest": { + "type": "object", + "required": [ + "host_creates_manifest", + "do_not_return_manifest_json", + "agent_plugin_schema", + "entrypoint", + "plugin_id_allowed_chars", + "plugin_id_forbidden_chars", + "roles", + "connector_messaging_ops", + "secret_kinds", + "never_embed_credentials_in_go_source", + "host_lists_connectors_under_messaging" + ], + "additionalProperties": false, + "properties": { + "host_creates_manifest": { "type": "boolean" }, + "do_not_return_manifest_json": { "type": "boolean" }, + "agent_plugin_schema": { "type": "string" }, + "entrypoint": { "type": "string" }, + "plugin_id_allowed_chars": { "type": "string" }, + "plugin_id_forbidden_chars": { + "type": "array", + "items": { "type": "string" } + }, + "roles": { + "type": "array", + "items": { "type": "string" } + }, + "connector_messaging_ops": { + "type": "array", + "items": { "type": "string" } + }, + "secret_kinds": { + "type": "array", + "items": { "type": "string" } + }, + "never_embed_credentials_in_go_source": { "type": "boolean" }, + "host_lists_connectors_under_messaging": { "type": "boolean" } + } + }, + "plugin_factory_builder": { + "type": "object", + "required": [ + "response_keys", + "skill_files_path_pattern", + "empty_skill_files_when_unused", + "test_input_is_serialized_json_object", + "test_input_must_not_be_empty", + "test_input_exercises_terminal_result", + "connector_test_input_uses_hops_array", + "connector_parse_json_http_results_when_vendor_returns_json" + ], + "additionalProperties": false, + "properties": { + "response_keys": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "skill_files_path_pattern": { "type": "string" }, + "empty_skill_files_when_unused": { "type": "boolean" }, + "test_input_is_serialized_json_object": { "type": "boolean" }, + "test_input_must_not_be_empty": { "type": "boolean" }, + "test_input_exercises_terminal_result": { "type": "boolean" }, + "connector_test_input_uses_hops_array": { "type": "boolean" }, + "connector_parse_json_http_results_when_vendor_returns_json": { "type": "boolean" } + } + }, + "plugin_factory_review": { + "type": "object", + "required": [ + "compilation_success_not_approval", + "response_schema", + "reject_non_go_source", + "reject_raw_network_outside_http_request_envelopes", + "reject_missing_stdin_read", + "source_derived_titles_may_be_fragments", + "reject_unsupported_direct_test_claims", + "connector_rules_document" + ], + "additionalProperties": false, + "properties": { + "compilation_success_not_approval": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": ["decision", "summary", "findings"], + "additionalProperties": false, + "properties": { + "decision": { "type": "string" }, + "summary": { "type": "string" }, + "findings": { "type": "string" } + } + }, + "reject_non_go_source": { "type": "boolean" }, + "reject_raw_network_outside_http_request_envelopes": { "type": "boolean" }, + "reject_missing_stdin_read": { "type": "boolean" }, + "source_derived_titles_may_be_fragments": { "type": "boolean" }, + "reject_unsupported_direct_test_claims": { "type": "boolean" }, + "connector_rules_document": { "type": "string" } + } + } + } +} diff --git a/workers/go/internal/contract/schemas/web-crawler-result.schema.json b/workers/go/internal/contract/schemas/web-crawler-result.schema.json new file mode 100644 index 00000000..9af39755 --- /dev/null +++ b/workers/go/internal/contract/schemas/web-crawler-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/web-crawler-result.json", + "title": "Web crawler worker stdout", + "description": "JSON object written to stdout by derrick-web-crawler.", + "$ref": "worker-product.schema.json#/$defs/web_crawler_result" +} diff --git a/workers/go/internal/contract/schemas/worker-product.schema.json b/workers/go/internal/contract/schemas/worker-product.schema.json new file mode 100644 index 00000000..874f6bfb --- /dev/null +++ b/workers/go/internal/contract/schemas/worker-product.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/worker-product.json", + "title": "Derrick trusted worker product contracts", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "$defs": { + "string_list": { + "type": "array", + "items": { "type": "string" } + }, + "web_crawler_stop_reason": { + "type": "string", + "enum": [ + "completed", + "max_pages", + "max_depth", + "timeout", + "total_bytes", + "queue_limit", + "cancelled", + "blocked" + ] + }, + "web_crawler_page": { + "type": "object", + "required": ["url", "depth", "status_code", "title", "text", "links_found"], + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "depth": { "type": "integer" }, + "status_code": { "type": "integer" }, + "content_type": { "type": "string" }, + "title": { "type": "string" }, + "text": { "type": "string" }, + "links_found": { "type": "integer" } + } + }, + "web_crawler_result": { + "type": "object", + "required": [ + "ok", + "start_url", + "pages", + "stop_reason", + "requests_made", + "bytes_read", + "truncated", + "diagnostics" + ], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "start_url": { "type": "string" }, + "pages": { + "type": "array", + "items": { "$ref": "#/$defs/web_crawler_page" } + }, + "stop_reason": { "$ref": "#/$defs/web_crawler_stop_reason" }, + "requests_made": { "type": "integer" }, + "bytes_read": { "type": "integer" }, + "truncated": { "type": "boolean" }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + }, + "file_extractor_operation": { + "type": "string", + "enum": ["extract", "convert"] + }, + "file_extractor_file_result": { + "type": "object", + "required": ["input_name", "kind", "byte_count"], + "additionalProperties": false, + "properties": { + "input_name": { "type": "string" }, + "output_name": { "type": "string" }, + "kind": { "type": "string" }, + "byte_count": { "type": "integer" }, + "preview": { "type": "string" }, + "error": { "type": "string" } + } + }, + "file_extractor_result": { + "type": "object", + "required": ["ok", "operation", "files", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "operation": { "$ref": "#/$defs/file_extractor_operation" }, + "files": { + "type": "array", + "items": { "$ref": "#/$defs/file_extractor_file_result" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + } + } +} diff --git a/workers/go/internal/crawler/engine.go b/workers/go/internal/crawler/engine.go new file mode 100644 index 00000000..890ea221 --- /dev/null +++ b/workers/go/internal/crawler/engine.go @@ -0,0 +1,372 @@ +package crawler + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +type extractedPage struct { + title string + text string + isHTML bool +} + +func Run(ctx context.Context, req ValidatedRequest, proxy *ProxyConfig) Result { + deadline := time.Now().Add(time.Duration(req.TimeoutSeconds) * time.Second) + engine := &bfsEngine{ + req: req, + deadline: deadline, + proxy: proxy, + client: newHTTPClient(proxy), + } + return engine.run(ctx) +} + +type queueItem struct { + url *url.URL + depth int +} + +type bfsEngine struct { + req ValidatedRequest + deadline time.Time + proxy *ProxyConfig + client *http.Client + queue []queueItem + queuedKeys map[string]bool + visitedKeys map[string]bool + pageDepths map[string]int + pages []Page + pageIndexes map[string]int + diagnostics []string + stopReason StopReason + reachedMaxDepth bool + requestsMade int + bytesRead int + lastRequestAt time.Time +} + +func (e *bfsEngine) run(ctx context.Context) Result { + startKey := URLKey(e.req.StartURL) + e.queuedKeys = map[string]bool{startKey: true} + e.visitedKeys = map[string]bool{} + e.pageDepths = map[string]int{startKey: 0} + e.pageIndexes = map[string]int{} + e.queue = []queueItem{{url: e.req.StartURL, depth: 0}} + + for len(e.queue) > 0 && e.stopReason == "" { + if time.Now().After(e.deadline) { + e.stopReason = StopTimeout + break + } + if len(e.pages) >= e.req.MaxPages { + e.stopReason = StopMaxPages + break + } + if ctx.Err() != nil { + e.stopReason = StopCancelled + break + } + + next := e.queue[0] + e.queue = e.queue[1:] + key := URLKey(next.url) + delete(e.queuedKeys, key) + + if !HostAllowed(e.req.AllowedHosts, next.url.Hostname()) || !IsHTTP(next.url) { + continue + } + if e.visitedKeys[key] { + continue + } + e.visitedKeys[key] = true + if len(e.visitedKeys) > e.req.MaxPages { + e.stopReason = StopMaxPages + break + } + + e.visit(ctx, next.url, next.depth) + } + + if e.stopReason == "" { + if e.reachedMaxDepth { + e.stopReason = StopMaxDepth + } else { + e.stopReason = StopCompleted + } + } + + truncated := false + totalChars := 0 + for i, p := range e.pages { + totalChars += len(p.Text) + if totalChars > MaximumOutputChars { + truncated = true + e.pages = e.pages[:i] + break + } + } + + return Result{ + OK: e.stopReason != StopBlocked && len(e.pages) > 0, + StartURL: e.req.StartURL.String(), + Pages: e.pages, + StopReason: e.stopReason, + RequestsMade: e.requestsMade, + BytesRead: e.bytesRead, + Truncated: truncated, + Diagnostics: e.diagnostics, + } +} + +func (e *bfsEngine) visit(ctx context.Context, pageURL *url.URL, depth int) { + if e.stopReason != "" || time.Now().After(e.deadline) { + return + } + + normalized := NormalizeURL(pageURL) + e.requestsMade++ + body, status, contentType, finalURL, err := e.fetch(ctx, normalized) + if err != nil { + e.diagnostics = append(e.diagnostics, fmt.Sprintf("%s: %s", normalized.String(), err.Error())) + return + } + + e.bytesRead += len(body) + if e.bytesRead > MaximumTotalBytes { + e.stopReason = StopTotalBytes + return + } + + extracted := extractContent(body, contentType, finalURL) + ct := contentType + page := Page{ + URL: finalURL.String(), + Depth: depth, + StatusCode: status, + ContentType: func() *string { + if ct == "" { + return nil + } + return &ct + }(), + Title: extracted.title, + Text: clipText(extracted.text, MaximumExtractedText), + } + e.pages = append(e.pages, page) + sourceKey := URLKey(finalURL) + e.pageIndexes[sourceKey] = len(e.pages) - 1 + e.pageDepths[sourceKey] = depth + + if !extracted.isHTML { + return + } + + links := parseLinks(body, finalURL) + accepted := e.enqueueLinks(sourceKey, links) + idx := e.pageIndexes[sourceKey] + e.pages[idx].LinksFound = accepted +} + +func (e *bfsEngine) enqueueLinks(sourceKey string, links []*url.URL) int { + sourceDepth := e.pageDepths[sourceKey] + accepted := 0 + for _, link := range links { + if e.stopReason != "" { + break + } + if len(links) > MaximumLinksPerPage && accepted >= MaximumLinksPerPage { + break + } + normalized := NormalizeURL(link) + key := URLKey(normalized) + if !HostAllowed(e.req.AllowedHosts, normalized.Hostname()) || !IsHTTP(normalized) { + continue + } + if e.visitedKeys[key] || e.queuedKeys[key] { + continue + } + nextDepth := sourceDepth + 1 + if nextDepth > e.req.MaxDepth { + e.reachedMaxDepth = true + continue + } + if len(e.queue) >= MaximumQueuedURLs { + e.stopReason = StopQueueLimit + break + } + e.queuedKeys[key] = true + e.pageDepths[key] = nextDepth + e.queue = append(e.queue, queueItem{url: normalized, depth: nextDepth}) + accepted++ + } + return accepted +} + +func newHTTPClient(proxy *ProxyConfig) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + if proxy != nil && proxy.Host != "" { + proxyURL, _ := url.Parse(fmt.Sprintf("http://%s:%d", proxy.Host, proxy.Port)) + transport.Proxy = http.ProxyURL(proxyURL) + if proxy.Token != "" { + transport.ProxyConnectHeader = http.Header{ + "X-Derrick-Crawler-Token": []string{proxy.Token}, + } + } + } + return &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Timeout: 20 * time.Second, + } +} + +func (e *bfsEngine) fetch(ctx context.Context, start *url.URL) (body string, status int, contentType string, final *url.URL, err error) { + e.waitForSpacing() + current := start + redirected := map[string]bool{URLKey(start): true} + + for i := 0; i <= MaximumRedirectsPerPage; i++ { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, current.String(), nil) + if err != nil { + return "", 0, "", start, err + } + req.Header.Set("User-Agent", "DerrickWebCrawler/1") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Accept-Encoding", "identity") + if e.proxy != nil && e.proxy.Token != "" { + req.Header.Set("X-Derrick-Crawler-Token", e.proxy.Token) + } + + resp, err := e.client.Do(req) + if err != nil { + return "", 0, "", start, err + } + limited := io.LimitReader(resp.Body, int64(MaximumPageBytes)) + data, err := io.ReadAll(limited) + resp.Body.Close() + if err != nil { + return "", 0, "", start, err + } + text := string(data) + ct := resp.Header.Get("Content-Type") + + if resp.StatusCode < 300 || resp.StatusCode >= 400 { + return text, resp.StatusCode, ct, current, nil + } + + location := resp.Header.Get("Location") + if location == "" { + return text, resp.StatusCode, ct, current, nil + } + redirectURL, err := url.Parse(location) + if err != nil { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + redirectURL = current.ResolveReference(redirectURL) + if !IsHTTP(redirectURL) { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + host := strings.ToLower(strings.TrimSpace(redirectURL.Hostname())) + if host == "" { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + e.req.AllowedHosts[host] = true + normalized := NormalizeURL(redirectURL) + key := URLKey(normalized) + if redirected[key] { + return "", 0, "", start, fmt.Errorf("redirect loop detected") + } + redirected[key] = true + current = normalized + e.waitForSpacing() + } + return "", 0, "", start, fmt.Errorf("redirect limit reached") +} + +func (e *bfsEngine) waitForSpacing() { + if e.lastRequestAt.IsZero() { + e.lastRequestAt = time.Now() + return + } + elapsed := time.Since(e.lastRequestAt) + minimum := time.Duration(MinRequestDelayMS) * time.Millisecond + if elapsed < minimum { + time.Sleep(minimum - elapsed) + } + e.lastRequestAt = time.Now() +} + +func extractContent(body string, contentType string, base *url.URL) extractedPage { + lower := strings.ToLower(contentType) + if strings.Contains(lower, "html") || strings.Contains(lower, "xhtml") { + return extractHTML(body) + } + prefix := body + if len(prefix) > MaximumExtractedText { + prefix = prefix[:MaximumExtractedText] + } + return extractedPage{title: "", text: prefix, isHTML: false} +} + +func extractHTML(body string) extractedPage { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + return extractedPage{text: body, isHTML: true} + } + doc.Find("script, style, noscript, template, svg").Remove() + title := strings.TrimSpace(doc.Find("title").First().Text()) + text := strings.TrimSpace(doc.Find("body").Text()) + if text == "" { + text = strings.TrimSpace(doc.Text()) + } + return extractedPage{title: title, text: text, isHTML: true} +} + +func parseLinks(body string, base *url.URL) []*url.URL { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + return nil + } + var links []*url.URL + seen := map[string]bool{} + doc.Find("a[href]").Each(func(_ int, s *goquery.Selection) { + href, ok := s.Attr("href") + if !ok || strings.TrimSpace(href) == "" { + return + } + parsed, err := url.Parse(href) + if err != nil { + return + } + resolved := base.ResolveReference(parsed) + resolved.Fragment = "" + resolved.RawFragment = "" + resolved = NormalizeURL(resolved) + key := URLKey(resolved) + if seen[key] { + return + } + seen[key] = true + links = append(links, resolved) + }) + return links +} + +func clipText(text string, limit int) string { + if len(text) <= limit { + return text + } + return text[:limit] +} diff --git a/workers/go/internal/crawler/safety.go b/workers/go/internal/crawler/safety.go new file mode 100644 index 00000000..4a7f80b5 --- /dev/null +++ b/workers/go/internal/crawler/safety.go @@ -0,0 +1,33 @@ +package crawler + +import "strings" + +var blockedPatterns = []struct { + substr string + reason string +}{ + {"ddos", "distributed denial-of-service behavior is not allowed."}, + {"denial of service", "denial-of-service behavior is not allowed."}, + {"dos attack", "denial-of-service behavior is not allowed."}, + {"flood", "flooding a website is not allowed."}, + {"hammer", "repeatedly hammering a website is not allowed."}, + {"stress test", "load or stress testing a third-party website is not allowed."}, + {"load test", "load or stress testing a third-party website is not allowed."}, + {"port scan", "port scanning is not a web crawl."}, + {"brute force", "brute-force activity is not allowed."}, + {"infinite loop", "unbounded or infinite crawling is not allowed."}, + {"loop forever", "unbounded or infinite crawling is not allowed."}, + {"crawl forever", "unbounded or infinite crawling is not allowed."}, + {"never stop crawling", "unbounded or infinite crawling is not allowed."}, + {"unbounded crawl", "unbounded or infinite crawling is not allowed."}, +} + +func MaliciousGoalReason(goal string) string { + normalized := strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(goal), "-", " "), "_", " ") + for _, p := range blockedPatterns { + if strings.Contains(normalized, p.substr) { + return p.reason + } + } + return "" +} diff --git a/workers/go/internal/crawler/types.go b/workers/go/internal/crawler/types.go new file mode 100644 index 00000000..98f33e50 --- /dev/null +++ b/workers/go/internal/crawler/types.go @@ -0,0 +1,69 @@ +package crawler + +// Wire types mirror worker-product.schema.json (web_crawler_result) in Structure Contract. + +const ( + DefaultMaxPages = 10 + MaximumMaxPages = 100 + DefaultMaxDepth = 2 + MaximumMaxDepth = 5 + DefaultTimeoutSeconds = 120 + MaximumTimeoutSeconds = 900 + MaximumPageBytes = 1_048_576 + MaximumTotalBytes = 10 * 1_048_576 + MaximumLinksPerPage = 200 + MaximumQueuedURLs = 400 + MaximumExtractedText = 12_000 + MaximumOutputChars = 500_000 + MaximumRedirectsPerPage = 5 + MinRequestDelayMS = 150 +) + +type Request struct { + StartURL string `json:"start_url"` + Goal string `json:"goal"` + MaxPages int `json:"max_pages"` + MaxDepth int `json:"max_depth"` + TimeoutSeconds int `json:"timeout_seconds"` + AllowedHosts []string `json:"allowed_hosts,omitempty"` +} + +type Page struct { + URL string `json:"url"` + Depth int `json:"depth"` + StatusCode int `json:"status_code"` + ContentType *string `json:"content_type,omitempty"` + Title string `json:"title"` + Text string `json:"text"` + LinksFound int `json:"links_found"` +} + +type StopReason string + +const ( + StopCompleted StopReason = "completed" + StopMaxPages StopReason = "max_pages" + StopMaxDepth StopReason = "max_depth" + StopTimeout StopReason = "timeout" + StopTotalBytes StopReason = "total_bytes" + StopQueueLimit StopReason = "queue_limit" + StopCancelled StopReason = "cancelled" + StopBlocked StopReason = "blocked" +) + +type Result struct { + OK bool `json:"ok"` + StartURL string `json:"start_url"` + Pages []Page `json:"pages"` + StopReason StopReason `json:"stop_reason"` + RequestsMade int `json:"requests_made"` + BytesRead int `json:"bytes_read"` + Truncated bool `json:"truncated"` + Diagnostics []string `json:"diagnostics"` +} + +type ProxyConfig struct { + Host string + Port int + Token string +} diff --git a/workers/go/internal/crawler/url.go b/workers/go/internal/crawler/url.go new file mode 100644 index 00000000..b3c1400d --- /dev/null +++ b/workers/go/internal/crawler/url.go @@ -0,0 +1,87 @@ +package crawler + +import ( + "net" + "net/url" + "strings" +) + +func IsHTTP(u *url.URL) bool { + scheme := strings.ToLower(u.Scheme) + return scheme == "http" || scheme == "https" +} + +func NormalizeURL(u *url.URL) *url.URL { + if u == nil { + return u + } + clone := *u + clone.Scheme = strings.ToLower(clone.Scheme) + clone.Host = strings.ToLower(clone.Host) + clone.Fragment = "" + if clone.Path == "" { + clone.Path = "/" + } + if clone.Port() != "" { + if (clone.Scheme == "http" && clone.Port() == "80") || + (clone.Scheme == "https" && clone.Port() == "443") { + clone.Host = clone.Hostname() + } + } + return &clone +} + +func URLKey(u *url.URL) string { + return NormalizeURL(u).String() +} + +func ParseStartURL(raw string) (*url.URL, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return nil, err + } + if !IsHTTP(u) { + return nil, errInvalidStartURL + } + host := strings.ToLower(strings.TrimSpace(u.Hostname())) + if host == "" { + return nil, errInvalidStartURL + } + if u.User != nil { + return nil, errInvalidStartURL + } + return NormalizeURL(u), nil +} + +func ResolvedAllowedHosts(startHost string, explicit []string) map[string]bool { + hosts := map[string]bool{strings.ToLower(startHost): true} + for _, h := range explicit { + n := strings.ToLower(strings.TrimSpace(h)) + if n != "" { + hosts[n] = true + } + } + return hosts +} + +func HostAllowed(hosts map[string]bool, host string) bool { + return hosts[strings.ToLower(strings.TrimSpace(host))] +} + +func IsPrivateHost(host string) bool { + host = strings.TrimSpace(strings.ToLower(host)) + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() +} + +var errInvalidStartURL = validationError("start_url must be an http or https URL without embedded credentials.") + +type validationError string + +func (e validationError) Error() string { return string(e) } diff --git a/workers/go/internal/crawler/validate.go b/workers/go/internal/crawler/validate.go new file mode 100644 index 00000000..ae08f384 --- /dev/null +++ b/workers/go/internal/crawler/validate.go @@ -0,0 +1,66 @@ +package crawler + +import ( + "net/url" + "strings" +) + +type ValidatedRequest struct { + StartURL *url.URL + Goal string + MaxPages int + MaxDepth int + TimeoutSeconds int + AllowedHosts map[string]bool +} + +func Validate(req Request) (ValidatedRequest, error) { + goal := strings.TrimSpace(req.Goal) + if goal == "" { + return ValidatedRequest{}, validationError("A crawl goal is required.") + } + if len(goal) > 2000 { + return ValidatedRequest{}, validationError("The crawl goal is too long.") + } + if reason := MaliciousGoalReason(goal); reason != "" { + return ValidatedRequest{}, validationError("Crawl blocked: " + reason) + } + + maxPages := req.MaxPages + if maxPages == 0 { + maxPages = DefaultMaxPages + } + if maxPages < 1 || maxPages > MaximumMaxPages { + return ValidatedRequest{}, validationError("max_pages must be between 1 and 100.") + } + + maxDepth := req.MaxDepth + if maxDepth == 0 { + maxDepth = DefaultMaxDepth + } + if maxDepth < 0 || maxDepth > MaximumMaxDepth { + return ValidatedRequest{}, validationError("max_depth must be between 0 and 5.") + } + + timeout := req.TimeoutSeconds + if timeout == 0 { + timeout = DefaultTimeoutSeconds + } + if timeout < 1 || timeout > MaximumTimeoutSeconds { + return ValidatedRequest{}, validationError("timeout_seconds must be between 1 and 900.") + } + + start, err := ParseStartURL(req.StartURL) + if err != nil { + return ValidatedRequest{}, err + } + + return ValidatedRequest{ + StartURL: start, + Goal: goal, + MaxPages: maxPages, + MaxDepth: maxDepth, + TimeoutSeconds: timeout, + AllowedHosts: ResolvedAllowedHosts(start.Hostname(), req.AllowedHosts), + }, nil +} diff --git a/workers/go/internal/extractor/engine.go b/workers/go/internal/extractor/engine.go new file mode 100644 index 00000000..6e86a4f4 --- /dev/null +++ b/workers/go/internal/extractor/engine.go @@ -0,0 +1,383 @@ +package extractor + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/xuri/excelize/v2" +) + +func Run(req Request, inputDir, outputDir string) Result { + if len(req.Files) == 0 { + return errorResult(req.Operation, "Choose at least one attached file.") + } + if len(req.Files) > MaximumFiles { + return errorResult(req.Operation, fmt.Sprintf("You can process at most %d files.", MaximumFiles)) + } + if req.OutputFormat == "" { + req.OutputFormat = FormatMarkdown + } + if req.Operation == "" { + req.Operation = OperationExtract + } + + var files []FileResult + var diagnostics []string + previewBudget := MaximumTotalPreviewChars + + for _, name := range req.Files { + safeName, err := validatedFilename(name) + if err != nil { + files = append(files, FileResult{ + InputName: name, + Kind: kindFor(name), + Error: strPtr(err.Error()), + }) + continue + } + inputPath := filepath.Join(inputDir, safeName) + if _, err := os.Stat(inputPath); err != nil { + msg := fmt.Sprintf("%s was not found in /data/in.", safeName) + files = append(files, FileResult{ + InputName: safeName, + Kind: kindFor(safeName), + Error: &msg, + }) + continue + } + + processed, err := process(inputPath, req.Operation, req.OutputFormat) + if err != nil { + msg := err.Error() + diagnostics = append(diagnostics, msg) + files = append(files, FileResult{ + InputName: safeName, + Kind: kindFor(safeName), + Error: &msg, + }) + continue + } + outPath := filepath.Join(outputDir, processed.OutputName) + if err := os.WriteFile(outPath, processed.Data, 0o644); err != nil { + msg := err.Error() + diagnostics = append(diagnostics, msg) + files = append(files, FileResult{ + InputName: safeName, + Kind: processed.Kind, + Error: &msg, + }) + continue + } + preview := clipPreview(processed.Preview, &previewBudget) + files = append(files, FileResult{ + InputName: safeName, + OutputName: &processed.OutputName, + Kind: processed.Kind, + ByteCount: len(processed.Data), + Preview: preview, + }) + } + + ok := false + for _, f := range files { + if f.Error == nil { + ok = true + break + } + } + return Result{ + OK: ok, + Operation: req.Operation, + Files: files, + Diagnostics: diagnostics, + } +} + +func errorResult(op Operation, message string) Result { + return Result{ + OK: false, + Operation: op, + Files: []FileResult{}, + Diagnostics: []string{message}, + } +} + +type processedFile struct { + OutputName string + Kind string + Data []byte + Preview string +} + +func validatedFilename(name string) (string, error) { + trimmed := strings.TrimSpace(name) + if trimmed == "" || strings.Contains(trimmed, "/") || strings.Contains(trimmed, "\\") || + strings.Contains(trimmed, "\x00") || trimmed == "." || trimmed == ".." { + return "", fmt.Errorf("%s is not a safe file name.", name) + } + base := filepath.Base(trimmed) + if base != trimmed { + return "", fmt.Errorf("%s is not a safe file name.", name) + } + return base, nil +} + +func kindFor(filename string) string { + return strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".") +} + +func process(path string, operation Operation, format OutputFormat) (processedFile, error) { + filename := filepath.Base(path) + fileKind := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".") + extracted, err := extractText(path, fileKind) + if err != nil { + return processedFile{}, err + } + + if operation == OperationExtract { + ext := "md" + body := extracted + if format == FormatTXT { + ext = "txt" + } else { + body = fmt.Sprintf("# %s\n\n%s\n", filename, extracted) + } + return processedFile{ + OutputName: replaceExt(filename, ext), + Kind: fileKind, + Data: []byte(body), + Preview: extracted, + }, nil + } + + switch format { + case FormatXLSX: + if fileKind != "csv" && fileKind != "tsv" && fileKind != "txt" { + return processedFile{}, fmt.Errorf("That conversion is not supported for %s files.", fileKind) + } + csv := extracted + if fileKind == "tsv" { + csv = strings.ReplaceAll(extracted, "\t", ",") + } + data, err := csvToXLSX(csv) + if err != nil { + return processedFile{}, err + } + return processedFile{ + OutputName: replaceExt(filename, "xlsx"), + Kind: fileKind, + Data: data, + Preview: extracted, + }, nil + case FormatCSV: + if fileKind == "xlsx" { + raw, err := os.ReadFile(path) + if err != nil { + return processedFile{}, err + } + csv, err := xlsxToCSV(raw) + if err != nil { + return processedFile{}, err + } + return processedFile{ + OutputName: replaceExt(filename, "csv"), + Kind: fileKind, + Data: []byte(csv), + Preview: csv, + }, nil + } + if fileKind != "csv" && fileKind != "tsv" && fileKind != "txt" { + return processedFile{}, fmt.Errorf("That conversion is not supported for %s files.", fileKind) + } + return processedFile{ + OutputName: replaceExt(filename, "csv"), + Kind: fileKind, + Data: []byte(extracted), + Preview: extracted, + }, nil + default: + ext := "md" + body := extracted + if format == FormatTXT { + ext = "txt" + } else { + body = fmt.Sprintf("# %s\n\n%s\n", filename, extracted) + } + return processedFile{ + OutputName: replaceExt(filename, ext), + Kind: fileKind, + Data: []byte(body), + Preview: extracted, + }, nil + } +} + +func extractText(path, kind string) (string, error) { + switch kind { + case "pdf": + return pdfToText(path) + case "docx": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return docxToText(raw) + case "xlsx": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return xlsxToCSV(raw) + case "html", "htm": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return htmlToText(string(raw)), nil + default: + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(raw), nil + } +} + +func pdfToText(path string) (string, error) { + if _, err := exec.LookPath("pdftotext"); err != nil { + return "", fmt.Errorf("PDF text extraction is unavailable in this image.") + } + out, err := exec.Command("pdftotext", "-layout", path, "-").Output() + if err != nil { + return "", fmt.Errorf("PDF text extraction failed.") + } + return string(out), nil +} + +func docxToText(data []byte) (string, error) { + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", err + } + for _, f := range reader.File { + if f.Name != "word/document.xml" { + continue + } + rc, err := f.Open() + if err != nil { + return "", err + } + defer rc.Close() + return parseDocxXML(rc) + } + return "", fmt.Errorf("document.xml missing from docx") +} + +func parseDocxXML(r io.Reader) (string, error) { + decoder := xml.NewDecoder(r) + var parts []string + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "t" { + var text string + if err := decoder.DecodeElement(&text, &se); err != nil { + return "", err + } + if text != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, ""), nil +} + +func xlsxToCSV(data []byte) (string, error) { + book, err := excelize.OpenReader(bytes.NewReader(data)) + if err != nil { + return "", err + } + sheets := book.GetSheetList() + if len(sheets) == 0 { + return "", nil + } + rows, err := book.GetRows(sheets[0]) + if err != nil { + return "", err + } + var lines []string + for _, row := range rows { + lines = append(lines, strings.Join(row, ",")) + } + return strings.Join(lines, "\n"), nil +} + +func csvToXLSX(csv string) ([]byte, error) { + book := excelize.NewFile() + sheet := book.GetSheetName(0) + lines := strings.Split(csv, "\n") + for i, line := range lines { + if line == "" { + continue + } + cells := strings.Split(line, ",") + for j, cell := range cells { + cellName, _ := excelize.CoordinatesToCellName(j+1, i+1) + _ = book.SetCellValue(sheet, cellName, cell) + } + } + buf, err := book.WriteToBuffer() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func htmlToText(html string) string { + var out strings.Builder + inTag := false + for _, r := range html { + switch { + case r == '<': + inTag = true + case r == '>': + inTag = false + out.WriteRune(' ') + case !inTag: + out.WriteRune(r) + } + } + return strings.Join(strings.Fields(out.String()), " ") +} + +func replaceExt(filename, ext string) string { + base := strings.TrimSuffix(filename, filepath.Ext(filename)) + return base + "." + ext +} + +func clipPreview(text string, remaining *int) *string { + if *remaining <= 0 { + return nil + } + limit := min(MaximumPreviewCharacters, *remaining) + preview := text + if len(preview) > limit { + preview = preview[:limit] + "…" + } + *remaining -= len(preview) + return &preview +} + +func strPtr(s string) *string { return &s } diff --git a/workers/go/internal/extractor/types.go b/workers/go/internal/extractor/types.go new file mode 100644 index 00000000..6463359a --- /dev/null +++ b/workers/go/internal/extractor/types.go @@ -0,0 +1,48 @@ +// Wire types mirror worker-product.schema.json (file_extractor_result) in Structure Contract. +package extractor + +const ( + InputDirectory = "/data/in" + OutputDirectory = "/data/out" + MaximumFiles = 5 + MaximumPreviewCharacters = 8000 + MaximumTotalPreviewChars = 32000 +) + +type Operation string + +const ( + OperationExtract Operation = "extract" + OperationConvert Operation = "convert" +) + +type OutputFormat string + +const ( + FormatMarkdown OutputFormat = "markdown" + FormatTXT OutputFormat = "txt" + FormatCSV OutputFormat = "csv" + FormatXLSX OutputFormat = "xlsx" +) + +type Request struct { + Operation Operation `json:"operation"` + OutputFormat OutputFormat `json:"output_format"` + Files []string `json:"files"` +} + +type FileResult struct { + InputName string `json:"input_name"` + OutputName *string `json:"output_name,omitempty"` + Kind string `json:"kind"` + ByteCount int `json:"byte_count"` + Preview *string `json:"preview,omitempty"` + Error *string `json:"error,omitempty"` +} + +type Result struct { + OK bool `json:"ok"` + Operation Operation `json:"operation"` + Files []FileResult `json:"files"` + Diagnostics []string `json:"diagnostics"` +}