From e522f0869390d186db74ae2d6cb7dda641d3e84e Mon Sep 17 00:00:00 2001 From: David Choi Date: Wed, 9 Sep 2026 21:57:57 -0400 Subject: [PATCH 01/12] new go container; go crawlers, script, plugins, converter --- SECURITY.md | 4 +- docker/worker/.dockerignore | 3 + docker/worker/Dockerfile | 31 ++ docs/Design.md | 2 +- .../DBRepositoryTests/DBRepositoryTests.swift | 6 +- .../DerrickBackend/DaemonRuntime.swift | 2 +- .../DerrickBackend/HITLApprovalNotifier.swift | 25 +- .../PluginMessagingIngressAdapterTests.swift | 4 +- .../DockerRunRequestValidator.swift | 66 ++- .../DockerRunnerXPCTests.swift | 21 +- .../FactoryHarness/FactoryHarnessMain.swift | 2 +- .../LiveFactoryModels.swift | 16 +- .../ReferenceSlackConnectorDraft.swift | 344 ++++++++++++---- .../MCPServer/DockerImageInspector.swift | 36 ++ .../DockerProductImagePrewarmer.swift | 51 ++- .../FileExtractorDockerExecutor.swift | 10 +- .../MCPServer/FileExtractorToolModule.swift | 13 +- .../JobOrchestrationToolModule.swift | 2 +- .../MCPServer/PluginFactoryToolModule.swift | 2 +- .../Script/GoGuestDockerExecutor.swift | 266 ++++++++++++ .../GoPluginFactoryDockerExecutor.swift | 61 +++ .../MCPServer/Script/GoScriptVerifier.swift | 17 + .../MCPServer/Script/GuestHopLoop.swift | 2 +- .../MCPServer/Script/GuestPluginRunner.swift | 14 +- .../Script/PythonGuestDockerExecutor.swift | 133 ------ .../PythonPluginFactoryDockerExecutor.swift | 41 -- .../Script/PythonScriptVerifier.swift | 13 - .../Script/ScriptExecutionRuntime.swift | 56 +-- .../Script/ScriptExecutionSupport.swift | 6 +- .../Script/ScriptExecutionToolModule.swift | 4 +- .../MCPServer/WebCrawlerDockerExecutor.swift | 13 +- .../MCPServer/WebCrawlerToolModule.swift | 18 + .../E2EEnvironment.swift | 2 +- .../SlackConnectorInstallReferenceMain.swift | 2 +- .../LiveHarnessEnvironment.swift | 2 +- .../Tests/MCPServerTests/MCPServerTests.swift | 178 +++++--- ...thonPluginFactoryDockerExecutorTests.swift | 41 -- .../Factory/GuestGoSourceValidator.swift | 35 ++ .../Factory/GuestPythonSourceValidator.swift | 44 -- .../Factory/PluginFactoryImplementation.swift | 14 +- .../Plugin/Factory/PluginFactoryRuntime.swift | 5 +- .../PluginTests/PluginFactoryTests.swift | 85 ++-- .../ContainerLifecyclePolicy.swift | 2 +- .../AppServices/ServiceHealth.swift | 9 +- .../MCPService/EffectorAdmissionPolicy.swift | 12 +- .../MCPService/MCPServiceXPC.swift | 2 +- .../PluginFactoryCreateFailureMessage.swift | 4 +- .../MCPService/PluginFactoryCreateInput.swift | 2 +- .../SharedAgentRuntime/PromptResources.swift | 10 +- .../TurnProcessContextTypes.swift | 2 +- .../Sources/Contract/GuestContract.swift | 4 + .../Contract/GuestContractValidation.swift | 8 + .../schemas/file-extractor-result.schema.json | 7 + .../schemas/guest-runtime.schema.json | 29 ++ .../schemas/web-crawler-result.schema.json | 7 + .../schemas/worker-product.schema.json | 97 +++++ .../DockerRunnerXPC/DerrickGoToolchain.swift | 78 ++++ .../DockerRunnerXPC/DockerImageDigest.swift | 47 +++ .../DockerProductImageDigests.generated.swift | 6 + .../DockerProductImagePolicy.swift | 29 +- .../DockerRunnerXPC/DockerWorkerRuntime.swift | 22 + .../MCPServer/MCPServerContractTypes.swift | 8 +- .../MCPServer/ScriptExecutionModels.swift | 4 +- .../MCPToolCatalog/AllowedMCPTool.swift | 4 +- .../Plugin/Envelope/DerrickGuestGo.swift | 106 +++++ .../Plugin/Envelope/DerrickGuestPython.swift | 81 ---- .../Factory/PluginFactoryRuntimeTypes.swift | 8 +- .../Factory/PluginFactoryTestScript.swift | 5 +- .../Plugin/Factory/PluginFactoryTypes.swift | 33 +- .../Sources/Plugin/Manifest/PluginPath.swift | 10 +- .../AppLayerServicesWireTests.swift | 22 +- .../ConnectorContractTests.swift | 2 +- .../DerrickGoToolchainTests.swift | 12 + .../StructureTests/GuestContractTests.swift | 38 ++ readme.md | 16 +- scripts/record-docker-image-digests.sh | 23 ++ scripts/sync-guest-contract-schemas.sh | 11 + .../verify-guest-contract-schemas-in-sync.sh | 15 + .../AgentServiceExportedObject.swift | 2 +- ui/AgentService/AgentServiceTurnHost.swift | 1 - .../MCPServiceDockerHelperRunner.swift | 7 +- ui/MCPService/MCPServiceToolHost.swift | 28 +- .../Conversation/ConversationPipeline.swift | 2 +- .../Job/JobNetworkPreflight.swift | 107 ++--- .../conversation_rag_instructions.md | 5 + .../Resources/mcp_tool_instructions.md | 16 +- .../Resources/script_reviewer_instructions.md | 2 +- .../Resources/web_crawler_skill.md | 6 +- .../Services/DaemonProcessHygiene.swift | 2 +- .../Support/AppBootstrapStatus.swift | 37 ++ .../DockerRunner/XPCDockerRunner.swift | 13 +- .../Egress/EgressAllowlistService.swift | 273 ------------- .../Support/PluginFactoryModels.swift | 24 +- .../TurnProcessContext.swift | 2 +- ui/ui/Jobs/DerrickNotificationService.swift | 12 - ui/ui/Jobs/HITLLiveApprovalHandlers.swift | 50 +-- ui/ui/Views/ContentView.swift | 1 - ui/ui/Views/LLMModelSettingsView.swift | 2 +- ui/uiTests/MessagingNavigationTests.swift | 4 +- ui/uiTests/PromptResourcesTests.swift | 6 +- workers/go/cmd/derrick-crawler/main.go | 103 +++++ workers/go/cmd/derrick-file-extractor/main.go | 77 ++++ workers/go/go.mod | 21 + workers/go/go.sum | 100 +++++ workers/go/internal/contract/contract.go | 120 ++++++ workers/go/internal/contract/contract_test.go | 49 +++ .../schemas/connector-contract.schema.json | 97 +++++ .../schemas/connector-params.schema.json | 20 + .../schemas/connector-result-emit.schema.json | 54 +++ .../schemas/connector-vendor.schema.json | 28 ++ .../schemas/envelope-list.schema.json | 55 +++ .../execution-context-wire.schema.json | 46 +++ .../schemas/file-extractor-result.schema.json | 7 + .../schemas/guest-runtime.schema.json | 29 ++ .../contract/schemas/hop-event.schema.json | 42 ++ .../schemas/web-crawler-result.schema.json | 7 + .../schemas/worker-product.schema.json | 97 +++++ workers/go/internal/crawler/engine.go | 372 +++++++++++++++++ workers/go/internal/crawler/safety.go | 33 ++ workers/go/internal/crawler/types.go | 69 ++++ workers/go/internal/crawler/url.go | 87 ++++ workers/go/internal/crawler/validate.go | 66 +++ workers/go/internal/extractor/engine.go | 383 ++++++++++++++++++ workers/go/internal/extractor/types.go | 48 +++ 124 files changed, 3848 insertions(+), 1231 deletions(-) create mode 100644 docker/worker/.dockerignore create mode 100644 docker/worker/Dockerfile create mode 100644 packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift create mode 100644 packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift create mode 100644 packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift create mode 100644 packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift delete mode 100644 packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift delete mode 100644 packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift delete mode 100644 packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift delete mode 100644 packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift create mode 100644 packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift delete mode 100644 packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json create mode 100644 packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift create mode 100644 packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift create mode 100644 packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift create mode 100644 packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift create mode 100644 packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift delete mode 100644 packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift create mode 100644 packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift create mode 100755 scripts/record-docker-image-digests.sh create mode 100755 scripts/sync-guest-contract-schemas.sh create mode 100755 scripts/verify-guest-contract-schemas-in-sync.sh delete mode 100644 ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift create mode 100644 workers/go/cmd/derrick-crawler/main.go create mode 100644 workers/go/cmd/derrick-file-extractor/main.go create mode 100644 workers/go/go.mod create mode 100644 workers/go/go.sum create mode 100644 workers/go/internal/contract/contract.go create mode 100644 workers/go/internal/contract/contract_test.go create mode 100644 workers/go/internal/contract/schemas/connector-contract.schema.json create mode 100644 workers/go/internal/contract/schemas/connector-params.schema.json create mode 100644 workers/go/internal/contract/schemas/connector-result-emit.schema.json create mode 100644 workers/go/internal/contract/schemas/connector-vendor.schema.json create mode 100644 workers/go/internal/contract/schemas/envelope-list.schema.json create mode 100644 workers/go/internal/contract/schemas/execution-context-wire.schema.json create mode 100644 workers/go/internal/contract/schemas/file-extractor-result.schema.json create mode 100644 workers/go/internal/contract/schemas/guest-runtime.schema.json create mode 100644 workers/go/internal/contract/schemas/hop-event.schema.json create mode 100644 workers/go/internal/contract/schemas/web-crawler-result.schema.json create mode 100644 workers/go/internal/contract/schemas/worker-product.schema.json create mode 100644 workers/go/internal/crawler/engine.go create mode 100644 workers/go/internal/crawler/safety.go create mode 100644 workers/go/internal/crawler/types.go create mode 100644 workers/go/internal/crawler/url.go create mode 100644 workers/go/internal/crawler/validate.go create mode 100644 workers/go/internal/extractor/engine.go create mode 100644 workers/go/internal/extractor/types.go diff --git a/SECURITY.md b/SECURITY.md index 470cbe9..0209604 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,7 +18,7 @@ We will acknowledge receipt and work on a fix before public disclosure when poss Derrick runs LLM agents with tools on the user's Mac. The design assumes: - **Untrusted model output** — tools and scripts are gated before execution. -- **Untrusted guest code** — Python plugins/scripts run in Docker with no network; the host performs HTTP. +- **Untrusted guest code** — Go plugins/scripts compile and run in Docker with no network; the host performs HTTP. - **Untrusted remote content** — fetched HTML and tool output are sanitized before display. - **Secrets stay on the host** — API keys and connector tokens live in Keychain or local `.env` (dev only); they are not injected into guest containers. @@ -26,7 +26,7 @@ Derrick runs LLM agents with tools on the user's Mac. The design assumes: | Layer | Mechanism | |-------|-----------| -| Script execution | Docker `--network none`, static Python verifier, LLM script reviewer | +| Script execution | Docker `--network none`, static Go verifier, in-container compile, LLM script reviewer | | Network egress | Host HTTP client, egress blacklist, user approval for new destinations | | Plugins | Factory build + review; hop-limited `http.request`; Keychain-attached auth | | Inter-process | Code-signed XPC peers, HMAC-signed service messages (release: Keychain secret) | diff --git a/docker/worker/.dockerignore b/docker/worker/.dockerignore new file mode 100644 index 0000000..b01fe14 --- /dev/null +++ b/docker/worker/.dockerignore @@ -0,0 +1,3 @@ +**/.git +**/.build +**/node_modules diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile new file mode 100644 index 0000000..bab3814 --- /dev/null +++ b/docker/worker/Dockerfile @@ -0,0 +1,31 @@ +# Unified Go worker image: web crawl, file extract (plugin guest binaries are copied per invoke). +# Build: docker build -f docker/worker/Dockerfile -t derrick-worker:go-v1 . +FROM golang:1.27.1 AS build + +WORKDIR /src +COPY workers/go/go.mod workers/go/go.sum ./ +RUN go mod download +COPY workers/go ./ + +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-web-crawler ./cmd/derrick-crawler +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-file-extractor ./cmd/derrick-file-extractor + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends poppler-utils ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 worker + +COPY --from=build /usr/local/go /usr/local/go +COPY --from=build /out/derrick-web-crawler /usr/local/bin/derrick-web-crawler +COPY --from=build /out/derrick-file-extractor /usr/local/bin/derrick-file-extractor +RUN mkdir -p /data/in /data/out && chown -R worker:worker /data + +ENV PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV GOROOT=/usr/local/go +ENV GOTOOLCHAIN=local +ENV CGO_ENABLED=0 + +USER worker +WORKDIR /home/worker diff --git a/docs/Design.md b/docs/Design.md index 6bcc9bf..5a9171d 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -10,4 +10,4 @@ This application is Protocol first. All major features must have a Protocol and ## Plugins - Plugins are using the Agent Plugin standard -- Plugins run in the plugins docker containers as either Python or Go (Go is not yet supported) +- Plugins and `script_exec` run in the unified Go worker Docker image (`derrick-worker:go-v1`) diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift index a546a2c..9adf82a 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift @@ -85,7 +85,7 @@ final class DBRepositoryTests: XCTestCase { let files: [String: Data] = [ "plugin.json": Data(#"{"name":"keep-me"}"#.utf8), "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), + "app.derrick/plugin.go": Data("package main".utf8), "app.derrick/plugin": artifact, ] let release = PluginFactoryRelease( @@ -124,7 +124,7 @@ final class DBRepositoryTests: XCTestCase { let files: [String: Data] = [ "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), + "app.derrick/plugin.go": Data("package main".utf8), "app.derrick/plugin": artifact, "skills/weather/SKILL.md": Data("# Weather".utf8), ] @@ -169,7 +169,7 @@ final class DBRepositoryTests: XCTestCase { let files: [String: Data] = [ "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.py": Data("print(\"[]\")".utf8), + "app.derrick/plugin.go": Data("package main".utf8), "app.derrick/plugin": artifact, "skills/weather/SKILL.md": Data("# Weather".utf8), ] diff --git a/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift b/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift index 22131ac..96c27f9 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/DaemonRuntime.swift @@ -52,7 +52,7 @@ public actor DaemonRuntime { service: .daemon, status: ok ? .ok : .degraded, detail: ok ? nil : "not bootstrapped", - guestRuntimeImage: DerrickGuestRuntime.pythonGuestDockerImage, + guestRuntimeImage: DerrickGuestRuntime.guestDockerImage, executableFingerprint: DaemonSelfRetirement.launchedFingerprint ) } diff --git a/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift b/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift index e617f6a..a74708e 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/HITLApprovalNotifier.swift @@ -31,11 +31,15 @@ public enum HITLApprovalNotifier: Sendable { let isNetwork = isNetworkToolName(row.toolName) let host = host(fromNetworkToolName: row.toolName) - let title = isNetwork ? "Network access needed" : "Approval needed" + let blacklistPattern = blacklistPattern(from: row.argumentsJSON) + let title = isNetwork + ? (blacklistPattern == nil ? "Network access needed" : "Network blacklist") + : "Approval needed" let body: String - if isNetwork, let host { - let suffix = Self.registrableSuffix(for: host) - body = "Allow *.\(suffix)? Tap to approve or deny. Always Allow covers all subdomains." + if isNetwork, let blacklistPattern { + body = "This request matches blacklist \(blacklistPattern). Tap to allow this run, remove from blacklist, or deny." + } else if isNetwork, let host { + body = "Network access to \(host). Tap to approve or deny." } else { let preview = truncated(row.argumentsJSON, limit: 160) body = preview.isEmpty @@ -80,12 +84,13 @@ public enum HITLApprovalNotifier: Sendable { return host.isEmpty ? nil : host } - /// Last two labels — same rule as egress permanent allow (`*.apple.com` ← `securemetrics.apple.com`). - private static func registrableSuffix(for host: String) -> String { - let normalized = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - let parts = normalized.split(separator: ".").map(String.init) - guard parts.count >= 2 else { return normalized } - return parts.suffix(2).joined(separator: ".") + private static func blacklistPattern(from argumentsJSON: String) -> String? { + guard let data = argumentsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + object["kind"] as? String == "blacklist" else { + return nil + } + return object["pattern"] as? String } private static func truncated(_ text: String, limit: Int) -> String { diff --git a/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift b/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift index c2c11ed..717f1f0 100644 --- a/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift +++ b/packages/DerrickBackend/Tests/DerrickBackendTests/PluginMessagingIngressAdapterTests.swift @@ -542,10 +542,10 @@ import Testing _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["send_message"]}}} """ let guestSource = "import json, sys\njson.dump([], sys.stdout)" - let runtimeJSON = #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + let runtimeJSON = #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# let draft = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", diff --git a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift index 478c0ad..ec7cd57 100644 --- a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift +++ b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift @@ -93,11 +93,9 @@ public enum DockerRunRequestValidator: Sendable { return .disallowedDockerSubcommand(subcommand) } - if subcommand == "image" { - guard let second = dockerArgs.dropFirst().first, - DockerHostLaunch.allowedImageSubcommands.contains(second) else { - return .disallowedDockerSubcommand("image \(dockerArgs.dropFirst().first ?? "")") - } + if subcommand == "image", + let error = validateImageArguments(dockerArgs) { + return error } if subcommand == "exec", let error = validateExecArguments(dockerArgs) { @@ -206,20 +204,17 @@ public enum DockerRunRequestValidator: Sendable { case "sh": guard args.count == 3, args[1] == "-c", - args[2] == "cat > /tmp/guest.py" + args[2] == "cat > /tmp/guest && chmod +x /tmp/guest" + || args[2] == DockerWorkerRuntime.guestWriteSourceShell + || args[2] == DockerWorkerRuntime.guestCompileShell + || args[2] == DockerWorkerRuntime.guestReadBinaryShell else { return .disallowedDockerFlag("exec \(command)") } - case "python3": - guard args.count == 2, args[1] == "/tmp/guest.py" else { - return .disallowedDockerFlag("exec \(command)") - } - case "/usr/local/bin/derrick-web-crawler": - guard args == ["/usr/local/bin/derrick-web-crawler"] else { - return .disallowedDockerFlag("exec \(command)") - } - case "/usr/local/bin/derrick-file-extractor": - guard args == ["/usr/local/bin/derrick-file-extractor"] else { + case DockerWorkerRuntime.crawlerBinary, + DockerWorkerRuntime.extractorBinary, + DockerWorkerRuntime.guestBinaryPath: + guard args == [command] else { return .disallowedDockerFlag("exec \(command)") } default: @@ -244,12 +239,45 @@ public enum DockerRunRequestValidator: Sendable { guard args.count == 5 else { return .disallowedDockerFlag("build extra arguments") } - guard DockerProductImagePolicy.isAllowedWebCrawlerBuild( + if DockerProductImagePolicy.isAllowedWorkerBuild( + dockerfilePath: dockerfile, + imageTag: tag, + contextPath: context + ) { + return nil + } + if DockerProductImagePolicy.isAllowedWebCrawlerBuild( dockerfilePath: dockerfile, imageTag: tag, contextPath: context - ) else { - return .disallowedDockerFlag("build product image") + ) { + return nil + } + return .disallowedDockerFlag("build product image") + } + + private static func validateImageArguments( + _ dockerArgs: [String] + ) -> DockerRunRequestValidationError? { + let args = Array(dockerArgs.dropFirst()) + guard let second = args.first, + DockerHostLaunch.allowedImageSubcommands.contains(second) else { + return .disallowedDockerSubcommand("image \(args.first ?? "")") + } + guard second == "inspect" else { return nil } + if args.count == 2 { + return nil + } + guard args.count == 4, + args[1] == "--format", + args[2] == "{{.Id}}" else { + return .disallowedDockerFlag("image inspect") + } + let tag = args[3] + guard tag == DockerWorkerRuntime.image + || tag == DockerProductImagePolicy.webCrawlerImage + || tag == DerrickGuestRuntime.guestDockerImage else { + return .disallowedDockerFlag("image inspect tag") } return nil } diff --git a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift index e806f4a..7daa176 100644 --- a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift +++ b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift @@ -211,8 +211,11 @@ struct DockerRunnerXPCTests { ["version"], ["image", "inspect", "img"], ["pull", "img"], - ["exec", "-i", "c", "python3", "/tmp/guest.py"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest.py"], + ["exec", "-i", "c", DockerWorkerRuntime.guestBinaryPath], + ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest && chmod +x /tmp/guest"], + ["exec", "-i", "c", "sh", "-c", DockerWorkerRuntime.guestWriteSourceShell], + ["exec", "c", "sh", "-c", DockerWorkerRuntime.guestCompileShell], + ["exec", "c", "sh", "-c", DockerWorkerRuntime.guestReadBinaryShell], ["exec", "-i", "c", "/usr/local/bin/derrick-web-crawler"], ["create", "--label", "app.derrick=runtime", "--entrypoint", "/bin/sleep", "--name", "c", "derrick-web-crawler:swift-6.4-v1", "infinity"], [ @@ -229,7 +232,7 @@ struct DockerRunnerXPCTests { "--memory", "1g", "--security-opt", "no-new-privileges", "--cap-drop", "ALL", - "python:3.14.7", + DockerWorkerRuntime.image, "/bin/sleep", "infinity", ], @@ -256,13 +259,13 @@ struct DockerRunnerXPCTests { } } - @Test func rejectsInvalidPythonGuestExecCommands() { + @Test func rejectsInvalidGoGuestExecCommands() { for args in [ - ["exec", "-i", "c", "python3", "/tmp/other.py"], - ["exec", "-i", "c", "python3", "/tmp/guest.py", "extra"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/other.py"], + ["exec", "-i", "c", "python3", "/tmp/guest.py"], + ["exec", "-i", "c", DockerWorkerRuntime.guestBinaryPath, "extra"], + ["exec", "-i", "c", "sh", "-c", "cat > /tmp/other"], ["exec", "-i", "c", "sh", "-c", "rm -rf /"], - ["exec", "-i", "c", "sh", "-c", "cat > /tmp/guest.py", "extra"], + ["exec", "-i", "c", "sh", "-c", "go build /tmp/plugin.go"], ["exec", "-i", "c", "swift", "/tmp/plugin.swift"], ["exec", "-i", "c", "/tmp/plugin"], ["exec", "-i", "c", "swiftc", "-O", "/tmp/plugin.swift", "-o", "/tmp/plugin"], @@ -354,7 +357,7 @@ struct DockerRunnerXPCTests { @Test func rejectsCreateWithoutRuntimeLabel() { let r = request(arguments: DockerHostLaunch.dockerCLIArguments([ - "create", "--name", "c", "python:3.14.7", "/bin/sleep", "infinity", + "create", "--name", "c", DockerWorkerRuntime.image, "/bin/sleep", "infinity", ])) #expect(DockerRunRequestValidator.validate(r) == .disallowedDockerFlag("create missing runtime label")) } diff --git a/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift b/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift index 0359bf1..dfe1be7 100644 --- a/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift +++ b/packages/MCPServer/Sources/FactoryHarness/FactoryHarnessMain.swift @@ -26,7 +26,7 @@ enum FactoryHarnessMain { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("FactoryHarness: building slack full-sync connector…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: DirectShellDocker.executor()) + let executor = GoPluginFactoryDockerExecutor(executor: DirectShellDocker.executor()) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 3) ).build( diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift index 889e2aa..af2a46c 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift @@ -34,7 +34,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { """ The host already assigned plugin_id \(host.pluginID) and these secret ids: \ \(host.secrets.map(\.id).joined(separator: ", ")). \ - Return python_source and test_input_json. Do not pick a different plugin_id or secret ids. + Return go_source and test_input_json. Do not pick a different plugin_id or secret ids. """ ) } @@ -76,7 +76,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), python_source (string), + plugin_id (string), version (string), description (string), go_source (string), test_input_json (string containing valid JSON — a serialized object, not prose), skill_files (array of objects with path and body), secrets (array of objects with id, label, and kind; required for connector plugins), @@ -87,7 +87,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { If the plugin needs a username, password, token, or API key, declare them in secrets. kind must be username, password, token, or api_key. id is a stable Keychain key such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in python_source. + Never put real credentials in go_source. Set role to "connector" when the plugin sends and receives messages with an external messaging service (any chat or mail connector). Omit role or use "standard" otherwise. For role connector, include messaging_ops: an array of implemented ops @@ -95,12 +95,12 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { Do not return manifest_json. The host creates the canonical Agent Plugin manifest. If skill_files is not needed, return an empty array. Every skill file path must be exactly skills//SKILL.md. Never use manifest.json or other paths in skill_files. - \(DerrickGuestPython.modelContract) + \(DerrickGuestGo.modelContract) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) Before returning the draft, self-check the implementation: - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - Match host responses by the emitted request_id. - - Use only the Python standard library (no pip, requests, urllib, socket, or subprocess). + - Use only the Go standard library (no net/http, os/exec, or filesystem access). - The direct test input must exercise the terminal result path with matching http_results fixtures. For messaging connector plugins (role connector) that call a vendor HTTP API: - Declare secrets in the manifest only. Never hard-code credentials. @@ -120,7 +120,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { "plugin_id": AgentSchema(type: .string), "version": AgentSchema(type: .string), "description": AgentSchema(type: .string), - "python_source": AgentSchema(type: .string), + "go_source": AgentSchema(type: .string), "test_input_json": AgentSchema(type: .string), "skill_files": AgentSchema( type: .array, @@ -149,7 +149,7 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { "messaging_ops": AgentSchema(type: .array, items: AgentSchema(type: .string)), ], required: [ - "plugin_id", "version", "description", "python_source", + "plugin_id", "version", "description", "go_source", "test_input_json", "skill_files", ] ) @@ -181,7 +181,7 @@ public actor LiveFactoryReviewer: PluginFactoryReviewer { test_input_json: \(String(decoding: draft.testInput, as: UTF8.self)) - Python source: + Go source: \(draft.guestSource) Direct test output: diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift index 8e10253..54110bc 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/ReferenceSlackConnectorDraft.swift @@ -41,7 +41,7 @@ public enum ReferenceSlackConnectorDraft { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ "description":"Slack messaging connector",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"permissions":["channels:history","channels:read","chat:write","groups:history","groups:read","im:history","im:read","mpim:history","mpim:read","users:read"],"messaging_ops":[\(opsJSON)]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"permissions":["channels:history","channels:read","chat:write","groups:history","groups:read","im:history","im:read","mpim:history","mpim:read","users:read"],"messaging_ops":[\(opsJSON)]}}} """ return PluginFactoryDraft( manifestJSON: manifestJSON, @@ -52,104 +52,278 @@ public enum ReferenceSlackConnectorDraft { } private static let fullSyncSource = """ - import json, sys - def emit(x): json.dump(x, sys.stdout, separators=(",", ":")) - def sorted_results(results): - seen = set() - out = [] - for item in sorted(results or [], key=lambda r: str(r.get("request_id",""))): - rid = str(item.get("request_id","")) - if rid and rid not in seen: - seen.add(rid) - out.append(item) + package main + + import ( + "encoding/json" + "os" + "sort" + ) + + func emit(v any) { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) + } + + func sortedResults(results []map[string]any) []map[string]any { + if len(results) == 0 { + return nil + } + sort.Slice(results, func(i, j int) bool { + left, _ := results[i]["request_id"].(string) + right, _ := results[j]["request_id"].(string) + return left < right + }) + seen := map[string]bool{} + out := []map[string]any{} + for _, item := range results { + rid, _ := item["request_id"].(string) + if rid == "" || seen[rid] { + continue + } + seen[rid] = true + out = append(out, item) + } return out - def message_from(msg, default_channel): - if not isinstance(msg, dict): return None - channel = msg.get("channel") or default_channel - ts = msg.get("ts") - if not channel or not ts: return None - thread_ts = msg.get("thread_ts") - parent = None - if thread_ts and str(thread_ts) != str(ts): - parent = str(thread_ts) - reply_count = int(msg.get("reply_count") or 0) - row = {"vendor_thread_id":channel,"vendor_message_id":str(ts),"direction":"inbound","sender":msg.get("user") or "slack","body":msg.get("text") or "","created_at":str(ts),"reply_count":reply_count} - if parent: + } + + func asMap(v any) map[string]any { + if m, ok := v.(map[string]any); ok { + return m + } + return map[string]any{} + } + + func asString(v any) string { + if s, ok := v.(string); ok { + return s + } + return "" + } + + func messageFrom(msg map[string]any, defaultChannel string) map[string]any { + channel := asString(msg["channel"]) + if channel == "" { + channel = defaultChannel + } + ts := asString(msg["ts"]) + if channel == "" || ts == "" { + return nil + } + threadTS := asString(msg["thread_ts"]) + parent := "" + if threadTS != "" && threadTS != ts { + parent = threadTS + } + replyCount := 0 + if n, ok := msg["reply_count"].(float64); ok { + replyCount = int(n) + } + row := map[string]any{ + "vendor_thread_id": channel, + "vendor_message_id": ts, + "direction": "inbound", + "sender": func() string { + if s := asString(msg["user"]); s != "" { + return s + } + return "slack" + }(), + "body": asString(msg["text"]), + "created_at": ts, + "reply_count": replyCount, + } + if parent != "" { row["parent_vendor_message_id"] = parent + } return row - def main(): - event = json.load(sys.stdin) - params = event.get("params") or {} - op = params.get("messaging_op") - if event.get("kind") == "manual" and op == "sync_threads": - emit([{"verb":"http.request","request_id":"sync-1","method":"GET","url":"https://slack.com/api/conversations.list?types=public_channel,private_channel&limit=200&exclude_archived=true","headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + + func main() { + var event map[string]any + if err := json.NewDecoder(os.Stdin).Decode(&event); err != nil { return - if event.get("kind") == "manual" and op == "poll_inbox": - channel = params.get("vendor_thread_id") or params.get("channel") - parent = params.get("parent_vendor_message_id") or params.get("thread_ts") - if not channel: - emit([{"verb":"result.emit","messages":[]}]) + } + params := asMap(event["params"]) + op := asString(params["messaging_op"]) + kind := asString(event["kind"]) + + switch { + case kind == "manual" && op == "sync_threads": + emit([]map[string]any{{ + "verb": "http.request", "request_id": "sync-1", "method": "GET", + "url": "https://slack.com/api/conversations.list?types=public_channel,private_channel&limit=200&exclude_archived=true", + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) + return + case kind == "manual" && op == "poll_inbox": + channel := asString(params["vendor_thread_id"]) + if channel == "" { + channel = asString(params["channel"]) + } + parent := asString(params["parent_vendor_message_id"]) + if parent == "" { + parent = asString(params["thread_ts"]) + } + if channel == "" { + emit([]map[string]any{{"verb": "result.emit", "messages": []map[string]any{}}}) return - if parent: - url = "https://slack.com/api/conversations.replies?channel=" + str(channel) + "&ts=" + str(parent) + "&limit=50" - emit([{"verb":"http.request","request_id":"replies-1","method":"GET","url":url,"headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + if parent != "" { + url := "https://slack.com/api/conversations.replies?channel=" + channel + "&ts=" + parent + "&limit=50" + emit([]map[string]any{{ + "verb": "http.request", "request_id": "replies-1", "method": "GET", "url": url, + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) return - url = "https://slack.com/api/conversations.history?channel=" + str(channel) + "&limit=50" - emit([{"verb":"http.request","request_id":"poll-1","method":"GET","url":url,"headers":{"Authorization":"Bearer {{secret:bot_token}}"}}]) + } + url := "https://slack.com/api/conversations.history?channel=" + channel + "&limit=50" + emit([]map[string]any{{ + "verb": "http.request", "request_id": "poll-1", "method": "GET", "url": url, + "headers": map[string]any{"Authorization": "Bearer {{secret:bot_token}}"}, + }}) return - if event.get("kind") == "message_in_room" and op == "send_message": - channel = params.get("vendor_thread_id") - text = params.get("text", "") - parent = params.get("parent_vendor_message_id") or params.get("thread_ts") - payload = {"channel":channel,"text":text} - if parent: + case kind == "message_in_room" && op == "send_message": + channel := asString(params["vendor_thread_id"]) + text := asString(params["text"]) + parent := asString(params["parent_vendor_message_id"]) + if parent == "" { + parent = asString(params["thread_ts"]) + } + payload := map[string]any{"channel": channel, "text": text} + if parent != "" { payload["thread_ts"] = parent - emit([{"verb":"http.request","request_id":"send-1","method":"POST","url":"https://slack.com/api/chat.postMessage","headers":{"Authorization":"Bearer {{secret:bot_token}}","Content-Type":"application/json"},"json":payload}]) + } + emit([]map[string]any{{ + "verb": "http.request", "request_id": "send-1", "method": "POST", + "url": "https://slack.com/api/chat.postMessage", + "headers": map[string]any{ + "Authorization": "Bearer {{secret:bot_token}}", + "Content-Type": "application/json", + }, + "json": payload, + }}) return - if event.get("kind") == "http_results" and op == "sync_threads": - threads = [] - for item in sorted_results(event.get("http_results")): - if item.get("request_id") != "sync-1": continue - payload = json.loads(item.get("body") or "{}") - if payload.get("ok") is False: - emit([{"verb":"result.emit","title":"Slack list failed","summary":str(payload.get("error") or "unknown")}]) + case kind == "http_results" && op == "sync_threads": + threads := []map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + if asString(item["request_id"]) != "sync-1" { + continue + } + var payload map[string]any + _ = json.Unmarshal([]byte(asString(item["body"])), &payload) + if payload["ok"] == false { + emit([]map[string]any{{ + "verb": "result.emit", + "title": "Slack list failed", + "summary": asString(payload["error"]), + }}) return - for ch in sorted(payload.get("channels") or [], key=lambda c: str(c.get("id",""))): - if ch.get("is_member") is False: + } + channels, _ := payload["channels"].([]any) + sort.Slice(channels, func(i, j int) bool { + left := asMap(channels[i]) + right := asMap(channels[j]) + return asString(left["id"]) < asString(right["id"]) + }) + for _, chAny := range channels { + ch := asMap(chAny) + if ch["is_member"] == false { continue - cid = ch.get("id") - name = ch.get("name") or cid - if cid: - threads.append({"vendor_thread_id":cid,"title":"#" + str(name)}) - emit([{"verb":"result.emit","threads":threads}]) + } + cid := asString(ch["id"]) + name := asString(ch["name"]) + if name == "" { + name = cid + } + if cid != "" { + threads = append(threads, map[string]any{ + "vendor_thread_id": cid, + "title": "#" + name, + }) + } + } + } + emit([]map[string]any{{"verb": "result.emit", "threads": threads}}) return - if event.get("kind") == "http_results" and op == "poll_inbox": - channel = params.get("vendor_thread_id") or params.get("channel") - messages = [] - for item in sorted_results(event.get("http_results")): - if item.get("request_id") not in ("poll-1", "replies-1"): continue - payload = json.loads(item.get("body") or "{}") - if payload.get("ok") is False: - emit([{"verb":"result.emit","title":"Slack blocked this thread","summary":str(payload.get("error") or "unknown")}]) + case kind == "http_results" && op == "poll_inbox": + channel := asString(params["vendor_thread_id"]) + if channel == "" { + channel = asString(params["channel"]) + } + messages := []map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + rid := asString(item["request_id"]) + if rid != "poll-1" && rid != "replies-1" { + continue + } + var payload map[string]any + _ = json.Unmarshal([]byte(asString(item["body"])), &payload) + if payload["ok"] == false { + emit([]map[string]any{{ + "verb": "result.emit", + "title": "Slack blocked this thread", + "summary": asString(payload["error"]), + }}) return - for msg in sorted(payload.get("messages") or [], key=lambda m: str(m.get("ts",""))): - parsed = message_from(msg, channel) - if parsed: messages.append(parsed) - emit([{"verb":"result.emit","messages":messages}]) + } + rawMessages, _ := payload["messages"].([]any) + sort.Slice(rawMessages, func(i, j int) bool { + left := asMap(rawMessages[i]) + right := asMap(rawMessages[j]) + return asString(left["ts"]) < asString(right["ts"]) + }) + for _, msgAny := range rawMessages { + parsed := messageFrom(asMap(msgAny), channel) + if parsed != nil { + messages = append(messages, parsed) + } + } + } + emit([]map[string]any{{"verb": "result.emit", "messages": messages}}) return - if event.get("kind") == "http_results" and op == "send_message": - body = {} - for item in sorted_results(event.get("http_results")): - if item.get("request_id") == "send-1": - body = json.loads(item.get("body") or "{}") - if body.get("ok"): - ts = body.get("ts") or (body.get("message") or {}).get("ts") - emit([{"verb":"result.emit","sent_message":{"vendor_message_id":ts,"created_at":ts}}]) - else: - emit([{"verb":"result.emit","summary":"send failed"}]) + case kind == "http_results" && op == "send_message": + body := map[string]any{} + rawResults, _ := event["http_results"].([]any) + results := []map[string]any{} + for _, item := range rawResults { + results = append(results, asMap(item)) + } + for _, item := range sortedResults(results) { + if asString(item["request_id"]) == "send-1" { + _ = json.Unmarshal([]byte(asString(item["body"])), &body) + } + } + if body["ok"] == true { + ts := asString(body["ts"]) + if ts == "" { + ts = asString(asMap(body["message"])["ts"]) + } + emit([]map[string]any{{ + "verb": "result.emit", + "sent_message": map[string]any{ + "vendor_message_id": ts, + "created_at": ts, + }, + }}) + } else { + emit([]map[string]any{{"verb": "result.emit", "summary": "send failed"}}) + } return - emit([{"verb":"result.emit","summary":"unsupported"}]) - if __name__ == "__main__": - main() + default: + emit([]map[string]any{{"verb": "result.emit", "summary": "unsupported"}}) + } + } """ } diff --git a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift new file mode 100644 index 0000000..63835ad --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift @@ -0,0 +1,36 @@ +import Foundation +import Structure + +/// Reads and verifies pinned Docker product image digests. +public enum DockerImageInspector: Sendable { + public static func localImageID( + tag: String, + executor: @escaping DockerCLIExecutor + ) async throws -> DockerImageDigest { + let response = try await executor( + ["image", "inspect", "--format", "{{.Id}}", tag], + Data(), + 30 + ) + guard response.exitCode == 0 else { + throw DockerImageDigestError.imageMissing(tag) + } + let raw = String(decoding: response.stdout, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard let digest = DockerImageDigest(hexDigest: raw) else { + throw DockerImageDigestError.imageMissing(tag) + } + return digest + } + + public static func verifyPinned( + tag: String, + expected: DockerImageDigest, + executor: @escaping DockerCLIExecutor + ) async throws { + let actual = try await localImageID(tag: tag, executor: executor) + guard actual == expected else { + throw DockerImageDigestError.digestMismatch(tag: tag, expected: expected, actual: actual) + } + } +} diff --git a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift index 3522edc..fcff866 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift @@ -2,29 +2,45 @@ import Foundation import DockerRunnerXPC import Structure -/// Ensures trusted product Docker images exist (pull or local build). +/// Ensures trusted product Docker images exist (local build) and match pinned digests. public enum DockerProductImagePrewarmer: Sendable { - public static func ensureWebCrawlerImage( + public static func ensureWorkerImage( executor: @escaping DockerCLIExecutor ) async throws { try await ensureImage( - tag: DockerProductImagePolicy.webCrawlerImage, - dockerfileRelativePath: DockerProductImagePolicy.webCrawlerDockerfileRelativePath, - contextRelativePath: DockerProductImagePolicy.webCrawlerBuildContextRelativePath, + tag: DockerProductImagePolicy.workerImage, + dockerfileRelativePath: DockerProductImagePolicy.workerDockerfileRelativePath, + contextRelativePath: DockerProductImagePolicy.workerBuildContextRelativePath, + pinnedDigest: DockerWorkerRuntime.pinnedDigest, + buildValidator: DockerProductImagePolicy.isAllowedWorkerBuild, executor: executor, buildTimeoutSeconds: 1_200 ) } + /// Legacy alias used by crawler startup paths. + public static func ensureWebCrawlerImage( + executor: @escaping DockerCLIExecutor + ) async throws { + try await ensureWorkerImage(executor: executor) + } + public static func ensureImage( tag: String, dockerfileRelativePath: String, contextRelativePath: String, + pinnedDigest: DockerImageDigest, + buildValidator: @escaping (String, String, String) -> Bool, executor: @escaping DockerCLIExecutor, buildTimeoutSeconds: Int = 1_200 ) async throws { let inspect = try await executor(["image", "inspect", tag], Data(), 30) if inspect.exitCode == 0 { + try await DockerImageInspector.verifyPinned( + tag: tag, + expected: pinnedDigest, + executor: executor + ) return } @@ -39,6 +55,9 @@ public enum DockerProductImagePrewarmer: Sendable { guard FileManager.default.fileExists(atPath: context.path) else { throw DockerProductImagePrewarmerError.dockerfileMissing(context.path) } + guard buildValidator(dockerfile.path, tag, context.path) else { + throw DockerProductImagePrewarmerError.buildFailed(tag, "build policy rejected image build") + } let build = try await executor( [ @@ -58,14 +77,16 @@ public enum DockerProductImagePrewarmer: Sendable { detail.isEmpty ? "exit \(build.exitCode)" : detail ) } + + try await DockerImageInspector.verifyPinned( + tag: tag, + expected: pinnedDigest, + executor: executor + ) } } -/// One in-flight crawler image build per process. -/// -/// Chat and daemon start this in the background. A `web.crawl` that arrives -/// while it is still running waits on the same task and does not start a second -/// `docker build`. +/// One in-flight worker image build per process. public actor WebCrawlerImageGate { public static let shared = WebCrawlerImageGate() @@ -79,7 +100,7 @@ public actor WebCrawlerImageGate { return } let task = Task { - try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) + try await DockerProductImagePrewarmer.ensureWorkerImage(executor: executor) } inFlight = task do { @@ -97,8 +118,6 @@ public enum DockerProductImagePrewarmerError: Error, LocalizedError, Equatable, case dockerfileMissing(String) case buildFailed(String, String) - /// First compiler `error:` line, if the docker build log has one. Not shown as the - /// user-facing `errorDescription` (that stays a short human sentence). public var compilerDiagnostic: String? { switch self { case .buildFailed(_, let detail): @@ -111,11 +130,11 @@ public enum DockerProductImagePrewarmerError: Error, LocalizedError, Equatable, public var errorDescription: String? { switch self { case .repositoryRootNotFound: - return "The web crawler image is not installed and Derrick could not find its source to build it." + return "The worker image is not installed and Derrick could not find its source to build it." case .dockerfileMissing: - return "The web crawler image is not installed and Derrick could not find its build files." + return "The worker image is not installed and Derrick could not find its build files." case .buildFailed: - return "Derrick could not build the web crawler image. Make sure Docker Desktop is running, has enough disk space, and can reach the network." + return "Derrick could not build the worker image. Make sure Docker Desktop is running, has enough disk space, and can reach the network." } } diff --git a/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift index 21abfdc..0d85bbf 100644 --- a/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift +++ b/packages/MCPServer/Sources/MCPServer/FileExtractorDockerExecutor.swift @@ -5,8 +5,9 @@ import Structure /// Own queue (max 1). Job folders are bind-mounted; the image must already /// exist (`docker image inspect` happens outside the permit). public struct FileExtractorDockerExecutor: Sendable { - public static let image = "derrick-file-extractor:swift-6.4-v1" + public static let image = DockerWorkerRuntime.image public static let containerPrefix = "derrick-file-extractor" + public static let binaryPath = DockerWorkerRuntime.extractorBinary public static let maximumTimeoutSeconds = 180 private let executor: DockerCLIExecutor @@ -27,10 +28,7 @@ public struct FileExtractorDockerExecutor: Sendable { timeoutSeconds: Int ) async throws -> DockerCLIResult { let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) - let imageCheck = try await executor(["image", "inspect", Self.image], Data(), 30) - guard imageCheck.exitCode == 0 else { - throw FileExtractorDockerExecutorError.imageUnavailable(Self.image) - } + try await WorkerImageGate.shared.ensureReady(executor: executor) let executor = self.executor do { return try await queue.withPermit { @@ -48,7 +46,7 @@ public struct FileExtractorDockerExecutor: Sendable { startStep: "start file extractor container", body: { name in try await executor( - ["exec", "-i", name, "/usr/local/bin/derrick-file-extractor"], + ["exec", "-i", name, Self.binaryPath], input, timeout ) diff --git a/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift b/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift index 21b09ba..35edd1f 100644 --- a/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/FileExtractorToolModule.swift @@ -64,8 +64,17 @@ public enum FileExtractorToolModule: MCPToolModule { let payload = try JSONEncoder().encode(workerRequest) let dockerResult = try await run(payload, workspace, parsed.timeoutSeconds) let workerJSON = String(decoding: dockerResult.stdout, as: UTF8.self) - let worker = (try? JSONDecoder().decode(FileExtractorWireResult.self, from: dockerResult.stdout)) - ?? FileExtractorWireResult(ok: false, files: [], diagnostics: [workerJSON]) + let worker: FileExtractorWireResult + if (try? GuestContractValidation.validateFileExtractorResultJSON(dockerResult.stdout)) != nil { + worker = (try? JSONDecoder().decode(FileExtractorWireResult.self, from: dockerResult.stdout)) + ?? FileExtractorWireResult(ok: false, files: [], diagnostics: [workerJSON]) + } else { + worker = FileExtractorWireResult( + ok: false, + files: [], + diagnostics: ["File extractor returned invalid JSON output."] + ) + } let exported = (try? workspace.publishOutputs()) ?? [] if dockerResult.exitCode != 0 && !worker.ok { return try failure( diff --git a/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift b/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift index 629d0bc..abdae01 100644 --- a/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/Orchestration/JobOrchestrationToolModule.swift @@ -34,7 +34,7 @@ public enum JobOrchestrationToolModule { ]), "tool_arguments": .object([ "type": .string("object"), - "description": .string("Frozen effector args. For web.crawl use {start_url,goal,max_pages,max_depth,timeout_seconds}. For script_exec use {description,reason,script} where script is standalone Python reading JSON from stdin and writing Derrick envelope JSON to stdout.") + "description": .string("Frozen effector args. For web.crawl use {start_url,goal,max_pages,max_depth,timeout_seconds}. For script_exec use {description,reason,script} where script is standalone Go reading JSON from stdin and writing Derrick envelope JSON to stdout.") ]), "wake_after": .object([ "type": .string("boolean"), diff --git a/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift b/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift index baf82d7..3bcb736 100644 --- a/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/PluginFactoryToolModule.swift @@ -18,7 +18,7 @@ public enum PluginFactoryToolModule: MCPToolModule { ]), "host_manifest_json": .object([ "type": .string("string"), - "description": .string("Host-owned Agent Plugin plugin.json. When set, the builder only supplies Python and tests.") + "description": .string("Host-owned Agent Plugin plugin.json. When set, the builder only supplies Go source and tests.") ]), ]), "required": .array([.string("goal")]) diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift new file mode 100644 index 0000000..2c23020 --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoGuestDockerExecutor.swift @@ -0,0 +1,266 @@ +import Foundation +import Plugin +import Structure + +/// Offline Go guest executor for `script_exec` and `plugin.invoke`. +/// +/// Compiles `package main` source and runs the Linux binary inside the pinned +/// worker image. Untrusted source never touches the host toolchain. +public struct GoGuestDockerExecutor: Sendable { + public static let containerPrefix = "derrick-guest-runtime" + private static let compileTimeoutSeconds = 120 + private static let writeTimeoutSeconds = 60 + + public let image: String + private let executor: DockerCLIExecutor + private let queue: DerrickDockerRunQueue + + public init( + image: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor, + queue: DerrickDockerRunQueue = .guest + ) { + self.image = image.trimmingCharacters(in: .whitespacesAndNewlines) + self.executor = executor + self.queue = queue + } + + /// Compile guest source in a one-shot container and return the Linux binary bytes. + public func compileSource(_ source: String) async throws -> Data { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await prepareCompiledGuest(source: source, in: name) + return try await readBinary(from: name) + } + } + + /// Compile once, run once (factory single-hop path). + public func runSource( + source: String, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + try await withCompiledGuest(source: source) { name in + try await runCompiledGuest( + container: name, + input: input, + timeoutSeconds: timeoutSeconds + ) + } + } + + /// Compile once, then run the body with a live container name (multi-hop loops). + public func withCompiledGuest( + source: String, + _ body: @escaping @Sendable (String) async throws -> T + ) async throws -> T { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await prepareCompiledGuest(source: source, in: name) + return try await body(name) + } + } + + /// Run a previously compiled guest binary in a one-shot container. + public func runArtifact( + artifact: Data, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + try await WorkerImageGate.shared.ensureReady(executor: executor) + return try await withGuestContainer { name in + try await write(binary: artifact, to: name) + return try await runCompiledGuest( + container: name, + input: input, + timeoutSeconds: timeoutSeconds + ) + } + } + + /// Execute `/tmp/guest` in an existing guest container. + public func runCompiledGuest( + container name: String, + input: Data, + timeoutSeconds: Int = 300 + ) async throws -> PluginFactoryExecutionResult { + result( + from: try await executor( + [ + "exec", "-i", name, + DockerWorkerRuntime.guestBinaryPath, + ], + input, + min(max(timeoutSeconds, 1), GuestRuntimeLimits.maxTimeoutSeconds) + ) + ) + } + + private func prepareCompiledGuest(source: String, in container: String) async throws { + try await writeSource(source, to: container) + try await compileGuest(in: container) + } + + private func withGuestContainer( + _ body: @escaping @Sendable (String) async throws -> T + ) async throws -> T { + try await DockerImageInspector.verifyPinned( + tag: image, + expected: DockerWorkerRuntime.pinnedDigest, + executor: executor + ) + let image = self.image + let executor = self.executor + do { + return try await queue.withPermit { + try await OneshotDockerContainer.run( + executor: executor, + prefix: Self.containerPrefix, + createArguments: { name in + [ + "create", + ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ + "--network", "none", + "--name", name, + "--env", "HOME=/tmp", + "--env", "GOCACHE=/tmp/gocache", + "--env", "GOTMPDIR=/tmp", + "--read-only", + "--tmpfs", "/tmp:rw,exec,nosuid,size=256m", + "--pids-limit", "128", + "--cpus", "2.0", + "--memory", "1g", + "--security-opt", "no-new-privileges", + "--cap-drop", "ALL", + image, + "/bin/sleep", + "infinity", + ] + }, + createStep: "create go guest runtime container", + startStep: "start go guest runtime container", + body: body + ) + } + } catch let error as OneshotDockerContainerError { + throw mappedGuestError(error) + } + } + + private func writeSource(_ source: String, to container: String) async throws { + try check( + try await executor( + ["exec", "-i", container, "sh", "-c", DockerWorkerRuntime.guestWriteSourceShell], + Data(source.utf8), + Self.writeTimeoutSeconds + ), + step: "write Go guest source" + ) + } + + private func write(binary: Data, to container: String) async throws { + try check( + try await executor( + [ + "exec", "-i", container, "sh", "-c", + "cat > \(DockerWorkerRuntime.guestBinaryPath) && chmod +x \(DockerWorkerRuntime.guestBinaryPath)", + ], + binary, + Self.writeTimeoutSeconds + ), + step: "write Go guest binary" + ) + } + + private func compileGuest(in container: String) async throws { + try check( + try await executor( + ["exec", container, "sh", "-c", DockerWorkerRuntime.guestCompileShell], + Data(), + Self.compileTimeoutSeconds + ), + step: "compile Go guest" + ) + } + + private func readBinary(from container: String) async throws -> Data { + let response = try await executor( + ["exec", container, "sh", "-c", DockerWorkerRuntime.guestReadBinaryShell], + Data(), + Self.writeTimeoutSeconds + ) + try check(response, step: "read compiled Go guest") + guard !response.stdout.isEmpty else { + throw GoGuestDockerExecutorError.commandFailed( + "read compiled Go guest", + "compiled binary was empty" + ) + } + return response.stdout + } + + private func result(from response: DockerCLIResult) -> PluginFactoryExecutionResult { + PluginFactoryExecutionResult( + exitCode: response.exitCode, + stdout: response.stdout, + stderr: response.stderr + ) + } + + private func check(_ response: DockerCLIResult, step: String) throws { + guard response.exitCode == 0 else { + throw GoGuestDockerExecutorError.commandFailed(step, detail(from: response)) + } + } + + private func mappedGuestError(_ error: OneshotDockerContainerError) -> Error { + switch error { + case .commandFailed(let step, let detail): + return GoGuestDockerExecutorError.commandFailed(step, detail) + case .imageUnavailable: + return error + } + } + + private func detail(from result: DockerCLIResult) -> String { + let stderr = String(decoding: result.stderr, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + return stderr.isEmpty ? "exit \(result.exitCode)" : stderr + } +} + +public enum GoGuestDockerExecutorError: Error, LocalizedError, Equatable, Sendable { + case commandFailed(String, String) + + public var errorDescription: String? { + switch self { + case .commandFailed(let step, let detail): + return "\(step) failed: \(detail)" + } + } +} + +/// Shared gate for worker image readiness (crawl, extract, plugin guest). +public actor WorkerImageGate { + public static let shared = WorkerImageGate() + + private var inFlight: Task? + + public func ensureReady(executor: @escaping DockerCLIExecutor) async throws { + if let inFlight { + try await inFlight.value + return + } + let task = Task { + try await DockerProductImagePrewarmer.ensureWorkerImage(executor: executor) + } + inFlight = task + do { + try await task.value + inFlight = nil + } catch { + inFlight = nil + throw error + } + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift new file mode 100644 index 0000000..cc8c9ab --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoPluginFactoryDockerExecutor.swift @@ -0,0 +1,61 @@ +import Foundation +import Plugin +import Structure + +/// Production adapter for the Go plugin factory. +public struct GoPluginFactoryDockerExecutor: PluginFactoryExecutor, PluginFactoryCompiledGuestExecutor, Sendable { + private let runtime: GoGuestDockerExecutor + + public var image: String { runtime.image } + + public init( + image: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor + ) { + runtime = GoGuestDockerExecutor(image: image, executor: executor) + } + + public func runGuestSource( + source: String, + input: Data + ) async throws -> PluginFactoryExecutionResult { + try await runtime.runSource(source: source, input: input) + } + + public func runGuestSourceHops( + source: String, + testInput: Data + ) async throws -> PluginFactoryHopTestRun { + let script = try PluginFactoryTestScript.parse(testInput) + return try await runtime.withCompiledGuest(source: source) { container in + var hopResults: [PluginFactoryExecutionResult] = [] + var lastResult = PluginFactoryExecutionResult(exitCode: 1) + + for hop in script.hops { + let input = try hop.encodeValidated() + let result = try await runtime.runCompiledGuest( + container: container, + input: input + ) + hopResults.append(result) + lastResult = result + guard result.exitCode == 0 else { + return PluginFactoryHopTestRun(final: result, hopResults: hopResults) + } + } + + return PluginFactoryHopTestRun(final: lastResult, hopResults: hopResults) + } + } + + public func packageGuestSource(source: String) async throws -> Data { + try await runtime.compileSource(source) + } + + public func runPackagedArtifact( + _ artifact: Data, + input: Data + ) async throws -> PluginFactoryExecutionResult { + try await runtime.runArtifact(artifact: artifact, input: input) + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift b/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift new file mode 100644 index 0000000..7a3acb1 --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/Script/GoScriptVerifier.swift @@ -0,0 +1,17 @@ +import Foundation +import Plugin +import Structure + +/// Conservative source checks for standalone Go guest scripts. +public enum GoScriptVerifier: Sendable { + public static func validate( + source: String, + dependencies: [String: String] = [:] + ) -> [String] { + var findings = GuestGoSourceValidator.validate(source: source) + if !dependencies.isEmpty { + findings.append("Guest script dependencies are not supported.") + } + return findings + } +} diff --git a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift index 070b662..943248a 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift @@ -2,7 +2,7 @@ import Foundation import Plugin import Structure -/// Shared host hop loop for offline Python guest programs. +/// Shared host hop loop for offline Go guest programs. public enum GuestHopLoop: Sendable { public typealias HTTPResultEventBuilder = @Sendable ( [PluginEnvelope], diff --git a/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift b/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift index 54dd21d..207bc7f 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GuestPluginRunner.swift @@ -2,7 +2,7 @@ import Foundation import Plugin import Structure -/// Runs an approved factory release through the offline Python guest runtime. +/// Runs an approved factory release through the offline Go guest runtime. public enum GuestPluginRunner: Sendable { public static func run( release: PluginFactoryRelease, @@ -15,14 +15,20 @@ public enum GuestPluginRunner: Sendable { let invokeID = UUID().uuidString let initialEvent = (try? PluginHopEvent.decodeValidated(input)) ?? PluginHopEvent(kind: .manual) - let executor = PythonGuestDockerExecutor(executor: dockerExecutor) + let executor = GoGuestDockerExecutor(executor: dockerExecutor) + guard !release.compiledArtifact.isEmpty else { + throw GoGuestDockerExecutorError.commandFailed( + "load plugin artifact", + "compiled artifact is empty" + ) + } return try await GuestHopLoop.runForPluginInvoke( initialEvent: initialEvent, invokeID: invokeID, timeoutSeconds: timeoutSeconds, execute: { hopInput in - try await executor.runSource( - source: release.guestSource, + try await executor.runArtifact( + artifact: release.compiledArtifact, input: hopInput, timeoutSeconds: timeoutSeconds ) diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift deleted file mode 100644 index ad46487..0000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonGuestDockerExecutor.swift +++ /dev/null @@ -1,133 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Offline Python guest executor for script_exec and plugin.invoke. -/// -/// Recreate-on-handoff: one fresh `--network none` container per run, deleted -/// when the hop loop finishes (host done). The image is reused if already pulled. -public struct PythonGuestDockerExecutor: Sendable { - public static let containerPrefix = "derrick-guest-runtime" - - public let image: String - private let executor: DockerCLIExecutor - private let queue: DerrickDockerRunQueue - - public init( - image: String = DerrickGuestRuntime.pythonGuestDockerImage, - executor: @escaping DockerCLIExecutor, - queue: DerrickDockerRunQueue = .guest - ) { - self.image = image.trimmingCharacters(in: .whitespacesAndNewlines) - self.executor = executor - self.queue = queue - } - - public func runSource( - source: String, - input: Data, - timeoutSeconds: Int = 300 - ) async throws -> PluginFactoryExecutionResult { - try await withGuestContainer { name in - try await write(source: Data(source.utf8), to: name) - return result( - from: try await executor( - ["exec", "-i", name, "python3", "/tmp/guest.py"], - input, - min(max(timeoutSeconds, 1), GuestRuntimeLimits.maxTimeoutSeconds) - ) - ) - } - } - - private func withGuestContainer( - _ body: @escaping @Sendable (String) async throws -> T - ) async throws -> T { - try await OneshotDockerContainer.ensurePulledImage(image, executor: executor) - let image = self.image - let executor = self.executor - do { - return try await queue.withPermit { - try await OneshotDockerContainer.run( - executor: executor, - prefix: Self.containerPrefix, - createArguments: { name in - [ - "create", - ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ - "--network", "none", - "--name", name, - "--env", "HOME=/tmp", - "--read-only", - "--tmpfs", "/tmp:rw,exec,nosuid,size=128m", - "--pids-limit", "128", - "--cpus", "2.0", - "--memory", "1g", - "--security-opt", "no-new-privileges", - "--cap-drop", "ALL", - image, - "/bin/sleep", - "infinity", - ] - }, - createStep: "create guest runtime container", - startStep: "start guest runtime container", - body: body - ) - } - } catch let error as OneshotDockerContainerError { - throw mappedGuestError(error) - } - } - - private func write(source: Data, to container: String) async throws { - try check( - try await executor( - ["exec", "-i", container, "sh", "-c", "cat > /tmp/guest.py"], - source, - 60 - ), - step: "write Python source" - ) - } - - private func result(from response: DockerCLIResult) -> PluginFactoryExecutionResult { - PluginFactoryExecutionResult( - exitCode: response.exitCode, - stdout: response.stdout, - stderr: response.stderr - ) - } - - private func check(_ response: DockerCLIResult, step: String) throws { - guard response.exitCode == 0 else { - throw PythonGuestDockerExecutorError.commandFailed(step, detail(from: response)) - } - } - - private func mappedGuestError(_ error: OneshotDockerContainerError) -> Error { - switch error { - case .commandFailed(let step, let detail): - return PythonGuestDockerExecutorError.commandFailed(step, detail) - case .imageUnavailable: - return error - } - } - - private func detail(from result: DockerCLIResult) -> String { - let stderr = String(decoding: result.stderr, as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) - return stderr.isEmpty ? "exit \(result.exitCode)" : stderr - } -} - -public enum PythonGuestDockerExecutorError: Error, LocalizedError, Equatable, Sendable { - case commandFailed(String, String) - - public var errorDescription: String? { - switch self { - case .commandFailed(let step, let detail): - return "\(step) failed: \(detail)" - } - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift deleted file mode 100644 index d52be34..0000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonPluginFactoryDockerExecutor.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Production adapter for the Python factory. -public struct PythonPluginFactoryDockerExecutor: PluginFactoryExecutor, Sendable { - private let runtime: PythonGuestDockerExecutor - - public var image: String { runtime.image } - - public init( - image: String = DerrickGuestRuntime.pythonGuestDockerImage, - executor: @escaping DockerCLIExecutor - ) { - runtime = PythonGuestDockerExecutor(image: image, executor: executor) - } - - public func runGuestSource( - source: String, - input: Data - ) async throws -> PluginFactoryExecutionResult { - try await runtime.runSource(source: source, input: input) - } - - public func packageGuestSource(source: String) async throws -> Data { - Data(source.utf8) - } - - public func runPackagedArtifact( - _ artifact: Data, - input: Data - ) async throws -> PluginFactoryExecutionResult { - guard let source = String(data: artifact, encoding: .utf8) else { - throw PythonGuestDockerExecutorError.commandFailed( - "decode Python artifact", - "artifact is not valid UTF-8" - ) - } - return try await runtime.runSource(source: source, input: input) - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift b/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift deleted file mode 100644 index 8825d2b..0000000 --- a/packages/MCPServer/Sources/MCPServer/Script/PythonScriptVerifier.swift +++ /dev/null @@ -1,13 +0,0 @@ -import Foundation -import Plugin -import Structure - -/// Conservative source checks for standalone Python guest scripts. -public enum PythonScriptVerifier: Sendable { - public static func validate( - source: String, - dependencies: [String: String] = [:] - ) -> [String] { - GuestPythonSourceValidator.validate(source: source, dependencies: dependencies) - } -} diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift index 5145a42..c1a380d 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionRuntime.swift @@ -3,7 +3,7 @@ import MCP import Plugin import Structure -/// Runs standalone Python guest source and dispatches host-owned capability hops. +/// Runs standalone Go guest source and dispatches host-owned capability hops. public enum ScriptExecutionRuntime { public static func run( arguments: [String: Value], @@ -16,12 +16,12 @@ public enum ScriptExecutionRuntime { ) async throws -> String { let started = Date() let parsed = try parse(arguments) - let language = GuestScriptLanguage.python + let language = GuestScriptLanguage.go logger("[script_exec] \(language.rawValue) source chars=\(parsed.script.count)") if GuestScriptLanguage.requestedLanguageIsUnsupported(arguments) { return finish(blocked( - findings: ["script_exec only runs Python. Swift guest scripts are not supported."], + findings: ["script_exec only runs Go. Python and other guest languages are not supported."], stage: .staticValidation, started: started, parsed: parsed, @@ -30,7 +30,7 @@ public enum ScriptExecutionRuntime { } let staticStarted = Date() - let staticFindings = PythonScriptVerifier.validate( + let staticFindings = GoScriptVerifier.validate( source: parsed.script, dependencies: parsed.dependencies ) @@ -113,30 +113,34 @@ public enum ScriptExecutionRuntime { logger("[script_exec] skipping LLM reviewer") } + let compileStarted = Date() let timeout = GuestRuntimeLimits.effectiveScriptTimeoutSeconds( requested: parsed.timeoutSeconds ) let invokeID = UUID().uuidString do { - let executor = PythonGuestDockerExecutor(executor: stdinExecutor) - let result = try await GuestHopLoop.run( - initialEvent: initialEvent, - invokeID: invokeID, - timeoutSeconds: timeout, - verifier: language.verifierID, - execute: { input in - try await executor.runSource( - source: parsed.script, - input: input, - timeoutSeconds: timeout - ) - }, - logger: logger, - hopHandler: hopHandler - ) + let executor = GoGuestDockerExecutor(executor: stdinExecutor) + let result = try await executor.withCompiledGuest(source: parsed.script) { container in + try await GuestHopLoop.run( + initialEvent: initialEvent, + invokeID: invokeID, + timeoutSeconds: timeout, + verifier: language.verifierID, + execute: { input in + try await executor.runCompiledGuest( + container: container, + input: input, + timeoutSeconds: timeout + ) + }, + logger: logger, + hopHandler: hopHandler + ) + } + let compileMS = ScriptPhaseTiming.elapsedMS(from: compileStarted) let metrics = ScriptPhaseTiming.scriptMetrics(parsed.script) var phaseTiming = result.phaseTiming ?? ScriptPhaseTiming() - phaseTiming.staticValidateMS = staticValidateMS + phaseTiming.staticValidateMS = staticValidateMS + compileMS phaseTiming.totalMS = ScriptPhaseTiming.elapsedMS(from: started) phaseTiming.scriptCharCount = metrics.chars phaseTiming.scriptLineCount = metrics.lines @@ -156,11 +160,17 @@ public enum ScriptExecutionRuntime { phaseTiming: phaseTiming ) return finish(decorated, logger: logger) - } catch let error as PythonGuestDockerExecutorError { + } catch let error as GoGuestDockerExecutorError { logger("[script_exec] guest runtime failed: \(error.localizedDescription)") + let stage: ScriptFailureStage + if case .commandFailed(let step, _) = error, step.contains("compile") { + stage = .typecheck + } else { + stage = .execution + } return finish(runtimeFailure( findings: [error.localizedDescription], - stage: .execution, + stage: stage, started: started, parsed: parsed, assessment: reviewerAssessment, diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift index 87272b1..322efb9 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionSupport.swift @@ -144,8 +144,8 @@ public enum ScriptExecutionVerifier { private static func readonlyViolations(in script: String) -> [String] { let patterns: [(String, String)] = [ - (#"(?m)\bopen\s*\("#, "Readonly mode cannot mutate filesystem."), - (#"(?m)\b(subprocess|os\.system|os\.popen|socket|urllib|requests|httpx)\b"#, "Readonly mode cannot execute nested commands or access the network.") + (#"os\.(Open|ReadFile|WriteFile|Remove|Create|Mkdir)"#, "Readonly mode cannot mutate filesystem."), + (#"\"net/http\"|\"net\"|exec\.Command|crypto/tls"#, "Readonly mode cannot execute nested commands or access the network.") ] return patterns.compactMap { pattern, message in script.range(of: pattern, options: .regularExpression) != nil ? message : nil @@ -159,7 +159,7 @@ extension ScriptExecutionResult { durationMS: Int, maxSeconds: Int = GuestRuntimeLimits.containerRunMaxTTLSeconds, phaseTiming: ScriptPhaseTiming? = nil, - verifier: String = "python-check-v1" + verifier: String = "go-check-v1" ) -> ScriptExecutionResult { let explanation = GuestRuntimeLimits.containerLeaseExceededExplanation(maxSeconds: maxSeconds) return ScriptExecutionResult( diff --git a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift index 12e2e3d..dd3c474 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/ScriptExecutionToolModule.swift @@ -20,11 +20,11 @@ public enum ScriptExecutionToolModule: MCPToolModule { ]), "script": .object([ "type": .string("string"), - "description": .string("Standalone Python source. Reads one JSON event from standard input and writes a JSON array of Derrick envelopes to standard output. Use http.request envelopes for host HTTP and result.emit/message.post for terminal output.") + "description": .string("Standalone Go `package main` source. Reads one hop-event JSON object from standard input and writes an envelope-list JSON array to standard output. Use http.request envelopes for host HTTP and result.emit/message.post for terminal output.") ]), "language": .object([ "type": .string("string"), - "description": .string("Must be python when set. Swift guest scripts are not supported.") + "description": .string("Must be go when set. Python and Swift guest scripts are not supported.") ]), "user_prompt": .object([ "type": .string("string"), diff --git a/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift index c53322b..1b8ca9e 100644 --- a/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift +++ b/packages/MCPServer/Sources/MCPServer/WebCrawlerDockerExecutor.swift @@ -7,23 +7,20 @@ import Structure /// The image is trusted product code. User input is passed only as JSON on /// stdin; it is never interpolated into a shell command. public struct WebCrawlerDockerExecutor: Sendable { - public static let image = "derrick-web-crawler:swift-6.4-v1" + public static let image = DockerWorkerRuntime.image public static let containerPrefix = "derrick-web-crawler" + public static let binaryPath = DockerWorkerRuntime.crawlerBinary public static let maximumTimeoutSeconds = 900 public static let dockerNetwork = "bridge" private let executor: DockerCLIExecutor private let queue: DerrickDockerRunQueue - private let imageGate: WebCrawlerImageGate - public init( executor: @escaping DockerCLIExecutor, - queue: DerrickDockerRunQueue = .crawler, - imageGate: WebCrawlerImageGate = .shared + queue: DerrickDockerRunQueue = .crawler ) { self.executor = executor self.queue = queue - self.imageGate = imageGate } public func run( @@ -31,7 +28,7 @@ public struct WebCrawlerDockerExecutor: Sendable { timeoutSeconds: Int ) async throws -> DockerCLIResult { let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) - try await imageGate.ensureReady(executor: executor) + try await WorkerImageGate.shared.ensureReady(executor: executor) let prepared = try await WebCrawlerDockerInputPreparer.enrich(input) let executor = self.executor do { @@ -53,7 +50,7 @@ public struct WebCrawlerDockerExecutor: Sendable { startStep: "start crawler container", body: { name in try await executor( - ["exec", "-i", name, "/usr/local/bin/derrick-web-crawler"], + ["exec", "-i", name, Self.binaryPath], prepared.data, timeout ) diff --git a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift index 2c6bc08..29d8d17 100644 --- a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift @@ -63,6 +63,17 @@ public enum WebCrawlerToolModule: MCPToolModule { ).encodedJSON() } + do { + try GuestContractValidation.validateWebCrawlerResultJSON(dockerResult.stdout) + } catch { + return try failure( + status: .failed, + stage: .execution, + code: "web_crawl_invalid_output", + message: "Crawler returned invalid JSON output." + ).encodedJSON() + } + guard let result = try? JSONDecoder().decode( WebCrawlerWireResult.self, from: dockerResult.stdout @@ -274,6 +285,13 @@ private struct WebCrawlerWireResult: Decodable, Sendable { case stopReason = "stop_reason" case diagnostics } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + ok = try container.decode(Bool.self, forKey: .ok) + stopReason = try container.decode(String.self, forKey: .stopReason) + diagnostics = try container.decodeIfPresent([String].self, forKey: .diagnostics) ?? [] + } } private enum WebCrawlerToolError: Error, LocalizedError, Sendable { diff --git a/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift b/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift index f26bde7..dac39f4 100644 --- a/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift +++ b/packages/MCPServer/Sources/SlackConnectorE2EHarness/E2EEnvironment.swift @@ -271,7 +271,7 @@ struct E2EEnvironment { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("[E2E] factory build scope=\(scope.rawValue)…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: dockerExecutor) + let executor = GoPluginFactoryDockerExecutor(executor: dockerExecutor) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 5) ).build( diff --git a/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift b/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift index 7c72d07..9fd283b 100644 --- a/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift +++ b/packages/MCPServer/Sources/SlackConnectorInstallReference/SlackConnectorInstallReferenceMain.swift @@ -83,7 +83,7 @@ enum SlackConnectorInstallReference { userGoal: goal, hostManifest: input.hostManifest, builder: E2EFactoryBuilder(scope: .fullSync), - executor: PythonPluginFactoryDockerExecutor(executor: dockerExecutor), + executor: GoPluginFactoryDockerExecutor(executor: dockerExecutor), reviewer: E2EHarnessReviewer(), logger: { fputs("\($0)\n", stderr) } ) diff --git a/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift b/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift index 06aa406..202ed35 100644 --- a/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift +++ b/packages/MCPServer/Sources/SlackConnectorLiveHarness/LiveHarnessEnvironment.swift @@ -79,7 +79,7 @@ struct LiveHarnessEnvironment { let goal = input.connectorBuildGoal(crawlSummary: SlackConnectorFactoryInput.defaultCrawlSummary) fputs("[live] building full-sync connector via LLM factory…\n", stderr) - let executor = PythonPluginFactoryDockerExecutor(executor: dockerExecutor) + let executor = GoPluginFactoryDockerExecutor(executor: dockerExecutor) let release = try await PluginFactorySession( configuration: PluginFactoryConfiguration(maxBuilderAttempts: 3) ).build( diff --git a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift index 1896241..1ce0f0a 100644 --- a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift +++ b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift @@ -8,20 +8,68 @@ import WebCrawler @testable import MCPServer @Suite struct MCPServerTests { - private static let dummyPythonScript = """ - import json, sys - _ = sys.stdin.read() - json.dump([{"verb":"result.emit","summary":"ok"}], sys.stdout) + private static let dummyGoScript = """ + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{"verb": "result.emit", "summary": "ok"}}) + } """ - private static let dummyStdin: @Sendable ([String], Data, Int) async throws -> DockerCLIResult = { arguments, _, _ in - if arguments.contains("python3"), arguments.contains("/tmp/guest.py") { + private static func isGuestBinaryExec(_ arguments: [String]) -> Bool { + arguments.contains(DockerWorkerRuntime.guestBinaryPath) + && !arguments.contains("cat >") + } + + private static func mockWorkerImageInspect(_ arguments: [String]) -> DockerCLIResult? { + guard arguments.first == "image", arguments.contains("inspect") else { + return nil + } + if arguments.contains("{{.Id}}") { + let digest = DockerWorkerRuntime.pinnedDigest.rawValue + "\n" + return DockerCLIResult(exitCode: 0, stdout: Data(digest.utf8), stderr: Data()) + } + return DockerCLIResult(exitCode: 0, stdout: Data("[]".utf8), stderr: Data()) + } + + private static let dummyCompiledGuest = Data([0x7f, 0x45, 0x4c, 0x46, 0x02]) + + private static func mockGuestDocker(_ arguments: [String]) -> DockerCLIResult? { + if arguments.contains("cat > /tmp/guest") { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestWriteSourceShell) { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestCompileShell) { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if arguments.contains(DockerWorkerRuntime.guestReadBinaryShell) { + return DockerCLIResult(exitCode: 0, stdout: dummyCompiledGuest, stderr: Data()) + } + if isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), stderr: Data() ) } + return mockWorkerImageInspect(arguments) + } + + private static let dummyStdin: @Sendable ([String], Data, Int) async throws -> DockerCLIResult = { arguments, _, _ in + if let mocked = mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } @@ -825,9 +873,9 @@ import WebCrawler #expect(results?.first?["json"] == nil) } - @Test func pythonGuestRuntimeUsesPinnedImage() { - #expect(DerrickGuestRuntime.pythonGuestDockerImage == "python:3.14.7") - #expect(PythonGuestDockerExecutor.containerPrefix == "derrick-guest-runtime") + @Test func goGuestRuntimeUsesWorkerImage() { + #expect(DerrickGuestRuntime.guestDockerImage == DockerWorkerRuntime.image) + #expect(GoGuestDockerExecutor.containerPrefix == "derrick-guest-runtime") #expect(DerrickDockerRunQueue.guest.maxConcurrentContainers == 1) #expect(DerrickDockerRunQueue.crawler.maxConcurrentContainers == 2) #expect(DerrickDockerRunQueue.extractor.maxConcurrentContainers == 1) @@ -931,40 +979,35 @@ import WebCrawler } } - @Test func pythonSourceVerifierRejectsNetworkAndDependencies() { - let findings = PythonScriptVerifier.validate( - source: "import sys\nimport requests", + @Test func goSourceVerifierRejectsNetworkAndDependencies() { + let findings = GoScriptVerifier.validate( + source: "package main\nimport \"net/http\"", dependencies: ["example": "1.0.0"] ) - #expect(findings.contains("Direct network access is not allowed; emit http.request envelopes.")) - #expect(findings.contains("Guest plugin dependencies are not supported; use the standard library.")) + #expect(findings.contains("Go guest must not import \"net/http\".")) + #expect(findings.contains("Guest script dependencies are not supported.")) } - @Test func pythonSourceVerifierRequiresStdin() { - let findings = PythonScriptVerifier.validate(source: "print('[]')") - #expect(findings.contains("Python source must read its JSON event from standard input.")) + @Test func goSourceVerifierRequiresPackageMain() { + let findings = GoScriptVerifier.validate(source: "package plugin") + #expect(findings.contains("Go guest source must declare package main.")) } - @Test func pythonExecutorUsesReadOnlyOfflineContainer() async throws { + @Test func goExecutorUsesReadOnlyOfflineContainer() async throws { let recorder = DockerCallRecorder() - let runner = PythonGuestDockerExecutor( - image: "python:3.14.7", + let runner = GoGuestDockerExecutor( + image: DockerWorkerRuntime.image, executor: { arguments, _, _ in await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) + if let mocked = Self.mockGuestDocker(arguments) { + return mocked } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) ) _ = try await runner.runSource( - source: "import json, sys\njson.dump([], sys.stdout)", + source: Self.dummyGoScript, input: Data(#"{"kind":"script"}"#.utf8) ) @@ -975,15 +1018,17 @@ import WebCrawler #expect(create.contains("--read-only")) #expect(create.contains("--label")) #expect(create.contains(DerrickDockerRuntimeIdentity.labelAssignment)) - let exec = calls.first(where: { $0.contains("python3") }) ?? [] - #expect(exec.contains("/tmp/guest.py")) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestWriteSourceShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestCompileShell) }) + let exec = calls.first(where: { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) ?? [] + #expect(exec.contains(DockerWorkerRuntime.guestBinaryPath)) #expect(calls.contains { $0.first == "rm" && $0.contains("-f") }) } - @Test func pythonGuestDockerCommandsPassXPCValidation() async throws { + @Test func goGuestDockerCommandsPassXPCValidation() async throws { let recorder = DockerCallRecorder() - let runner = PythonGuestDockerExecutor( - image: "python:3.14.7", + let runner = GoGuestDockerExecutor( + image: DockerWorkerRuntime.image, executor: { arguments, _, _ in await recorder.append(arguments) if let error = DockerRunRequestValidator.validate( @@ -995,26 +1040,22 @@ import WebCrawler stderr: Data(error.launchErrorMessage.utf8) ) } - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) + if let mocked = Self.mockGuestDocker(arguments) { + return mocked } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) ) let result = try await runner.runSource( - source: "import json, sys\njson.dump([], sys.stdout)", + source: Self.dummyGoScript, input: Data(#"{"kind":"script"}"#.utf8) ) #expect(result.exitCode == 0) let calls = await recorder.calls - #expect(calls.contains { $0.contains("sh") && $0.contains("cat > /tmp/guest.py") }) - #expect(calls.contains { $0.contains("python3") && $0.contains("/tmp/guest.py") }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestWriteSourceShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestCompileShell) }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) } @Test func leftoverSwiftRuntimePrefixIsStillSwept() { @@ -1022,15 +1063,15 @@ import WebCrawler #expect(GuestRuntimeLimits.maxTimeoutSeconds == 300) } - @Test func pythonScriptCanReturnHTMLResult() async throws { + @Test func goScriptCanReturnHTMLResult() async throws { let resultText = try await ScriptExecutionRuntime.run( arguments: [ "description": .string("render a safe card"), "reason": .string("manual HTML output check"), - "script": .string(Self.dummyPythonScript) + "script": .string(Self.dummyGoScript) ], stdinExecutor: { arguments, _, _ in - if arguments.contains("python3"), arguments.contains("/tmp/guest.py") { + if Self.isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data( @@ -1039,6 +1080,9 @@ import WebCrawler stderr: Data() ) } + if let mocked = Self.mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, reviewer: StubReviewer( @@ -1058,13 +1102,13 @@ import WebCrawler #expect(result.output?.value == "

Safe

") } - @Test func scriptExecRejectsSwiftLanguage() async throws { + @Test func scriptExecRejectsUnsupportedLanguage() async throws { let resultText = try await ScriptExecutionRuntime.run( arguments: [ - "description": .string("legacy swift"), + "description": .string("legacy python"), "reason": .string("should be blocked"), - "script": .string(Self.dummyPythonScript), - "language": .string("swift") + "script": .string(Self.dummyGoScript), + "language": .string("python") ], stdinExecutor: Self.dummyStdin, reviewer: nil, @@ -1074,10 +1118,10 @@ import WebCrawler let result = try #require(ToolExecutionOutcome.decode(from: resultText)) #expect(result.status == .blocked) #expect(result.stage == .validation) - #expect(resultText.contains("only runs Python")) + #expect(resultText.contains("only runs Go")) } - @Test func pythonScriptToolBlocksFilesystemAccess() async throws { + @Test func goScriptToolBlocksFilesystemAccess() async throws { let bridge = try await MCPLocalBridge.make { server in await server.registerScriptExecutionTool( stdinExecutor: Self.dummyStdin, @@ -1098,7 +1142,7 @@ import WebCrawler arguments: [ "description": .string("attempt write"), "reason": .string("test"), - "script": .string("import sys\n_ = sys.stdin.read()\nopen('/tmp/a','w')") + "script": .string("package main\nimport \"os\"\nfunc main() { _, _ = os.Open(\"/tmp/a\") }") ] ) @@ -1108,18 +1152,18 @@ import WebCrawler @Test func leftoverSwiftGuestImageIsTreatedAsStaleHygieneTag() { #expect(DerrickGuestRuntime.swiftPluginDockerImage.contains("swift")) - #expect(DerrickGuestRuntime.pythonGuestDockerImage == "python:3.14.7") + #expect(DerrickGuestRuntime.guestDockerImage == DockerWorkerRuntime.image) } - @Test func guestPluginRunnerRunsPythonRelease() async throws { + @Test func guestPluginRunnerRunsGoRelease() async throws { let recorder = DockerCallRecorder() let release = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", manifestJSON: "{}", - runtimeJSON: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"#, - guestSource: "import json, sys\njson.dump([{\"verb\":\"result.emit\",\"summary\":\"ok\"}], sys.stdout)", - compiledArtifact: Data(), + runtimeJSON: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"#, + guestSource: Self.dummyGoScript, + compiledArtifact: Self.dummyCompiledGuest, skillFiles: [:], contentHash: try PluginContentHash(hex: String(repeating: "c", count: 64)), reviewSummary: "ok" @@ -1129,21 +1173,23 @@ import WebCrawler input: Data(#"{"kind":"manual"}"#.utf8), dockerExecutor: { arguments, _, _ in await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { + if Self.isGuestBinaryExec(arguments) { return DockerCLIResult( exitCode: 0, stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), stderr: Data() ) } + if let mocked = Self.mockGuestDocker(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } ) #expect(result.exitCode == 0) #expect(String(decoding: result.stdout, as: UTF8.self).contains("result.emit")) let calls = await recorder.calls - #expect(calls.contains { $0.contains("python3") }) + #expect(calls.contains { $0.contains(DockerWorkerRuntime.guestBinaryPath) }) #expect(!calls.contains { $0.contains("swift") }) } @@ -1274,12 +1320,12 @@ import WebCrawler #expect(hops == PluginContract.maxPluginInvokeHops) } - @Test func pythonGuestContainerArgumentsStayNetworkIsolated() { + @Test func goGuestContainerArgumentsStayNetworkIsolated() { let name = "derrick-guest-runtime-test" let args = [ "create", "--network", "none", "--name", name, "--read-only", "--tmpfs", "/tmp:rw,exec,nosuid,size=128m", - DerrickGuestRuntime.pythonGuestDockerImage, "/bin/sleep", "infinity", + DerrickGuestRuntime.guestDockerImage, "/bin/sleep", "infinity", ] #expect(args.contains("--name")) #expect(args.contains(name)) @@ -1301,7 +1347,7 @@ import WebCrawler "mode": .string("write"), "description": .string("create report file"), "reason": .string("user asked for file output"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "expected_effects": .array([.string("write /tmp/report.txt")]), "allow_network": .bool(true) ] @@ -1332,7 +1378,7 @@ import WebCrawler "mode": .string("readonly"), "description": .string("inspect csv"), "reason": .string("analyze user-provided data"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "user_prompt": .string("summarize this csv"), "allow_network": .bool(true) ] @@ -1448,7 +1494,7 @@ import WebCrawler "mode": .string("readonly"), "description": .string("fetch page"), "reason": .string("test"), - "script": .string(Self.dummyPythonScript), + "script": .string(Self.dummyGoScript), "allow_network": .bool(true) ] ) diff --git a/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift b/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift deleted file mode 100644 index b95215d..0000000 --- a/packages/MCPServer/Tests/MCPServerTests/PythonPluginFactoryDockerExecutorTests.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation -import MCPServer -import Plugin -import Testing -import Structure - -@Suite struct PythonPluginFactoryDockerExecutorTests { - @Test func packagesAndRunsPythonSource() async throws { - let recorder = DockerCallRecorder() - let runner = PythonPluginFactoryDockerExecutor( - image: "python:3.14.7", - executor: { arguments, _, _ in - await recorder.append(arguments) - if arguments.contains("/tmp/guest.py"), - !arguments.contains("cat") { - return DockerCLIResult( - exitCode: 0, - stdout: Data(#"[{"verb":"result.emit","summary":"ok"}]"#.utf8), - stderr: Data() - ) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - ) - let draft = try await runner.runGuestSource( - source: "import json, sys\njson.dump([], sys.stdout)", - input: Data(#"{"kind":"manual"}"#.utf8) - ) - #expect(draft.exitCode == 0) - let packaged = try await runner.packageGuestSource( - source: "import json, sys\njson.dump([], sys.stdout)" - ) - let released = try await runner.runPackagedArtifact( - packaged, - input: Data(#"{"kind":"manual"}"#.utf8) - ) - #expect(released.exitCode == 0) - let calls = await recorder.calls - #expect(calls.contains { $0.contains("python3") && $0.contains("/tmp/guest.py") }) - } -} diff --git a/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift b/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift new file mode 100644 index 0000000..e4bd529 --- /dev/null +++ b/packages/Plugin/Sources/Plugin/Factory/GuestGoSourceValidator.swift @@ -0,0 +1,35 @@ +import Foundation +import Structure + +public enum GuestGoSourceValidator: Sendable { + private static let forbiddenImports = [ + "\"net/http\"", + "\"net\"", + "\"os/exec\"", + "\"crypto/tls\"", + ] + + private static let forbiddenCalls = [ + "os.Open", + "os.ReadFile", + "os.WriteFile", + "exec.Command", + "http.Get", + "http.Post", + "http.Client", + ] + + public static func validate(source: String) -> [String] { + var findings: [String] = [] + for token in forbiddenImports where source.contains(token) { + findings.append("Go guest must not import \(token).") + } + for token in forbiddenCalls where source.contains(token) { + findings.append("Go guest must not call \(token).") + } + if !source.contains("package main") { + findings.append("Go guest source must declare package main.") + } + return findings + } +} diff --git a/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift b/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift deleted file mode 100644 index 2bc5673..0000000 --- a/packages/Plugin/Sources/Plugin/Factory/GuestPythonSourceValidator.swift +++ /dev/null @@ -1,44 +0,0 @@ -import Foundation -import Structure - -/// Conservative source checks for Python guest plugins and factory drafts. -public enum GuestPythonSourceValidator: Sendable { - public static func validate( - source: String, - dependencies: [String: String] = [:] - ) -> [String] { - var findings: [String] = [] - let trimmed = source.trimmingCharacters(in: .whitespacesAndNewlines) - - if trimmed.isEmpty { - findings.append("Python source is empty.") - } - - let forbiddenTokens: [(String, String)] = [ - ("import socket", "Direct socket access is not allowed; emit http.request envelopes."), - ("from socket", "Direct socket access is not allowed; emit http.request envelopes."), - ("import urllib", "Direct network access is not allowed; emit http.request envelopes."), - ("from urllib", "Direct network access is not allowed; emit http.request envelopes."), - ("import requests", "Direct network access is not allowed; emit http.request envelopes."), - ("import httpx", "Direct network access is not allowed; emit http.request envelopes."), - ("import subprocess", "Process execution is not allowed."), - ("os.system(", "Process execution is not allowed."), - ("os.popen(", "Process execution is not allowed."), - ("open(", "Filesystem access is not allowed in guest plugins."), - ] - for (token, message) in forbiddenTokens where source.contains(token) { - findings.append(message) - } - - let readsStdin = source.contains("sys.stdin") - || source.contains("input(") - || source.contains("stdin.read") - if !readsStdin { - findings.append("Python source must read its JSON event from standard input.") - } - if !dependencies.isEmpty { - findings.append("Guest plugin dependencies are not supported; use the standard library.") - } - return findings - } -} diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift index b588309..fb7cdfd 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift @@ -105,7 +105,7 @@ public struct PluginFactorySession: Sendable { {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ "params":{...}}]} Include http_results fixtures for every messaging_op you implement. Match request_id values \ - in fixtures to the http.request envelopes your python_source emits. De-duplicate http_results \ + in fixtures to the http.request envelopes your go_source emits. De-duplicate http_results \ by request_id using stable sorting — do not overwrite duplicates by response order. """ ) @@ -262,7 +262,7 @@ public struct PluginFactory: Sendable { var files: [String: Data] = [ "plugin.json": Data(draft.manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.py": Data(draft.guestSource.utf8), + "app.derrick/plugin.go": Data(draft.guestSource.utf8), "app.derrick/plugin": artifact, ] for (path, body) in draft.skillFiles { @@ -293,9 +293,9 @@ public struct PluginFactory: Sendable { do { let manifest = try AgentPluginManifest.decode(data) guard let entrypoint = manifest.derrick?.entrypoint, - entrypoint.hasSuffix(".py") else { + entrypoint.hasSuffix(".go") else { throw PluginFactoryError.invalidManifest( - "extensions.app.derrick.entrypoint must point to a Python file." + "extensions.app.derrick.entrypoint must point to a Go file." ) } guard !["create-plugin", "edit-plugin"].contains(manifest.name.rawValue) else { @@ -323,7 +323,7 @@ public struct PluginFactory: Sendable { } private func validateSource(_ source: String) throws { - let findings = GuestPythonSourceValidator.validate(source: source) + let findings = GuestGoSourceValidator.validate(source: source) if let first = findings.first { throw PluginFactoryError.invalidSource(first) } @@ -331,10 +331,10 @@ public struct PluginFactory: Sendable { private func runtimeJSON(for manifest: AgentPluginManifest) throws -> String { guard let entrypoint = manifest.derrick?.entrypoint else { - throw PluginFactoryError.invalidManifest("A Python entrypoint is required.") + throw PluginFactoryError.invalidManifest("A Go entrypoint is required.") } let object: [String: String] = [ - "language": "python", + "language": "go", "entrypoint": entrypoint, ] let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift index b5e45a7..2583c1e 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryRuntime.swift @@ -2,6 +2,7 @@ import Foundation import Structure public extension PluginFactoryRelease { - /// All approved releases run as Python guests. - var guestLanguage: PluginGuestLanguage { .python } + var guestLanguage: PluginGuestLanguage { + PluginFactoryRuntime.decode(from: runtimeJSON)?.language ?? .go + } } diff --git a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift index e84fea7..a8e3b7e 100644 --- a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift +++ b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift @@ -4,35 +4,46 @@ import Testing @testable import Plugin @Suite struct PluginFactoryTests { - private func guestPythonSource(emit: String = #"[]"#) -> String { + private func guestGoSource(summary: String = "ok") -> String { """ - import json, sys - _ = json.load(sys.stdin) - json.dump(\(emit), sys.stdout) + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{"verb": "result.emit", "summary": "\(summary)"}}) + } """ } - @Test func guestLanguageIsAlwaysPython() { + @Test func guestLanguageIsGoFromRuntimeJSON() { let release = PluginFactoryRelease( pluginID: "slack-connection", version: "1.0.0", manifestJSON: "{}", - runtimeJSON: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"#, - guestSource: guestPythonSource(), + runtimeJSON: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"#, + guestSource: guestGoSource(), compiledArtifact: Data(), skillFiles: [:], contentHash: try! PluginContentHash(hex: String(repeating: "b", count: 64)), reviewSummary: "ok" ) - #expect(release.guestLanguage == .python) + #expect(release.guestLanguage == .go) } - @Test func pluginFactoryRuntimeDecodesPythonEntrypoint() { + @Test func pluginFactoryRuntimeDecodesGoEntrypoint() { let runtime = PluginFactoryRuntime.decode( - from: #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + from: #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# ) - #expect(runtime?.language == .python) - #expect(runtime?.entrypoint.hasSuffix(".py") == true) + #expect(runtime?.language == .go) + #expect(runtime?.entrypoint.hasSuffix(".go") == true) } @Test func envelopeDecoderRejectsNestedResultAliases() { @@ -47,7 +58,7 @@ import Testing let reviewer = RecordingFactoryReviewer(result: PluginFactoryReview(approved: true, summary: "safe")) let draft = PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data(#"{"kind":"manual"}"#.utf8), skillFiles: ["skills/weather/SKILL.md": "# Weather\n\nReturn weather."] ) @@ -60,12 +71,12 @@ import Testing #expect(release.pluginID == "weather-tool") #expect(release.version == "1.2.3") - #expect(release.runtimeJSON.contains("\"language\":\"python\"")) - #expect(release.runtimeJSON.contains("plugin.py")) + #expect(release.runtimeJSON.contains("\"language\":\"go\"")) + #expect(release.runtimeJSON.contains("plugin.go")) #expect(!release.contentHash.rawValue.isEmpty) #expect(release.verifyIntegrity()) var tampered = release.packageFiles() - tampered["app.derrick/plugin.py"] = Data("changed".utf8) + tampered["app.derrick/plugin.go"] = Data("changed".utf8) #expect(!PluginFactoryRelease.verifyIntegrity(files: tampered, expected: release.contentHash)) #expect(await executor.draftRunCount == 1) #expect(await executor.packageCount == 1) @@ -83,7 +94,7 @@ import Testing _ = try await PluginFactory().build( draft: PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource() + guestSource: guestGoSource() ), executor: executor, reviewer: reviewer @@ -222,7 +233,7 @@ import Testing @Test func reservedPluginIDsCannotBeCreated() async { let draft = PluginFactoryDraft( manifestJSON: manifestJSON().replacingOccurrences(of: "weather-tool", with: "create-plugin"), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), ) do { _ = try await PluginFactory().build( @@ -242,13 +253,13 @@ import Testing @Test func missingSchemaIsRejectedAtTheFactoryBoundary() async { let manifest = """ - {"name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ do { _ = try await PluginFactory().build( draft: PluginFactoryDraft( manifestJSON: manifest, - guestSource: guestPythonSource() + guestSource: guestGoSource() ), executor: RecordingFactoryExecutor(), reviewer: RecordingFactoryReviewer( @@ -268,12 +279,12 @@ import Testing pluginID: "weather-tool", version: "1.0.0", description: "Weather summaries.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), ) let draft = try response.draft() let manifest = try AgentPluginManifest.decode(Data(draft.manifestJSON.utf8)) #expect(manifest.schema == PluginContract.agentPluginSchema) - #expect(manifest.derrick?.entrypoint == "./app.derrick/plugin.py") + #expect(manifest.derrick?.entrypoint == "./app.derrick/plugin.go") } @Test func builderNormalizesUnderscorePluginIDAndWritesSecretLabels() throws { @@ -281,7 +292,7 @@ import Testing pluginID: "slack_connection", version: "1.0.0", description: "Slack send and receive.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), secrets: [ try PluginSecretField(id: "username", label: "Slack username", kind: .username), try PluginSecretField(id: "password", label: "Slack password", kind: .password), @@ -300,7 +311,7 @@ import Testing pluginID: "slack-connection", version: "1.0.0", description: "Slack send and receive.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), role: .connector, messagingOps: ["send_message"] ) @@ -319,7 +330,7 @@ import Testing pluginID: "slack-connection", version: "1.0.0", description: "Slack full sync.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), role: .connector ) let draft = try response.draft() @@ -329,7 +340,7 @@ import Testing @Test func connectorTestScriptRequiresHopsAndFixtures() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) #expect(throws: PluginFactoryError.self) { @@ -337,7 +348,7 @@ import Testing } let draft = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data( #"{"kind":"message_in_room","params":{"messaging_op":"send_message"}}"#.utf8 ), @@ -374,7 +385,7 @@ import Testing @Test func fullSyncTestScriptRequiresReplyThreadPollHop() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) let channelOnly = Data( @@ -391,7 +402,7 @@ import Testing ) let missingReplies = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: channelOnly, userGoal: fullSyncGoal() ) @@ -401,7 +412,7 @@ import Testing let withReplies = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data( """ {"hops":[ @@ -424,7 +435,7 @@ import Testing @Test func dualFixtureAllowsUnsortedHttpResultsLoop() throws { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ let manifest = try AgentPluginManifest.decode(Data(manifestJSON.utf8)) let testInput = Data( @@ -519,7 +530,7 @@ import Testing @Test func missingRoleDefaultsToStandard() throws { let json = """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ let manifest = try AgentPluginManifest.decode(Data(json.utf8)) #expect(manifest.derrick?.role == .standard) @@ -528,7 +539,7 @@ import Testing @Test func invalidRoleIsRejected() { let json = """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"slack"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.0.0","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"slack"}}} """ do { _ = try AgentPluginManifest.decode(Data(json.utf8)) @@ -555,7 +566,7 @@ import Testing pluginID: "weather-tool", version: "1.0.0", description: "Weather summaries.", - guestSource: guestPythonSource(), + guestSource: guestGoSource(), skillFiles: [ PluginFactorySkillFile(path: "SKILL.md", body: "Invalid layout.") ] @@ -575,14 +586,14 @@ import Testing private func draft() -> PluginFactoryDraft { PluginFactoryDraft( manifestJSON: manifestJSON(), - guestSource: guestPythonSource(), + guestSource: guestGoSource(), testInput: Data(#"{"kind":"manual"}"#.utf8) ) } private func manifestJSON() -> String { """ - {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.2.3","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py"}}} + {"$schema":"\(PluginContract.agentPluginSchema)","name":"weather-tool","version":"1.2.3","extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go"}}} """ } } @@ -615,7 +626,7 @@ private func validConnectorTestInput() -> Data { private func connectorDraft(testInput: Data) -> PluginFactoryDraft { let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","auth_scheme":"bot_token","secrets":[{"id":"bot_token","label":"Bot Token","kind":"token"}],"messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ return PluginFactoryDraft( manifestJSON: manifestJSON, diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift b/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift index 4ac3d26..9f35e74 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/ContainerLifecyclePolicy.swift @@ -9,7 +9,7 @@ import Foundation public struct ContainerLifecyclePolicy: Sendable, Hashable { /// Maximum crawler containers at once (oneshot; own queue). public let maxNetworkContainers: Int - /// Maximum offline Python guest containers at once (`script_exec` / `plugin.invoke`). + /// Maximum offline Go guest containers at once (`script_exec` / `plugin.invoke`). public let maxOfflineContainers: Int /// Maximum file-extractor containers at once (oneshot; own queue). public let maxFileExtractContainers: Int diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift index 3cd8341..d7296f4 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift @@ -12,12 +12,11 @@ public enum DerrickGuestRuntime: Sendable { /// Leftover Swift guest image tag reported by older daemons. Hygiene retires a mismatch. public static let swiftPluginDockerImage = "swiftlang/swift:nightly-6.4.x-noble" - /// Pullable Python image for offline guests (script_exec primary). - public static let pythonGuestDockerImage = "python:3.14.7" + /// Unified Go worker image for offline guests (`script_exec` / `plugin.invoke`). + public static let guestDockerImage = DockerWorkerRuntime.image - /// Custom image with uv for packaged connector plugins. - /// Build: `docker build -f docker/guest-runtime/Dockerfile -t derrick-guest-runtime:python-v1 .` - public static let pythonGuestDockerImageWithUV = "derrick-guest-runtime:python-v1" + /// Legacy Python image reported by older daemons. Hygiene retires a mismatch. + public static let legacyPythonGuestDockerImage = "python:3.14.7" } public struct ServiceHealthReport: Codable, Sendable, Hashable { diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift b/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift index 0a69752..5604b30 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/EffectorAdmissionPolicy.swift @@ -6,10 +6,14 @@ public enum EffectorAdmissionPolicy: Sendable { context: ExecutionContextWire?, principal: ServicePrincipal ) -> Bool { - if case .job = principal { return true } - guard let context else { return false } - if context.capabilities.contains(.syncWebCrawl) { return true } - if context.workflow?.kind == .pluginFactoryCreate { return true } + switch principal { + case .job, .agent: + return true + default: + break + } + if let context, context.capabilities.contains(.syncWebCrawl) { return true } + if let context, context.workflow?.kind == .pluginFactoryCreate { return true } return false } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift index 1acf695..5d3808f 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift @@ -45,7 +45,7 @@ public struct MCPToolCallRequest: Codable, Sendable, Hashable { /// JSON `HelperModelWire` for script security reviewer model selection. /// When nil, MCPService uses the default helper model. public let helperReviewerModelJSON: String? - /// When true, MCPService allows synchronous `web.crawl` (interactive `/create-plugin` turns). + /// When true, plugin factory creation is active for this call. /// Deprecated: use `executionContextJSON` (ExecutionContextWire). public let pluginFactoryCreationActive: Bool /// JSON `ExecutionContextWire` for cross-boundary policy and effector admission. diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift index 0a72e14..10ef8ff 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateFailureMessage.swift @@ -92,8 +92,8 @@ public enum PluginFactoryCreateFailureMessage: Sendable { if message.count > 160 { return true } let prefixes = [ "Invalid Agent Plugin manifest", - "Invalid Python guest source", - "Python draft test failed", + "Invalid Go guest source", + "Go draft test failed", "Plugin review rejected", "Draft validation failed:", ] diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift index 001ae38..6452b0e 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift @@ -248,7 +248,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { } } extra.append( - "The host writes plugin.json. Return python_source and test_input_json only. Do not invent a plugin_id or secrets list." + "The host writes plugin.json. Return go_source and test_input_json only. Do not invent a plugin_id or secrets list." ) return try ConnectorContractPrompts.factoryGoal( vendorLabel: vendorLabel, diff --git a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift index 4a24d9b..9f83297 100644 --- a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift +++ b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift @@ -43,22 +43,22 @@ public enum PromptResources { try DerrickBundledText.load("user_facing_spawn_overlay.md", from: resourceRoot) } - /// Python guest contract wrapped for a model prompt. + /// Go guest contract wrapped for a model prompt. public static func guestSDKForModel( from resourceRoot: URL? = nil, spec: PluginSpec? = nil ) throws -> String { _ = resourceRoot return DerrickBundledText.formatCodeForModel( - try DerrickGuestPython.source(for: spec), - heading: "standalone Python guest contract", - language: "python" + try DerrickGuestGo.source(for: spec), + heading: "standalone Go guest contract", + language: "go" ) } public static func guestSDKSource(from resourceRoot: URL? = nil) throws -> String { _ = resourceRoot - return try DerrickGuestPython.source() + return try DerrickGuestGo.source() } private static func load(named name: String, from resourceRoot: URL?, prefixTxt: String? = nil) throws -> String { diff --git a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift index f5f57d3..1350aed 100644 --- a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift +++ b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/TurnProcessContextTypes.swift @@ -13,7 +13,7 @@ public struct ExecutionContextSlots: Sendable { public var networkAccessPrompt: TurnProcessContextTypes.NetworkPrompt? public var policyDecisionPrompt: TurnProcessContextTypes.PolicyDecisionPrompt? public var policyNoticePublisher: TurnProcessContextTypes.PolicyNoticePublisher? - /// When true, `web.crawl` may run synchronously (plugin factory turns). + /// When true, plugin factory creation is active for this turn. public var pluginFactoryCreationActive: Bool public init( diff --git a/packages/Structure/Sources/Contract/GuestContract.swift b/packages/Structure/Sources/Contract/GuestContract.swift index 0464da3..25568ca 100644 --- a/packages/Structure/Sources/Contract/GuestContract.swift +++ b/packages/Structure/Sources/Contract/GuestContract.swift @@ -11,6 +11,10 @@ public enum GuestContract: Sendable { case connectorParams = "connector-params.schema.json" case connectorResultEmit = "connector-result-emit.schema.json" case connectorVendor = "connector-vendor.schema.json" + case guestRuntime = "guest-runtime.schema.json" + case workerProduct = "worker-product.schema.json" + case webCrawlerResult = "web-crawler-result.schema.json" + case fileExtractorResult = "file-extractor-result.schema.json" } public static func loadSchemaText(_ schema: Schema) throws -> String { diff --git a/packages/Structure/Sources/Contract/GuestContractValidation.swift b/packages/Structure/Sources/Contract/GuestContractValidation.swift index 83fcb43..1da6e55 100644 --- a/packages/Structure/Sources/Contract/GuestContractValidation.swift +++ b/packages/Structure/Sources/Contract/GuestContractValidation.swift @@ -9,4 +9,12 @@ public enum GuestContractValidation: Sendable { public static func validateHopEventJSON(_ data: Data) throws { try GuestContract.validate(json: data, against: .hopEvent) } + + public static func validateWebCrawlerResultJSON(_ data: Data) throws { + try GuestContract.validate(json: data, against: .webCrawlerResult) + } + + public static func validateFileExtractorResultJSON(_ data: Data) throws { + try GuestContract.validate(json: data, against: .fileExtractorResult) + } } diff --git a/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json new file mode 100644 index 0000000..c4e12bc --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/file-extractor-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/file-extractor-result.json", + "title": "File extractor worker stdout", + "description": "JSON object written to stdout by derrick-file-extractor.", + "$ref": "worker-product.schema.json#/$defs/file_extractor_result" +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json new file mode 100644 index 0000000..03f4f62 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/guest-runtime.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/guest-runtime.json", + "title": "Derrick offline guest runtime", + "description": "Canonical I/O contract for script_exec and plugin.invoke guests. Trusted worker stdout (crawler, extractor) is defined in worker-product.schema.json. Swift host and Go workers must match these schemas.", + "type": "object", + "required": ["language", "stdin", "stdout"], + "additionalProperties": false, + "properties": { + "language": { + "type": "string", + "const": "go", + "description": "Guest implementation language." + }, + "stdin": { + "description": "One hop event JSON object read from standard input.", + "$ref": "hop-event.schema.json" + }, + "stdout": { + "description": "Envelope list JSON array written to standard output.", + "$ref": "envelope-list.schema.json" + }, + "binary": { + "type": "string", + "const": "/tmp/guest", + "description": "Linux guest binary path inside the worker container." + } + } +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json new file mode 100644 index 0000000..9af3975 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/web-crawler-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/web-crawler-result.json", + "title": "Web crawler worker stdout", + "description": "JSON object written to stdout by derrick-web-crawler.", + "$ref": "worker-product.schema.json#/$defs/web_crawler_result" +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json new file mode 100644 index 0000000..874f6bf --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/worker-product.json", + "title": "Derrick trusted worker product contracts", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "$defs": { + "string_list": { + "type": "array", + "items": { "type": "string" } + }, + "web_crawler_stop_reason": { + "type": "string", + "enum": [ + "completed", + "max_pages", + "max_depth", + "timeout", + "total_bytes", + "queue_limit", + "cancelled", + "blocked" + ] + }, + "web_crawler_page": { + "type": "object", + "required": ["url", "depth", "status_code", "title", "text", "links_found"], + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "depth": { "type": "integer" }, + "status_code": { "type": "integer" }, + "content_type": { "type": "string" }, + "title": { "type": "string" }, + "text": { "type": "string" }, + "links_found": { "type": "integer" } + } + }, + "web_crawler_result": { + "type": "object", + "required": [ + "ok", + "start_url", + "pages", + "stop_reason", + "requests_made", + "bytes_read", + "truncated", + "diagnostics" + ], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "start_url": { "type": "string" }, + "pages": { + "type": "array", + "items": { "$ref": "#/$defs/web_crawler_page" } + }, + "stop_reason": { "$ref": "#/$defs/web_crawler_stop_reason" }, + "requests_made": { "type": "integer" }, + "bytes_read": { "type": "integer" }, + "truncated": { "type": "boolean" }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + }, + "file_extractor_operation": { + "type": "string", + "enum": ["extract", "convert"] + }, + "file_extractor_file_result": { + "type": "object", + "required": ["input_name", "kind", "byte_count"], + "additionalProperties": false, + "properties": { + "input_name": { "type": "string" }, + "output_name": { "type": "string" }, + "kind": { "type": "string" }, + "byte_count": { "type": "integer" }, + "preview": { "type": "string" }, + "error": { "type": "string" } + } + }, + "file_extractor_result": { + "type": "object", + "required": ["ok", "operation", "files", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "operation": { "$ref": "#/$defs/file_extractor_operation" }, + "files": { + "type": "array", + "items": { "$ref": "#/$defs/file_extractor_file_result" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + } + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift new file mode 100644 index 0000000..dc08137 --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Optional host Go toolchain probe (development diagnostics). Guest compile runs in Docker. +public enum DerrickGoToolchain: Sendable { + public static let minimumVersion = "1.27.1" + + public static func ensureInstalled() throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/env") + process.arguments = ["go", "version"] + process.environment = [ + "PATH": "/opt/homebrew/bin:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin", + ] + let stdout = Pipe() + process.standardOutput = stdout + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw DerrickGoToolchainError.missing + } + let text = String(data: stdout.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" + guard let version = parseVersion(text) else { + throw DerrickGoToolchainError.unparseable(text.trimmingCharacters(in: .whitespacesAndNewlines)) + } + guard versionSatisfies(version, minimum: minimumVersion) else { + throw DerrickGoToolchainError.tooOld(found: version, required: minimumVersion) + } + } + + private static func parseVersion(_ text: String) -> String? { + // go version go1.27.1 darwin/arm64 + for part in text.split(separator: " ") { + let token = String(part) + if token.hasPrefix("go"), token.count > 2 { + return String(token.dropFirst(2)) + } + } + return nil + } + + private static func versionSatisfies(_ found: String, minimum: String) -> Bool { + compareVersions(found, minimum) != .orderedAscending + } + + private enum Ordering { + case orderedAscending, orderedSame, orderedDescending + } + + private static func compareVersions(_ lhs: String, _ rhs: String) -> Ordering { + let left = lhs.split(separator: ".").map { Int($0) ?? 0 } + let right = rhs.split(separator: ".").map { Int($0) ?? 0 } + let count = max(left.count, right.count) + for index in 0.. r { return .orderedDescending } + } + return .orderedSame + } +} + +public enum DerrickGoToolchainError: Error, LocalizedError, Sendable { + case missing + case unparseable(String) + case tooOld(found: String, required: String) + + public var errorDescription: String? { + switch self { + case .missing: + return "Go \(DerrickGoToolchain.minimumVersion) or later was expected for diagnostics. Guest compile runs in Docker." + case .unparseable(let detail): + return "Could not read the installed Go version (\(detail))." + case .tooOld(let found, let required): + return "Go \(required) or later is required to build plugins (found \(found))." + } + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift new file mode 100644 index 0000000..baa9c04 --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerImageDigest.swift @@ -0,0 +1,47 @@ +import Foundation + +/// SHA-256 image ID from `docker image inspect --format '{{.Id}}'`. +public struct DockerImageDigest: RawRepresentable, Sendable, Hashable, Codable { + public let rawValue: String + + public init(rawValue: String) { + self.rawValue = Self.normalize(rawValue) + } + + public init?(hexDigest: String) { + let normalized = Self.normalize(hexDigest) + guard Self.isValid(normalized) else { return nil } + rawValue = normalized + } + + public static func normalize(_ value: String) -> String { + let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if trimmed.hasPrefix("sha256:") { + return trimmed + } + if trimmed.count == 64 { + return "sha256:\(trimmed)" + } + return trimmed + } + + public static func isValid(_ value: String) -> Bool { + let hex = value.hasPrefix("sha256:") ? String(value.dropFirst(7)) : value + guard hex.count == 64 else { return false } + return hex.unicodeScalars.allSatisfy { CharacterSet(charactersIn: "0123456789abcdef").contains($0) } + } +} + +public enum DockerImageDigestError: Error, LocalizedError, Sendable, Equatable { + case imageMissing(String) + case digestMismatch(tag: String, expected: DockerImageDigest, actual: DockerImageDigest) + + public var errorDescription: String? { + switch self { + case .imageMissing(let tag): + return "The worker image \(tag) is not installed." + case .digestMismatch(let tag, _, _): + return "The worker image \(tag) does not match the version shipped with Derrick. Rebuild or reinstall product images." + } + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift new file mode 100644 index 0000000..cd1299f --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift @@ -0,0 +1,6 @@ +import Foundation + +/// Generated by scripts/record-docker-image-digests.sh — do not edit. +public enum DockerProductImageDigests: Sendable { + public static let worker = DockerImageDigest(rawValue: "sha256:d686d16d5fb8fe0a54a1eb9bb9c8a27763b5d80145ad330c5205536d73afdda6") +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift index bfbbfd3..9f323fe 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImagePolicy.swift @@ -1,16 +1,39 @@ import Foundation -/// Trusted product images built from in-repo Dockerfiles (not pulled from a registry). +/// Trusted product Docker images built from in-repo Dockerfiles (not pulled from a registry). public enum DockerProductImagePolicy: Sendable { + public static let workerImage = DockerWorkerRuntime.image + public static let workerDockerfileRelativePath = DockerWorkerRuntime.dockerfileRelativePath + public static let workerBuildContextRelativePath = DockerWorkerRuntime.buildContextRelativePath + + /// Legacy crawler tag — retained for orphan sweeps only. public static let webCrawlerImage = "derrick-web-crawler:swift-6.4-v1" public static let webCrawlerDockerfileRelativePath = "docker/web-crawler/Dockerfile" - /// Sibling Swift packages only — not the whole git checkout. public static let webCrawlerBuildContextRelativePath = "packages" public static let allowedBuildImageTags: Set = [ - webCrawlerImage, + workerImage, ] + public static func workerBuildContext(repoRoot: URL) -> URL { + repoRoot.standardizedFileURL + } + + public static func isAllowedWorkerBuild( + dockerfilePath: String, + imageTag: String, + contextPath: String + ) -> Bool { + guard imageTag == workerImage else { return false } + let dockerfileURL = URL(fileURLWithPath: dockerfilePath).standardizedFileURL + let contextURL = URL(fileURLWithPath: contextPath).standardizedFileURL + let repoRoot = contextURL.standardizedFileURL + let expectedDockerfile = repoRoot + .appendingPathComponent(workerDockerfileRelativePath) + .standardizedFileURL + return dockerfileURL.path == expectedDockerfile.path + } + public static func webCrawlerBuildContext(repoRoot: URL) -> URL { repoRoot.appendingPathComponent(webCrawlerBuildContextRelativePath).standardizedFileURL } diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift new file mode 100644 index 0000000..dca7e12 --- /dev/null +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift @@ -0,0 +1,22 @@ +import Foundation + +/// Unified Go worker image shared by crawl, extract, and plugin guest execution. +public enum DockerWorkerRuntime: Sendable { + public static let image = "derrick-worker:go-v1" + public static let dockerfileRelativePath = "docker/worker/Dockerfile" + public static let buildContextRelativePath = "." + + public static let crawlerBinary = "/usr/local/bin/derrick-web-crawler" + public static let extractorBinary = "/usr/local/bin/derrick-file-extractor" + public static let guestBinaryPath = "/tmp/guest" + public static let guestSourcePath = "/tmp/plugin.go" + public static let goBinaryPath = "/usr/local/go/bin/go" + + /// Allowed `docker exec … sh -c` payloads for guest source I/O and in-container compile. + public static let guestWriteSourceShell = "cat > /tmp/plugin.go" + public static let guestCompileShell = + "cd /tmp && /usr/local/go/bin/go build -trimpath -ldflags=-s -w -o guest plugin.go && chmod +x guest" + public static let guestReadBinaryShell = "cat /tmp/guest" + + public static let pinnedDigest = DockerProductImageDigests.worker +} diff --git a/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift b/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift index 5fb77b5..80d556d 100644 --- a/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift +++ b/packages/Structure/Sources/MCPServer/MCPServerContractTypes.swift @@ -2,11 +2,11 @@ import Foundation import MCP public enum GuestScriptLanguage: String, Sendable, Equatable { - case python + case go - public var verifierID: String { "python-check-v1" } + public var verifierID: String { "go-check-v1" } - /// `language` is optional and must be Python when set. + /// `language` is optional and must be Go when set. public static func requestedLanguageIsUnsupported(_ arguments: [String: Value]) -> Bool { guard let raw = arguments["language"]?.stringValue? .trimmingCharacters(in: .whitespacesAndNewlines) @@ -15,7 +15,7 @@ public enum GuestScriptLanguage: String, Sendable, Equatable { else { return false } - return raw != "python" && raw != "py" + return raw != "go" && raw != "golang" } } diff --git a/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift b/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift index 97481ba..3e59478 100644 --- a/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift +++ b/packages/Structure/Sources/MCPServer/ScriptExecutionModels.swift @@ -59,7 +59,7 @@ public enum ScriptFailureStage: String, Codable, Sendable, Equatable { case none /// Static verifier rejected the request before run. case staticValidation - /// Leftover Swift compiler stage. Guest scripts are Python; this case remains for stored outcomes. + /// Go compile stage before container execution. case typecheck /// LLM security reviewer rejected (or could not complete when required). case llmReview @@ -253,7 +253,7 @@ public struct ScriptExecutionResult: Sendable { stderr: String, durationMS: Int, phaseTiming: ScriptPhaseTiming?, - verifier: String = "python-check-v1" + verifier: String = "go-check-v1" ) -> ScriptExecutionResult { let combined = stdout + "\n" + stderr let looksLikeEgress = combined.localizedCaseInsensitiveContains("UNAUTHORIZED_EGRESS") diff --git a/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift b/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift index 2976abe..a2ff70b 100644 --- a/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift +++ b/packages/Structure/Sources/MCPToolCatalog/AllowedMCPTool.swift @@ -43,7 +43,7 @@ public enum AllowedMCPTool: String, CaseIterable, Sendable, Codable, Hashable { public var defaultDescription: String { switch self { case .scriptExec: - return "Run declared standalone Python in a constrained Docker container after verification. Emit HTTP request envelopes; the host performs the request." + return "Run declared standalone Go in a constrained Docker container after verification. Emit HTTP request envelopes; the host performs the request." case .sessionMemorySearch: return "Search prior session memory entries with optional query and paging." case .agentsSpawn: @@ -69,7 +69,7 @@ public enum AllowedMCPTool: String, CaseIterable, Sendable, Codable, Hashable { case .pluginInvoke: return "Run one approved compiled Agent Plugin by id with a JSON input object." case .webCrawl: - return "Crawl a bounded same-origin website in an isolated Swift container and return structured page summaries." + return "Crawl a bounded same-origin website in an isolated container and return structured page summaries. Call directly in chat; use jobs_create only for crawls likely to exceed about one minute." case .filesExtract: return "Extract text or convert attached chat files (PDF, DOCX, XLSX, CSV, HTML) in an isolated Swift container. Call this tool directly; do not submit it through jobs_create." } diff --git a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift new file mode 100644 index 0000000..175d314 --- /dev/null +++ b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift @@ -0,0 +1,106 @@ +import Foundation + +/// Go contract shown to models that generate Derrick plugin guest programs. +public enum DerrickGuestGo: Sendable { + public static let modelContract = """ + Go guest contract: + - The program is a standalone `package main` built to a Linux binary and run as `/tmp/guest`. + - Read one JSON object from standard input matching the hop-event schema. + - Write one JSON array of envelope objects to standard output matching the envelope-list schema. + - Every envelope object needs `verb` from the envelope-list schema. + - POST bodies go in `json`. The host decodes `http.request` as HostHTTPRequest and sends `json` as the HTTP body. + - On the first event, emit `http.request` envelopes for host HTTP. + - On an `http_results` event, emit `result.emit` or `message.post`. + - The host, not the guest container, performs HTTP and supplies response bodies. + - Do not import net/http, net, os/exec, os (except os.Stdin/os.Stdout), or perform filesystem access. + - Use only the Go standard library. + - For repeatable output, match HTTP responses by request_id, sort and de-duplicate collections + by stable keys, and never use current time, randomness, UUIDs, response arrival order, or + map iteration order for user-visible output. + - Later http_results events include earlier responses plus the newest ones. Match by request_id. + + Minimal output pattern: + ```go + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + if err := json.NewDecoder(os.Stdin).Decode(&event); err != nil { + return + } + emit([]map[string]any{{"verb": "result.emit", "title": "Result", "summary": "done"}}) + } + + func emit(envelopes []map[string]any) { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode(envelopes) + } + ``` + Inspect `event["kind"]` and `event["http_results"]` to choose the next envelopes. + """ + + public static func source(for spec: PluginSpec? = nil) throws -> String { + var sections = [modelContract] + sections.append( + """ + Canonical JSON schemas (Swift host and Go guest must match exactly): + --- \(GuestContract.Schema.guestRuntime.rawValue) --- + \(try GuestContract.loadSchemaText(.guestRuntime)) + + --- \(GuestContract.Schema.hopEvent.rawValue) --- + \(try GuestContract.loadSchemaText(.hopEvent)) + + --- \(GuestContract.Schema.envelopeList.rawValue) --- + \(try GuestContract.loadSchemaText(.envelopeList)) + """ + ) + if let spec { + sections.append( + """ + Plugin parameters are delivered in the input object's `params` object. + The parameter contract is: + \(try spec.goParameterDeclaration()) + + --- \(GuestContract.Schema.connectorParams.rawValue) --- + \(try GuestContract.loadSchemaText(.connectorParams)) + """ + ) + } + return sections.joined(separator: "\n\n") + } +} + +private extension PluginSpec { + func goParameterDeclaration() throws -> String { + _ = try validated() + let fields = parameters.map { parameter in + " \(parameter.name) \(parameterType(parameter.type))" + } + return """ + type PluginParams struct { + \(fields.joined(separator: "\n")) + } + """ + } + + func parameterType(_ type: PluginParameterType) -> String { + switch type { + case .string: + return "string" + case .number: + return "float64" + case .boolean: + return "bool" + case .stringList: + return "[]string" + case .numberList: + return "[]float64" + } + } +} diff --git a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift deleted file mode 100644 index 3ef8d5e..0000000 --- a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestPython.swift +++ /dev/null @@ -1,81 +0,0 @@ -import Foundation - -/// Python contract shown to models that generate standalone Derrick guest programs. -public enum DerrickGuestPython: Sendable { - public static let modelContract = """ - Python guest contract: - - The program is a standalone Python script run as `python3 /tmp/guest.py`. - - Read one JSON object from standard input (`sys.stdin`). - - Write one JSON array of envelope objects to standard output. Every object needs `verb` from the envelope-list schema. - - POST bodies go in `json`. The host decodes `http.request` as HostHTTPRequest and sends `json` as the HTTP body. - - On the first event, emit `http.request` envelopes for host HTTP. - - On an `http_results` event, emit `result.emit` or `message.post`. - - The host, not the Python container, performs HTTP and supplies response bodies. - - Do not use socket, urllib, requests, httpx, subprocess, or filesystem access. - - Use only the Python standard library (no pip/uv dependencies in script_exec). - - For repeatable output, match HTTP responses by request_id, sort and de-duplicate collections - by stable keys, and never use current time, randomness, UUIDs, response arrival order, or - dict/set iteration order for user-visible output. - - Later http_results events include earlier responses plus the newest ones. Match by request_id. - - Minimal output pattern: - ```python - import json, sys - event = json.load(sys.stdin) - def emit(envelopes): - json.dump(envelopes, sys.stdout, separators=(",", ":")) - ``` - Inspect `event.get("kind")` and `event.get("http_results")` to choose the next envelopes. - - Example request envelope: - {"verb":"http.request","request_id":"news-1","method":"GET","url":"https://example.com/feed.xml"} - - POST bodies: put the JSON value in `json`. The host deserializes `http.request` into HostHTTPRequest and sends `json` as the HTTP body. - {"verb":"http.request","request_id":"send-1","method":"POST","url":"https://example.com/api","headers":{"Content-Type":"application/json"},"json":{"channel":"C1","text":"hello"}} - - Example result envelope: - {"verb":"result.emit","title":"Result","summary":"User-readable output"} - """ - - public static func source(for spec: PluginSpec? = nil) throws -> String { - var sections = [modelContract] - if let spec { - sections.append( - """ - Plugin parameters are delivered in the input object's `params` object. - The parameter contract is: - \(try spec.pythonParameterDeclaration()) - """ - ) - } - return sections.joined(separator: "\n\n") - } -} - -private extension PluginSpec { - func pythonParameterDeclaration() throws -> String { - _ = try validated() - let fields = parameters.map { parameter in - " \(parameter.name): \(parameterType(parameter.type))" - } - return """ - class PluginParams(TypedDict): - \(fields.joined(separator: "\n")) - """ - } - - func parameterType(_ type: PluginParameterType) -> String { - switch type { - case .string: - return "str" - case .number: - return "float" - case .boolean: - return "bool" - case .stringList: - return "list[str]" - case .numberList: - return "list[float]" - } - } -} diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift index 5548c6a..4a812e6 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift @@ -1,7 +1,7 @@ import Foundation public enum PluginGuestLanguage: String, Sendable, Equatable, Codable { - case python + case go } /// Parsed `app.derrick/runtime.json` from an approved factory release. @@ -9,7 +9,7 @@ public struct PluginFactoryRuntime: Sendable, Equatable { public let language: PluginGuestLanguage public let entrypoint: String - public init(language: PluginGuestLanguage = .python, entrypoint: String) { + public init(language: PluginGuestLanguage = .go, entrypoint: String) { self.language = language self.entrypoint = entrypoint } @@ -22,6 +22,8 @@ public struct PluginFactoryRuntime: Sendable, Equatable { else { return nil } - return PluginFactoryRuntime(entrypoint: entrypoint) + let languageRaw = (object["language"] as? String) ?? PluginGuestLanguage.go.rawValue + let language = PluginGuestLanguage(rawValue: languageRaw) ?? .go + return PluginFactoryRuntime(language: language, entrypoint: entrypoint) } } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift index aaed199..7416987 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTestScript.swift @@ -77,7 +77,10 @@ public enum PluginFactoryHopTestRunner: Sendable { testInput: Data, executor: any PluginFactoryExecutor ) async throws -> PluginFactoryHopTestRun { - try await run(testInput: testInput) { input in + if let compiled = executor as? any PluginFactoryCompiledGuestExecutor { + return try await compiled.runGuestSourceHops(source: source, testInput: testInput) + } + return try await run(testInput: testInput) { input in try await executor.runGuestSource(source: source, input: input) } } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift index bea96cf..2b37ff7 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift @@ -2,9 +2,9 @@ import Foundation public typealias PluginFactoryLogger = @Sendable (String) async -> Void -/// The factory creates Agent Plugin packages whose Derrick entrypoint is Python. -/// A draft is a standalone file: the container runs it with `python3 /tmp/guest.py`. -/// A released version stores UTF-8 source as the packaged artifact. +/// The factory creates Agent Plugin packages whose Derrick entrypoint is Go. +/// A draft is compiled to a Linux binary and run as `/tmp/guest` in the worker image. +/// A released version stores the compiled binary as the packaged artifact. public struct PluginFactoryDraft: Sendable, Hashable { public let manifestJSON: String public let guestSource: String @@ -146,7 +146,7 @@ public struct PluginFactoryManifestInput: Sendable, Hashable { throw PluginFactoryError.invalidManifest("Version is required.") } var derrick: [String: Any] = [ - "entrypoint": "./app.derrick/plugin.py", + "entrypoint": "./app.derrick/plugin.go", ] if !secrets.isEmpty { derrick["secrets"] = secrets.map(\.jsonObject) @@ -204,7 +204,7 @@ public struct PluginFactoryBuilderRequest: Sendable, Hashable { public let userGoal: String public let previousDraft: PluginFactoryDraft? public let feedback: String? - /// When set, the host writes `plugin.json`. The builder only supplies Python and tests. + /// When set, the host writes `plugin.json`. The builder only supplies Go source and tests. public let hostManifest: PluginFactoryManifestInput? public init( @@ -288,7 +288,8 @@ public struct PluginFactoryBuilderResponse: Codable, Sendable, Hashable { enum CodingKeys: String, CodingKey { case pluginID = "plugin_id" case version, description - case guestSource = "python_source" + case guestSource = "go_source" + case legacyPythonSource = "python_source" case legacySwiftSource = "swift_source" case testInputJSON = "test_input_json" case skillFiles = "skill_files" @@ -303,6 +304,7 @@ public struct PluginFactoryBuilderResponse: Codable, Sendable, Hashable { version = try container.decode(String.self, forKey: .version) description = try container.decode(String.self, forKey: .description) guestSource = try container.decodeIfPresent(String.self, forKey: .guestSource) + ?? container.decodeIfPresent(String.self, forKey: .legacyPythonSource) ?? container.decode(String.self, forKey: .legacySwiftSource) testInputJSON = try container.decode(String.self, forKey: .testInputJSON) skillFiles = try container.decodeIfPresent([PluginFactorySkillFile].self, forKey: .skillFiles) ?? [] @@ -380,14 +382,19 @@ public struct PluginFactoryExecutionResult: Sendable, Hashable { } } -/// The host supplies this adapter. Its production implementation runs these -/// commands inside the restricted Linux Swift Docker container. +/// The host supplies this adapter. Its production implementation compiles and +/// runs guests inside the pinned Go worker Docker container. public protocol PluginFactoryExecutor: Sendable { func runGuestSource(source: String, input: Data) async throws -> PluginFactoryExecutionResult func packageGuestSource(source: String) async throws -> Data func runPackagedArtifact(_ artifact: Data, input: Data) async throws -> PluginFactoryExecutionResult } +/// Optional compile-once hop replay for factory direct tests. +public protocol PluginFactoryCompiledGuestExecutor: PluginFactoryExecutor { + func runGuestSourceHops(source: String, testInput: Data) async throws -> PluginFactoryHopTestRun +} + public enum PluginReviewDecision: String, Sendable, Hashable { case approved case rejected @@ -529,7 +536,7 @@ public struct PluginFactoryRelease: Sendable, Hashable { var files: [String: Data] = [ "plugin.json": Data(manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.py": Data(guestSource.utf8), + "app.derrick/plugin.go": Data(guestSource.utf8), "app.derrick/plugin": compiledArtifact, ] for (path, body) in skillFiles { @@ -571,15 +578,15 @@ public enum PluginFactoryError: Error, LocalizedError, Equatable, Sendable { case .invalidSkillPath(let path): return "Invalid skill path '\(path)'. Skill path must be skills//SKILL.md." case .reservedPluginID(let id): return "The plugin id '\(id)' is reserved by Derrick." - case .invalidSource(let message): return "Invalid Python guest source: \(message)" - case .directRunFailed(let message): return "Python draft test failed: \(message)" - case .invalidDirectOutput(let message): return "Python draft returned invalid plugin output: \(message)" + case .invalidSource(let message): return "Invalid Go guest source: \(message)" + case .directRunFailed(let message): return "Go draft test failed: \(message)" + case .invalidDirectOutput(let message): return "Go draft returned invalid plugin output: \(message)" case .reviewRejected(let summary, let findings): let detail = findings.isEmpty ? summary : "\(summary) \(findings.joined(separator: " "))" return "Plugin review rejected the draft: \(detail)" - case .packageFailed(let message): return "Python plugin packaging failed: \(message)" + case .packageFailed(let message): return "Go plugin packaging failed: \(message)" case .packagedRunFailed(let message): return "Packaged plugin test failed: \(message)" case .invalidPackagedOutput(let message): return "Packaged plugin returned invalid output: \(message)" case .draftValidationFailed(let findings): diff --git a/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift b/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift index f5cf914..fd5df8c 100644 --- a/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift +++ b/packages/Structure/Sources/Plugin/Manifest/PluginPath.swift @@ -19,18 +19,18 @@ public enum PluginPath { return trimmed } - /// Python factory entrypoints are standalone files run by `python3`. - public static func validatePythonEntrypoint(_ raw: String) throws -> String { + /// Go factory entrypoints are standalone `package main` files compiled to `/tmp/guest`. + public static func validateGoEntrypoint(_ raw: String) throws -> String { let path = try validateRelative(raw) - guard path.hasSuffix(".py") else { + guard path.hasSuffix(".go") else { throw PluginManifestError.invalidEntrypoint(raw) } return path } - /// Accepts the supported Derrick guest runtime source file (.py). + /// Accepts the supported Derrick guest runtime source file (.go). public static func validateRuntimeEntrypoint(_ raw: String) throws -> String { - try validatePythonEntrypoint(raw) + try validateGoEntrypoint(raw) } public static func resolve(root: URL, relative: String) throws -> URL { diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index 5d8b258..e594bbf 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -88,7 +88,7 @@ import Testing let scriptReviewer = try DerrickBundledText.load("script_reviewer_instructions.md") #expect(scriptReviewer.contains("intent alignment")) #expect(scriptReviewer.contains("secret literals")) - #expect(scriptReviewer.contains("Python verifier")) + #expect(scriptReviewer.contains("Go verifier")) } @Test func healthDecodesLegacyPayloadWithoutGuestRuntime() throws { @@ -369,7 +369,7 @@ import Testing @Test func slackConnectorFallsBackToBotTokenWhenManifestOmitsSecrets() { let json = """ {"$schema":"https://example.invalid/agent-plugin.json","name":"slack-connector","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads"]}}} """ let descriptors = PluginSecretField.resolvedDescriptors( pluginID: "slack-connector", @@ -1022,15 +1022,15 @@ import Testing reportedFingerprint: "a", expectedFingerprint: "a", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( !DerrickDaemonHygiene.shouldRetireConnectedDaemon( reportedFingerprint: "a", expectedFingerprint: "a", - reportedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + reportedGuestRuntime: DerrickGuestRuntime.guestDockerImage, + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1038,7 +1038,7 @@ import Testing reportedFingerprint: "old", expectedFingerprint: "new", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1046,7 +1046,7 @@ import Testing reportedFingerprint: "a", expectedFingerprint: "a", reportedGuestRuntime: "stale-guest:old", - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1054,7 +1054,7 @@ import Testing reportedFingerprint: nil, expectedFingerprint: "a", reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) #expect( @@ -1062,7 +1062,7 @@ import Testing reportedFingerprint: "a", expectedFingerprint: nil, reportedGuestRuntime: DerrickGuestRuntime.swiftPluginDockerImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) ) } @@ -1317,12 +1317,12 @@ import Testing ) } - @Test func effectorAdmissionDeniesLiveChatWithoutContext() { + @Test func effectorAdmissionAllowsLiveChatWithoutContext() { #expect( EffectorAdmissionPolicy.allowsSyncWebCrawl( context: nil, principal: .agent(sessionID: "s1", agentID: "a1") - ) == false + ) ) } diff --git a/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift b/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift index 8bde081..6ab15de 100644 --- a/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift +++ b/packages/Structure/Tests/StructureTests/ConnectorContractTests.swift @@ -204,7 +204,7 @@ private func slackFullSyncGoal() -> String { private func slackFullSyncManifestJSON() -> String { """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connection","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ } diff --git a/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift new file mode 100644 index 0000000..6587b5d --- /dev/null +++ b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift @@ -0,0 +1,12 @@ +import Structure +import Testing + +@Suite struct DerrickGoToolchainTests { + @Test func minimumVersionIsPinned() { + #expect(DerrickGoToolchain.minimumVersion == "1.27.1") + } + + @Test func ensureInstalledAcceptsCurrentGo() throws { + try DerrickGoToolchain.ensureInstalled() + } +} diff --git a/packages/Structure/Tests/StructureTests/GuestContractTests.swift b/packages/Structure/Tests/StructureTests/GuestContractTests.swift index d8c3d83..430d134 100644 --- a/packages/Structure/Tests/StructureTests/GuestContractTests.swift +++ b/packages/Structure/Tests/StructureTests/GuestContractTests.swift @@ -10,6 +10,12 @@ import Testing } } + @Test func guestRuntimeSchemaRequiresGoLanguage() throws { + let schema = try GuestContract.loadSchemaObject(.guestRuntime) + let language = (schema["properties"] as? [String: Any])?["language"] as? [String: Any] + #expect(language?["const"] as? String == "go") + } + @Test func executionContextSchemaExposesWorkflowKinds() throws { let kinds = try GuestContract.officialWorkflowKinds() #expect(kinds.contains("plugin_factory_create")) @@ -104,4 +110,36 @@ import Testing try GuestContract.validate(json: Data(json.utf8), against: .envelopeList) } } + + @Test func webCrawlerResultValidationAcceptsMinimalSuccess() throws { + let json = """ + {"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":[]} + """ + try GuestContractValidation.validateWebCrawlerResultJSON(Data(json.utf8)) + } + + @Test func webCrawlerResultValidationRejectsNullDiagnostics() { + let json = """ + {"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":null} + """ + #expect(throws: GuestContractError.self) { + try GuestContractValidation.validateWebCrawlerResultJSON(Data(json.utf8)) + } + } + + @Test func fileExtractorResultValidationAcceptsMinimalSuccess() throws { + let json = """ + {"ok":true,"operation":"extract","files":[],"diagnostics":[]} + """ + try GuestContractValidation.validateFileExtractorResultJSON(Data(json.utf8)) + } + + @Test func fileExtractorResultValidationRejectsNullFiles() { + let json = """ + {"ok":false,"operation":"extract","files":null,"diagnostics":[]} + """ + #expect(throws: GuestContractError.self) { + try GuestContractValidation.validateFileExtractorResultJSON(Data(json.utf8)) + } + } } diff --git a/readme.md b/readme.md index e02a8bc..dbf7d9f 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ A native Swift macOS 27 desktop agent harness: chat with LLM providers, run isol |------|-------------| | **Chat** | Multi-tab conversations with OpenAI, Gemini, and other configured models | | **Tools (MCP)** | Model Context Protocol tool host inside the headless daemon | -| **Scripts** | Agent-generated Python executed in isolated Docker containers. Includes a secondary agent code reviewer and approvals flow. | +| **Scripts** | Agent-generated Go executed in isolated Docker containers. Includes a secondary agent code reviewer and approvals flow. | | **Plugin factory** | LLM-assisted creation of versioned, reviewed connector plugins | | **Jobs** | Scheduled and deferred tool/agent runs that survive app quit | | **Messaging** | Connector plugins (e.g. Slack) with threads, history, and live sync | @@ -35,14 +35,14 @@ A native Swift macOS 27 desktop agent harness: chat with LLM providers, run isol └────────────────┘ └───────────┬────────────┘ │ ┌───────────▼────────────┐ - │ Python guest containers │ + │ Go worker containers │ │ --network none │ └────────────────────────┘ ``` - **UI** is a client: it does not own agent turns or MCP when the daemon is up. - **Daemon** (`derrickd`) is the single owner of OS notifications and in-process Agent/Job/MCP modules. -- **Docker** runs untrusted Python for `script_exec`, plugin factory builds, and approved plugin invocations. +- **Docker** runs untrusted Go for `script_exec`, plugin factory builds, and approved plugin invocations. See [docs/adr-headless-backend.md](docs/adr-headless-backend.md) and [docs/services-plan.md](docs/services-plan.md). @@ -50,10 +50,12 @@ See [docs/adr-headless-backend.md](docs/adr-headless-backend.md) and [docs/servi Derrick treats model output and guest code as untrusted. -### Docker sandbox (Python guest runtime) +### Docker sandbox (Go worker runtime) -- Guest programs run in `python:3.14.7` containers with **`--network none`**. -- No sockets, urllib/requests, subprocess, or credentials inside the guest. +- **`script_exec`, plugins, web crawl, and file extract** share one Go worker image `derrick-worker:go-v1` (digest-pinned, `--network none` for guests). +- Plugin and script sources are compiled **inside the worker container** (Go 1.27.1 in the image). Users only need Docker Desktop. +- Canonical I/O types live in `packages/Structure/Sources/Contract/Resources/schemas/` and are mirrored to `workers/go/internal/contract/schemas/`. +- No net/http, subprocess, filesystem access, or credentials inside the guest. - The host dispatches `http.request` envelopes, attaches secrets, and enforces egress policy. - Historical Swift guest notes: [docs/adr-swift-script-runtime.md](docs/adr-swift-script-runtime.md). @@ -65,7 +67,7 @@ Before `script_exec` writes to disk, a **configured LLM reviewer** checks: - No secret literals in source - Safe handling of fetched content (no raw HTML leakage unless requested) -Instructions live in `ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md`. A static **Python verifier** also rejects forbidden APIs. +Instructions live in `ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md`. A static **Go verifier** also rejects forbidden APIs. ### Egress & network diff --git a/scripts/record-docker-image-digests.sh b/scripts/record-docker-image-digests.sh new file mode 100755 index 0000000..4642aa4 --- /dev/null +++ b/scripts/record-docker-image-digests.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Builds the unified Go worker image and updates the pinned digest in Structure. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +IMAGE="derrick-worker:go-v1" +docker build -f docker/worker/Dockerfile -t "$IMAGE" . + +DIGEST="$(docker image inspect --format '{{.Id}}' "$IMAGE")" +OUT="packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift" + +cat >"$OUT" </dev/null; then + echo "Guest contract schemas are out of sync. Run: scripts/sync-guest-contract-schemas.sh" >&2 + diff -qr "$SRC" "$DST" >&2 || true + exit 1 +fi + +echo "Guest contract schemas are in sync." diff --git a/ui/AgentService/AgentServiceExportedObject.swift b/ui/AgentService/AgentServiceExportedObject.swift index 456bfa4..6336a22 100644 --- a/ui/AgentService/AgentServiceExportedObject.swift +++ b/ui/AgentService/AgentServiceExportedObject.swift @@ -67,7 +67,7 @@ final class AgentServiceExportedObject: NSObject, AgentServiceXPC { service: .agent, status: .ok, detail: "AgentService ready (DB+\(leaf))", - guestRuntimeImage: DerrickProcessRole.isDaemon ? DerrickGuestRuntime.pythonGuestDockerImage : nil + guestRuntimeImage: DerrickProcessRole.isDaemon ? DerrickGuestRuntime.guestDockerImage : nil ) let data = (try? AgentServiceXPCCodec.encodeHealth(report)) ?? Data("{}".utf8) reply(data as NSData) diff --git a/ui/AgentService/AgentServiceTurnHost.swift b/ui/AgentService/AgentServiceTurnHost.swift index 84e9efb..e4578ea 100644 --- a/ui/AgentService/AgentServiceTurnHost.swift +++ b/ui/AgentService/AgentServiceTurnHost.swift @@ -223,7 +223,6 @@ actor AgentServiceTurnHost { LLMModelSettings(repository: repo) } helperModelSettings = settings - await EgressAllowlistService.shared.configure(repository: repo) await ContentSensitivityGrantService.shared.configure(repository: repo) await UsageLimitsService.shared.configure(repository: repo) await ContainerLifecycleSettingsService.shared.configure(repository: repo) diff --git a/ui/MCPService/MCPServiceDockerHelperRunner.swift b/ui/MCPService/MCPServiceDockerHelperRunner.swift index 5ed26f9..2106122 100644 --- a/ui/MCPService/MCPServiceDockerHelperRunner.swift +++ b/ui/MCPService/MCPServiceDockerHelperRunner.swift @@ -63,12 +63,9 @@ final class MCPServiceDockerHelperRunner: @unchecked Sendable { await DerrickDockerOrphanSweeper.sweep(executor: makeStdinCLIExecutor()) } - /// Prewarm the shared offline guest runtime image. + /// Prewarm the shared Go worker image used by script_exec and plugin.invoke. func prewarmGuestRuntime() async throws { - try await OneshotDockerContainer.ensurePulledImage( - DerrickGuestRuntime.pythonGuestDockerImage, - executor: makeStdinCLIExecutor() - ) + try await WorkerImageGate.shared.ensureReady(executor: makeStdinCLIExecutor()) } /// Build the crawler image in the background. Joins an in-flight build if one exists. diff --git a/ui/MCPService/MCPServiceToolHost.swift b/ui/MCPService/MCPServiceToolHost.swift index b44967f..8808e4c 100644 --- a/ui/MCPService/MCPServiceToolHost.swift +++ b/ui/MCPService/MCPServiceToolHost.swift @@ -42,7 +42,7 @@ actor MCPServiceToolHost { LLMModelThinkingSettings(repository: repo) } await factoryThinkingSettings.loadSettings() - let factoryExecutor = PythonPluginFactoryDockerExecutor( + let factoryExecutor = GoPluginFactoryDockerExecutor( executor: MCPServiceDockerHelperRunner.shared.makeStdinCLIExecutor() ) let webCrawlerExecutor = WebCrawlerDockerExecutor( @@ -233,32 +233,6 @@ actor MCPServiceToolHost { message: "Tool \(toolName) is owned by AgentService, not MCPService." ) } - if toolName == AllowedMCPTool.webCrawl.rawValue, - !EffectorAdmissionPolicy.allowsSyncWebCrawl( - context: EffectorAdmissionPolicy.parseContextJSON(request.executionContextJSON) - ?? legacyExecutionContext(from: request), - principal: request.principal - ) { - let outcome = ToolExecutionOutcome.failure( - status: .blocked, - stage: .validation, - diagnostics: [ - ToolExecutionOutcome.Diagnostic( - code: "web_crawl_requires_notification", - message: "web.crawl must be submitted through jobs_create so the result can arrive in a notification banner." - ) - ], - retry: ToolExecutionOutcome.Retry(allowed: false) - ) - return MCPToolCallResultDTO( - requestID: request.requestID, - ok: true, - isError: true, - text: (try? outcome.encodedJSON()) ?? "", - message: "Submit web.crawl through jobs_create for notification delivery." - ) - } - let sessionKey: MemorySessionKey switch request.principal { case .agent(let sessionID, let agentID): diff --git a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift index edd1d54..de55fe9 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift @@ -122,7 +122,7 @@ struct ConversationPipeline: Sen if !toolInstructions.isEmpty { sections.append(toolInstructions) } - if !toolInstructions.contains("Python guest contract:") { + if !toolInstructions.contains("Go guest contract:") { if let sdk = try? PromptResources.guestSDKForModel() { sections.append(sdk) } diff --git a/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift b/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift index 540ea80..04547a7 100644 --- a/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift +++ b/ui/SharedAgentRuntime/Job/JobNetworkPreflight.swift @@ -5,9 +5,8 @@ import Plugin import PolicyUserInteraction import Structure -/// Before a scheduled network tool runs, ensure hosts are allowlisted. -/// Uncovered hosts use the HITL **banner** path (not live chat modals / schedule preflight). -/// +/// Before a scheduled network tool runs, prompt only on blacklist hits. +/// Public HTTPS is allowed by default; hard-blocked SSRF targets are denied without a prompt. public enum JobNetworkPreflight { public static func approveScriptNetworkIfNeeded( toolName: String, @@ -30,6 +29,13 @@ public enum JobNetworkPreflight { } guard !hosts.isEmpty else { return } + let hardBlockPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) + for host in hosts { + if hardBlockPolicy.isHardBlockedHostname(host) { + throw JobNetworkPreflightError.hardBlocked(host: host) + } + } + let blacklist = try await repository.listEgressBlacklist() let exceptions = try await repository.listEgressBlacklistExceptions() for host in hosts { @@ -40,13 +46,15 @@ public enum JobNetworkPreflight { ) else { continue } + let argumentsJSON = blacklistArgumentsJSON(host: host, entry: entry, toolName: toolName) let decision = await HITLOfflineNetworkService.awaitDecision( host: host, toolName: toolName, turnID: "job-\(jobID)", isJobContext: true, repository: repository, - timeoutNanoseconds: 300_000_000_000 + timeoutNanoseconds: 300_000_000_000, + argumentsJSON: argumentsJSON ) switch decision { case .approved, .approvedOnce: @@ -62,83 +70,28 @@ public enum JobNetworkPreflight { } } - let suffixes = try await loadEnabledSuffixes(repository: repository) - - let policy = DefaultDestinationPolicy(allowedDomainSuffixes: suffixes) - var uncovered: [String] = [] - for host in hosts { - if policy.isHardBlockedHostname(host) { - throw JobNetworkPreflightError.hardBlocked(host: host) - } - if policy.isHostCoveredByAllowlist(host) { - continue - } - uncovered.append(host) - } - guard !uncovered.isEmpty else { - fputs( - "[JobNetworkPreflight] job=\(jobID) hosts covered count=\(hosts.count)\n", - stderr - ) - return - } - fputs( - "[JobNetworkPreflight] job=\(jobID) banner approval needed hosts=\(uncovered.joined(separator: ","))\n", + "[JobNetworkPreflight] job=\(jobID) ok hosts=\(hosts.count)\n", stderr ) - - var sessionGrants: [String] = [] - var allowedSuffixes = suffixes - for host in uncovered { - let coverage = DefaultDestinationPolicy(allowedDomainSuffixes: allowedSuffixes) - if coverage.isHostCoveredByAllowlist(host) { - continue - } - // Session grants from earlier Allow Once in this preflight (suffix-scoped). - let sessionPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) - sessionPolicy.grantSessionHosts(sessionGrants) - if sessionPolicy.isHostCoveredByAllowlist(host) { - continue - } - let decision = await HITLOfflineNetworkService.awaitDecision( - host: host, - toolName: toolName, - turnID: "job-\(jobID)", - isJobContext: true, - repository: repository, - timeoutNanoseconds: 300_000_000_000 - ) - switch decision { - case .approvedPermanently(let actor): - let suffix = EgressHostExtractor.permanentSuffix(for: host) - try await repository.saveEgressAllowedDomainSuffix( - EgressAllowedDomainSuffix(suffix: suffix, source: actor ?? "job-banner", enabled: true) - ) - allowedSuffixes = try await loadEnabledSuffixes(repository: repository) - fputs( - "[JobNetworkPreflight] always host=\(host) suffix=\(suffix) actor=\(actor ?? "?")\n", - stderr - ) - case .approved(let actor), .approvedOnce(let actor): - sessionGrants.append(host) - fputs( - "[JobNetworkPreflight] once host=\(host) actor=\(actor ?? "?")\n", - stderr - ) - case .denied(let actor): - throw JobNetworkPreflightError.denied(host: host, actor: actor) - case .dismissed: - throw JobNetworkPreflightError.denied(host: host, actor: "system-dismissed") - case .timedOut: - throw JobNetworkPreflightError.denied(host: host, actor: "system-timeout") - } - } - } - private static func loadEnabledSuffixes(repository: DBRepository) async throws -> [String] { - let rows = try await repository.loadEgressAllowedDomainSuffixes(includeDisabled: false) - return rows.filter(\.enabled).map(\.suffix) + private static func blacklistArgumentsJSON( + host: String, + entry: BlacklistEntry, + toolName: String + ) -> String { + let payload: [String: String] = [ + "host": host, + "url": "https://\(host)", + "toolName": toolName, + "kind": "blacklist", + "pattern": entry.displayPattern, + ] + guard let data = try? JSONSerialization.data(withJSONObject: payload), + let json = String(data: data, encoding: .utf8) else { + return #"{"host":"\#(host)","toolName":"\#(toolName)","kind":"blacklist","pattern":"\#(entry.displayPattern)"}"# + } + return json } } diff --git a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md index 1c488a0..2e186cf 100644 --- a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md +++ b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md @@ -28,6 +28,11 @@ Always respond using the required JSON schema (`thinking` / `tool_call` / `tool_ When presenting a list of choices, options, steps, items, or alternative paths to the user, ALWAYS format them as a clean Markdown bulleted list (using `-` or `*`) or a numbered list (using `1.`, `2.`), instead of writing them as plain paragraphs. +## Website crawling (`web.crawl`) + +- Call `web.crawl` **directly** in live chat for typical same-origin crawls; wait for the result in the same turn. +- Use `jobs_create` with `tool_name` `web.crawl` only when you judge the crawl is likely to take **more than about one minute** (large `max_pages`, deep `max_depth`, or long `timeout_seconds`). Set `wake_after: true` and a short `wake_prompt` so the user gets a notification when it finishes. + ## Scheduled jobs (`jobs_create`) - `wake_after` (default **true**): when **true**, the agent is woken after a **successful** run to summarize the tool result and the user gets a notification + result panel. When **false**, success is silent (no wake, no notification). diff --git a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md index 8e01539..6fdc036 100644 --- a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md +++ b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md @@ -7,10 +7,10 @@ - Set `status` to "tool_call" when you need to execute a single tool, and populate the `tool_call` object with your target `tool_name` and a stringified, JSON-formatted string of tool arguments under the `arguments` key. - Set `status` to "tool_batch" when you need to execute multiple tools in parallel, and populate the `tool_batch` object with your list of `invocations`. - Set `status` to "complete" when you have finished and are responding directly to the user, and populate the `assistant_response` field with your Markdown reply. - - Pass tool `arguments` as a **stringified JSON object** under the `arguments` key (schema requirement). Prefer short Python source and avoid embedding unescaped double quotes in the script body. + - Pass tool `arguments` as a **stringified JSON object** under the `arguments` key (schema requirement). Prefer short Go source and avoid embedding unescaped double quotes in the script body. 6. Users should not have to name tools. Choose tools autonomously from intent. 7. Use `files.extract` for attached PDFs, Office documents, HTML, CSV, and Excel. Use `script_exec` for other scripting/automation. Use `web.crawl` for live website access. - 1. For `script_exec`, use standalone **Python** only. Read one JSON event from standard input and write a JSON **array** of envelopes to standard output. Do not use sockets, urllib/requests, subprocess, shell commands, credentials, or package dependencies. + 1. For `script_exec`, use standalone **Go** (`package main`) only. Read one hop-event JSON object from standard input and write an envelope-list JSON **array** to standard output. Do not import net/http, net, os/exec, or perform filesystem access. No third-party modules. 2. The container has no network. Emit `http.request` envelopes; the host performs HTTP and invokes the guest program again with an `http_results` event. 3. On the first hop emit `{"verb":"http.request","request_id":"…","method":"GET","url":"…"}`. On `http_results`, parse the supplied UTF-8 body and emit `result.emit` or `message.post`. 4. The script must complete the user's requested extraction or summary, not only prove that a fetch happened. Never emit `repr(http_results)` or copy an entire fetched body into `content` unless the user explicitly requested the raw source. For HTML/XML, remove scripts and styles, extract the relevant visible fields, normalize the text, and cap the result. If raw HTML is explicitly requested, emit it in `html`; the host sanitizes that field. @@ -21,11 +21,13 @@ findings as correction feedback and make at most one corrected `script_exec` call before answering. Do not repeat the same script unchanged. If the reviewer identifies a security refusal or the corrected call also fails, report the exact finding instead of claiming success. -9. Use `web.crawl` for website crawling instead of generating a crawler script. Because a crawl - can run for a long time, submit it through `jobs_create` with `tool_name` set to `web.crawl`, - `wake_after` set to true, and a short `wake_prompt` that tells the agent to present the crawl - result to the user. The immediate response must say the crawl was submitted and that the - result will arrive in a notification banner. Never request more than 900 seconds. +9. Use `web.crawl` for website crawling instead of generating a crawler script. Call it directly + in live chat so the result returns in the same turn. Only submit a crawl through `jobs_create` + when you judge it is likely to take more than about one minute (for example a large `max_pages`, + deep `max_depth`, many start URLs, or a long `timeout_seconds`). For background crawls use + `tool_name` `web.crawl`, `wake_after` true, and a short `wake_prompt`; tell the user the crawl + was submitted and that the result will arrive in a notification banner. Never request more than + 900 seconds. 10. A web crawl goal must describe the requested result. Never use it for DDoS, flooding, load/stress testing, port scanning, brute force, or other high-volume behavior. Keep the crawl same-origin and rely on the tool's page, depth, byte, rate, and timeout limits. diff --git a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md b/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md index 1c7b9c8..10b3640 100644 --- a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md +++ b/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md @@ -25,7 +25,7 @@ Checks (if any fail return failure JSON immediately): markup and validate generated links as http or https. Do not reject ordinary safe HTML in the `html` field solely because the host performs the final sanitization. -Do not deny for Python style, envelope construction, destination URLs, or the absence of dependencies. The static Python verifier enforces direct network and process restrictions. The guest has no network; the host performs HTTP and applies SSRF there. +Do not deny for Go style, envelope construction, destination URLs, or the absence of dependencies. The static Go verifier enforces direct network and process restrictions. The guest has no network; the host performs HTTP and applies SSRF there. Return only valid JSON with this exact schema: { diff --git a/ui/SharedAgentRuntime/Resources/web_crawler_skill.md b/ui/SharedAgentRuntime/Resources/web_crawler_skill.md index f9d7be7..4dc1eb6 100644 --- a/ui/SharedAgentRuntime/Resources/web_crawler_skill.md +++ b/ui/SharedAgentRuntime/Resources/web_crawler_skill.md @@ -3,8 +3,10 @@ Use the `web.crawl` MCP tool for website crawling. Do not generate a crawler script with `script_exec`. -Submit every crawl through `jobs_create` so the user receives the result in a -notification banner. +Call `web.crawl` directly in live chat for typical crawls. Submit through +`jobs_create` only when the crawl is likely to take more than about one minute +(large page budget, deep site, or long timeout). Background crawls should use +`wake_after: true` so the user gets a notification banner when they finish. Required `web.crawl` arguments: diff --git a/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift b/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift index 4b60200..fd909ce 100644 --- a/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift +++ b/ui/SharedAgentRuntime/Services/DaemonProcessHygiene.swift @@ -22,7 +22,7 @@ public enum DaemonProcessHygiene { reportedFingerprint: health.executableFingerprint, expectedFingerprint: expectedFingerprint(), reportedGuestRuntime: health.guestRuntimeImage, - expectedGuestRuntime: DerrickGuestRuntime.pythonGuestDockerImage + expectedGuestRuntime: DerrickGuestRuntime.guestDockerImage ) } diff --git a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift index 543dcda..5be1566 100644 --- a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift +++ b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift @@ -240,6 +240,9 @@ final class AppBootstrapStatus: ObservableObject { /// Maps prewarm / Docker errors into a short title and user-facing explanation. static func classifyError(_ error: Error) -> ClassifiedFailure { + if let goError = error as? DerrickGoToolchainError { + return classifyGoToolchainError(goError) + } if let agentError = error as? JobServiceLoginAgent.AgentError { return classifyDaemonAgentError(agentError) } @@ -338,6 +341,40 @@ final class AppBootstrapStatus: ObservableObject { ) } + private static func classifyGoToolchainError( + _ error: DerrickGoToolchainError + ) -> ClassifiedFailure { + switch error { + case .missing: + return ClassifiedFailure( + title: "Go Toolchain Required", + message: """ + Derrick could not find a Go toolchain for development diagnostics. Guest compile runs in Docker; this error is unexpected. + + Quit and reopen Derrick. If it persists, reinstall Docker Desktop. + """ + ) + case .unparseable: + return ClassifiedFailure( + title: "Go Toolchain Unreadable", + message: """ + Derrick could not read the installed Go version. Guest compile runs in Docker; this error is unexpected. + + \(error.localizedDescription) + """ + ) + case .tooOld(_, let required): + return ClassifiedFailure( + title: "Go Toolchain Too Old", + message: """ + Derrick found an older Go toolchain than \(required). Guest compile runs in Docker; this error is unexpected. + + Quit and reopen Derrick. + """ + ) + } + } + private static func classifyDaemonAgentError( _ error: JobServiceLoginAgent.AgentError ) -> ClassifiedFailure { diff --git a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift index f667eef..6c5f93e 100644 --- a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift +++ b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift @@ -280,18 +280,11 @@ public final class XPCDockerRunner: @unchecked Sendable { ] ) } - await reportBootstrap(phase: .preparingImage, message: "Preparing guest runtime…") + await reportBootstrap(phase: .preparingImage, message: "Preparing worker image…") let executor = makeDockerExecutor() - let image = DerrickGuestRuntime.pythonGuestDockerImage - let inspect = try await executor(["image", "inspect", image], Data(), 30) - if inspect.exitCode != 0 { - await MainActor.run { - AppBootstrapStatus.shared.revealModalIfStillInitializing() - } - try await OneshotDockerContainer.ensurePulledImage(image, executor: executor) - } + try await WorkerImageGate.shared.ensureReady(executor: executor) prewarmState.markCompleted() - await reportBootstrap(phase: .verifyingEnvironment, message: "Guest runtime ready.") + await reportBootstrap(phase: .verifyingEnvironment, message: "Worker image ready.") } catch { debugLog("Guest runtime prewarming failed: \(error.localizedDescription)") prewarmState.markFailed(error) diff --git a/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift b/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift deleted file mode 100644 index 03c3ef1..0000000 --- a/ui/SharedAgentRuntime/Support/Egress/EgressAllowlistService.swift +++ /dev/null @@ -1,273 +0,0 @@ -import Foundation -import Combine -import DBRepository -import EgressProxy -import AppEvents -import PolicyUserInteraction -import Structure - -/// App-owned egress allowlist: DB persistence + host HTTP preflight prompts. -/// Not exposed as MCP. Not part of tool/content policy rules. -@MainActor -final class EgressAllowlistService: ObservableObject { - static let shared = EgressAllowlistService() - - @Published private(set) var suffixes: [EgressAllowedDomainSuffix] = [] - - private var repository: DBRepository? - private let username = "ui" - private let password = "ui" - private let localPolicy = DefaultDestinationPolicy(allowedDomainSuffixes: []) - - private init() {} - - func configure(repository: DBRepository) async { - self.repository = repository - do { - let inserted = try await repository.seedEgressAllowedDomainSuffixesIfNeeded( - EgressProxyConfiguration.defaultSeedDomainSuffixes, - source: "seed" - ) - if inserted > 0 { - debugLog("Egress allowlist seed inserted \(inserted) suffix(es).") - } else { - debugLog("Egress allowlist seed skipped (suffixes already present).") - } - try await reload() - } catch { - debugLog("Egress allowlist configure failed: \(error.localizedDescription)") - } - } - - func reload(clearSessionHosts: Bool = false) async throws { - guard let repository else { - suffixes = [] - return - } - let rows = try await repository.loadEgressAllowedDomainSuffixes(includeDisabled: true) - suffixes = rows - localPolicy.setAllowedDomainSuffixes(rows.filter(\.enabled).map(\.suffix)) - if clearSessionHosts { - // Settings edits must not be shadowed by prior Allow-once / mid-flight grants. - localPolicy.clearSessionHosts() - } - } - - private var isAgentServiceProcess: Bool { - let bid = Bundle.main.bundleIdentifier ?? "" - return bid == DerrickServiceID.agent.rawValue || bid.hasSuffix(".AgentService") - } - - func addSuffix(_ raw: String, source: String = "user") async throws { - let suffix = EgressHostExtractor.permanentSuffix(for: raw) - guard EgressHostExtractor.isPlausibleHostname(suffix) || suffix.contains(".") else { - throw NSError( - domain: "EgressAllowlist", - code: 1, - userInfo: [NSLocalizedDescriptionKey: "Invalid domain suffix: \(raw)"] - ) - } - guard let repository else { return } - try await repository.saveEgressAllowedDomainSuffix( - EgressAllowedDomainSuffix(suffix: suffix, source: source, enabled: true) - ) - try await reload() - } - - func removeSuffix(id: String) async throws { - guard let repository else { return } - try await repository.deleteEgressAllowedDomainSuffix(id: id) - try await reload(clearSessionHosts: true) - debugLog("Egress allowlist removed id=\(id)") - } - - /// Preflight network hosts before script execution. - /// - Returns: nil if allowed to proceed; blocked tool-result JSON if user denied or hard-blocked. - func preflightScriptNetwork( - script: String, - allowNetwork: Bool, - toolName: String = "script_exec" - ) async -> String? { - guard allowNetwork else { return nil } - - let preflightStarted = Date() - let hosts = EgressHostExtractor.extractHosts(from: script) - guard !hosts.isEmpty else { - PipelineTiming.log("egress_preflight hosts=0 total_ms=0 modal_ms=0") - return nil - } - - var sessionGrants: [String] = [] - var modalMS = 0 - var needsPrompt: [String] = [] - - for host in hosts { - if localPolicy.isHardBlockedHostname(host) { - let message = "Network access to “\(host)” is permanently blocked (private/metadata host)." - PipelineTiming.log( - "egress_preflight blocked_hard host=\(host) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - // Modal is published by the pipeline from the common network outcome. - return Self.blockedResultJSON(findings: [message]) - } - - if localPolicy.isHostCoveredByAllowlist(host) { - continue - } - needsPrompt.append(host) - } - - if !needsPrompt.isEmpty { - let modalStarted = Date() - let decision = await promptForHosts(needsPrompt, toolName: toolName) - modalMS += PipelineTiming.elapsedMS(from: modalStarted) - switch decision { - case .approved(let actor), .approvedOnce(let actor): - debugLog("Egress allow once for \(needsPrompt.count) host(s) by \(actor ?? "user")") - sessionGrants.append(contentsOf: needsPrompt) - localPolicy.grantSessionHosts(needsPrompt) - case .approvedPermanently(let actor): - debugLog("Egress allow always for \(needsPrompt.count) host(s) by \(actor ?? "user")") - for host in needsPrompt { - let suffix = EgressHostExtractor.permanentSuffix(for: host) - do { - try await addSuffix(suffix, source: "user") - sessionGrants.append(host) - localPolicy.grantSessionHosts([host]) - } catch { - debugLog("Failed to persist egress suffix \(suffix): \(error.localizedDescription)") - PipelineTiming.log( - "egress_preflight persist_failed host=\(host) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return Self.blockedResultJSON( - findings: ["Failed to save permanent allow for \(host): \(error.localizedDescription)"] - ) - } - } - case .denied(let actor): - debugLog("Egress deny for \(needsPrompt.count) host(s) by \(actor ?? "user") — aborting entire script run") - let listed = needsPrompt.joined(separator: ", ") - let message = "User denied network access to “\(listed)”. The script was not run." - PipelineTiming.log( - "egress_preflight user_denied hosts=\(needsPrompt.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS) prompted_hosts=\(needsPrompt.count)" - ) - return Self.blockedResultJSON(findings: [message]) - case .dismissed, .timedOut: - let listed = needsPrompt.joined(separator: ", ") - let message = "Network access to “\(listed)” was not approved. The script was not run." - PipelineTiming.log( - "egress_preflight dismissed_or_timeout hosts=\(needsPrompt.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return Self.blockedResultJSON(findings: [message]) - } - } - - PipelineTiming.log( - "egress_preflight ok hosts=\(hosts.count) prompted=\(needsPrompt.count) session_grants=\(sessionGrants.count) total_ms=\(PipelineTiming.elapsedMS(from: preflightStarted)) modal_ms=\(modalMS)" - ) - return nil - } - - private func promptForHosts(_ hosts: [String], toolName: String) async -> PolicyUserDecision { - let unique = Self.uniqueHosts(hosts) - guard !unique.isEmpty else { - return .denied(actor: "system") - } - - if let remote = TurnProcessContext.effectiveNetworkAccessPrompt { - var last: PolicyUserDecision = .denied(actor: "system") - for host in unique { - debugLog("Egress prompt via AgentService path host=\(host)") - last = await remote(host, toolName) - switch last { - case .denied, .dismissed, .timedOut: - return last - case .approved, .approvedOnce, .approvedPermanently: - continue - } - } - return last - } - - if !isAgentServiceProcess { - debugLog("Egress prompt via UI modal hosts=\(unique.count)") - let event = PolicyUserEventFactory.egressAccessRequest( - hosts: unique, - toolName: toolName - ) - return await AppEventBus.shared.initDecision(event) - } - - guard let repository else { - return .denied(actor: "system-no-repository") - } - var last: PolicyUserDecision = .denied(actor: "system") - for host in unique { - debugLog("Egress prompt via notification path host=\(host)") - last = await HITLOfflineNetworkService.awaitDecision( - host: host, - toolName: toolName, - turnID: "egress-agent", - isJobContext: false, - repository: repository, - timeoutNanoseconds: 300_000_000_000 - ) - switch last { - case .denied, .dismissed, .timedOut: - return last - case .approved, .approvedOnce, .approvedPermanently: - continue - } - } - return last - } - - private static func uniqueHosts(_ hosts: [String]) -> [String] { - var seen = Set() - var ordered: [String] = [] - for raw in hosts { - let host = raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard !host.isEmpty, seen.insert(host).inserted else { continue } - ordered.append(host) - } - return ordered - } - - /// Apply a user egress decision in this process. - /// Call when the UI answers a network prompt so later requests do not re-prompt. - func applyUserNetworkDecision(host: String, decision: PolicyUserDecision) async { - let normalized = host.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - guard !normalized.isEmpty else { return } - - switch decision { - case .approved(let actor), .approvedOnce(let actor): - debugLog("Egress UI apply once host=\(normalized) actor=\(actor ?? "?")") - localPolicy.grantSessionHosts([normalized]) - case .approvedPermanently(let actor): - debugLog("Egress UI apply always host=\(normalized) actor=\(actor ?? "?")") - let suffix = EgressHostExtractor.permanentSuffix(for: normalized) - do { - try await addSuffix(suffix, source: "user") - } catch { - debugLog("Egress UI apply always persist failed: \(error.localizedDescription)") - } - localPolicy.grantSessionHosts([normalized]) - case .denied, .dismissed, .timedOut: - break - } - } - - private static func blockedResultJSON(findings: [String]) -> String { - let diagnostics = findings.isEmpty - ? [ToolExecutionOutcome.Diagnostic(code: "egress_denied", message: "Network access was denied.")] - : findings.map { - ToolExecutionOutcome.Diagnostic(code: "egress_denied", message: $0) - } - return (try? ToolExecutionOutcome.failure( - status: .blocked, - stage: .network, - diagnostics: diagnostics, - retry: ToolExecutionOutcome.Retry(allowed: false) - ).encodedJSON()) ?? #"{"status":"blocked","stage":"network","diagnostics":[]}"# - } -} diff --git a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift index 30ef495..e487ee7 100644 --- a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift +++ b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift @@ -149,7 +149,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), python_source (string), + plugin_id (string), version (string), description (string), go_source (string), test_input_json (string containing valid JSON — a serialized object, not prose), skill_files (array of objects with path and body), secrets (array of objects with id, label, and kind; required for connector plugins), @@ -159,7 +159,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { If the plugin needs a username, password, token, or API key, declare them in secrets. kind must be username, password, token, or api_key. id is a stable Keychain key such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in python_source. + Never put real credentials in go_source. Set role to "connector" when the plugin sends and receives messages with an external messaging service (any chat or mail connector). Omit role or use "standard" otherwise. For role connector, include messaging_ops: an array of implemented ops @@ -168,13 +168,13 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { The host lists connector plugins under Messaging. Do not guess this from the plugin_id. Do not return manifest_json. The host creates the canonical Agent Plugin manifest, including the exact `$schema` field for Agent Plugin 1.0 and the fixed - extensions.app.derrick.entrypoint ./app.derrick/plugin.py. - \(DerrickGuestPython.modelContract) + extensions.app.derrick.entrypoint ./app.derrick/plugin.go. + \(DerrickGuestGo.modelContract) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) Before returning the draft, self-check the implementation: - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - Match host responses by the emitted request_id. - - Use only the Python standard library (no pip, requests, urllib, socket, or subprocess). + - Use only the Go standard library (no third-party modules, raw sockets, or subprocess). - The direct test input must exercise the terminal result path with matching http_results fixtures. If skill_files is not needed, return an empty array. Every skill file path must be exactly skills//SKILL.md. @@ -189,7 +189,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ "params":{...}}]} Repeat additional hop pairs for each messaging_op in scope. request_id values in fixtures must \ - match the http.request envelopes your python_source emits. + match the http.request envelopes your go_source emits. - Match http_results by request_id and de-duplicate with stable sorting; never depend on response order. When vendor documentation is supplied in the user prompt, use it only to fill may_call HTTP details. """ @@ -201,7 +201,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { "plugin_id": AgentSchema(type: .string), "version": AgentSchema(type: .string), "description": AgentSchema(type: .string), - "python_source": AgentSchema(type: .string), + "go_source": AgentSchema(type: .string), "test_input_json": AgentSchema(type: .string), "skill_files": AgentSchema( type: .array, @@ -233,7 +233,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { ), ], required: [ - "plugin_id", "version", "description", "python_source", + "plugin_id", "version", "description", "go_source", "test_input_json", "skill_files", ] ) @@ -248,7 +248,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { """ The host already assigned plugin_id \(host.pluginID) and these secret ids: \ \(host.secrets.map(\.id).joined(separator: ", ")). \ - Return python_source and test_input_json. Do not pick a different plugin_id or secret ids. \ + Return go_source and test_input_json. Do not pick a different plugin_id or secret ids. \ Use {{secret:\(host.secrets.first?.id ?? "bot_token")}} in HTTP headers. """ ) @@ -395,7 +395,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ You are Derrick's independent plugin alignment and safety reviewer. - Review the user's goal, manifest, test_input_json, exact Python source, and direct test output. + Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. Return exactly one JSON object: {"decision":"approved|rejected","summary":"...","findings":[ {"severity":"info|warning|blocking","category":"alignment|safety|correctness|privacy|supplyChain","message":"..."} @@ -409,7 +409,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { - For connector plugins, obey the connector protocol JSON below. If a rule is not in that JSON, do not require it. \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) Compilation success is not approval. Do not rewrite the code or approve a draft that fails these checks. - Reject Swift source, socket/urllib/requests usage, or missing stdin reads. + Reject non-Go source, raw network usage outside http.request envelopes, or missing stdin reads. """ } @@ -449,7 +449,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { test_input_json: \(String(decoding: draft.testInput, as: UTF8.self)) - Python source: + Go source: \(draft.guestSource) Direct test output: diff --git a/ui/SharedAgentRuntime/TurnProcessContext.swift b/ui/SharedAgentRuntime/TurnProcessContext.swift index ab0f253..5e5759c 100644 --- a/ui/SharedAgentRuntime/TurnProcessContext.swift +++ b/ui/SharedAgentRuntime/TurnProcessContext.swift @@ -31,7 +31,7 @@ public enum TurnProcessContext { /// Active agent profile handle for the current user-facing turn (orchestrator-only tools). @TaskLocal public static var activeProfileHandle: String? - /// Active `/create-plugin` or `/edit-plugin` factory turn (enables sync `web.crawl`). + /// Active `/create-plugin` or `/edit-plugin` factory turn. @TaskLocal public static var pluginFactoryCreationActive: Bool = false public static func install( diff --git a/ui/ui/Jobs/DerrickNotificationService.swift b/ui/ui/Jobs/DerrickNotificationService.swift index 74b1c9a..e0c1d92 100644 --- a/ui/ui/Jobs/DerrickNotificationService.swift +++ b/ui/ui/Jobs/DerrickNotificationService.swift @@ -170,15 +170,6 @@ final class DerrickNotificationService { fputs("[HumanDecision] resolve skip id=\(approvalID)\n", stderr) return } - if approved, HITLOfflineNetworkService.isNetworkToolName(row.toolName), - let host = HITLOfflineNetworkService.host(fromNetworkToolName: row.toolName) { - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: host, - decision: always - ? .approvedPermanently(actor: actor) - : .approvedOnce(actor: actor) - ) - } let status: PendingHITLApprovalStatus = approved ? .approved : .cancelled let edited = approved ? row.argumentsJSON : nil try? await repository.resolveHITLApproval( @@ -264,9 +255,6 @@ final class DerrickNotificationService { databaseDirectoryURL: directory ) repository = repo - if !JobResultPanelSession.isPanelOnlyLaunch { - await EgressAllowlistService.shared.configure(repository: repo) - } return repo } catch { fputs("[HumanDecision] ensureRepository failed: \(error.localizedDescription)\n", stderr) diff --git a/ui/ui/Jobs/HITLLiveApprovalHandlers.swift b/ui/ui/Jobs/HITLLiveApprovalHandlers.swift index cd52c2b..8f70c35 100644 --- a/ui/ui/Jobs/HITLLiveApprovalHandlers.swift +++ b/ui/ui/Jobs/HITLLiveApprovalHandlers.swift @@ -166,51 +166,11 @@ enum HITLLiveApprovalHandlers { } private static func presentNetworkAccess(_ request: AgentNetworkAccessRequestDTO) async -> AgentNetworkAccessDecisionDTO { - let event = PolicyUserEventFactory.egressAccessRequest( - host: request.host, - toolName: request.toolName, - correlationId: request.requestID + // Blacklist-only egress: public hosts are allowed by default. Blacklist hits use PolicyEventBus. + AgentNetworkAccessDecisionDTO( + requestID: request.requestID, + decision: "once", + actor: "blacklist-only-auto" ) - let decision = await AppEventBus.shared.initDecision(event) - switch decision { - case .approvedOnce(let actor): - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: request.host, - decision: .approvedOnce(actor: actor) - ) - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "once", - actor: actor ?? "ui-modal-once" - ) - case .approvedPermanently(let actor): - await EgressAllowlistService.shared.applyUserNetworkDecision( - host: request.host, - decision: .approvedPermanently(actor: actor) - ) - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "always", - actor: actor ?? "ui-modal-always" - ) - case .timedOut: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "timeout", - actor: "ui-modal-timeout" - ) - case .dismissed: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "dismissed", - actor: "ui-modal-dismissed" - ) - case .approved, .denied: - return AgentNetworkAccessDecisionDTO( - requestID: request.requestID, - decision: "deny", - actor: "ui-modal-deny" - ) - } } } diff --git a/ui/ui/Views/ContentView.swift b/ui/ui/Views/ContentView.swift index 34845a8..617a6bd 100644 --- a/ui/ui/Views/ContentView.swift +++ b/ui/ui/Views/ContentView.swift @@ -838,7 +838,6 @@ struct ContentView: View { @MainActor private func configureClientRepositoryServices(repository: DBRepository) async { await ServiceLogRecorder.shared.configure(repository: repository) - await EgressAllowlistService.shared.configure(repository: repository) await ContentSensitivityGrantService.shared.configure(repository: repository) await UsageLimitsService.shared.configure(repository: repository) await ContainerLifecycleSettingsService.shared.configure(repository: repository) diff --git a/ui/ui/Views/LLMModelSettingsView.swift b/ui/ui/Views/LLMModelSettingsView.swift index 2321a60..d3209fc 100644 --- a/ui/ui/Views/LLMModelSettingsView.swift +++ b/ui/ui/Views/LLMModelSettingsView.swift @@ -349,7 +349,7 @@ struct LLMModelSettingsView: View { Text("Multi-agent") .font(.system(size: 26, weight: .semibold, design: .rounded)) - Text("Caps for agents_spawn and worker turns in a chat session. New tabs use saved values; open tabs keep the limits they started with. Parallel script_exec runs wait in line (one Python guest container at a time). Crawls and file conversions have their own lines.") + Text("Caps for agents_spawn and worker turns in a chat session. New tabs use saved values; open tabs keep the limits they started with. Parallel script_exec runs wait in line (one Go guest container at a time). Crawls and file conversions have their own lines.") .font(.subheadline) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) diff --git a/ui/uiTests/MessagingNavigationTests.swift b/ui/uiTests/MessagingNavigationTests.swift index 426c103..88476ae 100644 --- a/ui/uiTests/MessagingNavigationTests.swift +++ b/ui/uiTests/MessagingNavigationTests.swift @@ -343,9 +343,9 @@ import Testing _ = try await repository.createEmptyDatabaseIfNeeded(username: "ui", password: "ui") let manifestJSON = """ {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-bot","version":"1.0.0",\ - "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.py","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector","messaging_ops":["sync_threads","poll_inbox","send_message"]}}} """ - let runtimeJSON = #"{"language":"python","entrypoint":"./app.derrick/plugin.py"}"# + let runtimeJSON = #"{"language":"go","entrypoint":"./app.derrick/plugin.go"}"# let guestSource = "print([])" var release = PluginFactoryRelease( pluginID: "slack-bot", diff --git a/ui/uiTests/PromptResourcesTests.swift b/ui/uiTests/PromptResourcesTests.swift index 66950af..c63e47a 100644 --- a/ui/uiTests/PromptResourcesTests.swift +++ b/ui/uiTests/PromptResourcesTests.swift @@ -74,10 +74,10 @@ import Testing @Test func loadsGuestSDKFromResourcesDirectory() throws { let source = try PromptResources.guestSDKSource() - #expect(source.contains("standalone Python script")) + #expect(source.contains("package main")) let wrapped = try PromptResources.guestSDKForModel() - #expect(wrapped.contains("```python")) - #expect(wrapped.contains("Python guest contract")) + #expect(wrapped.contains("```go")) + #expect(wrapped.contains("Go guest contract")) } @Test func throwsWhenConversationRAGInstructionsAreMissing() throws { diff --git a/workers/go/cmd/derrick-crawler/main.go b/workers/go/cmd/derrick-crawler/main.go new file mode 100644 index 0000000..4ea3160 --- /dev/null +++ b/workers/go/cmd/derrick-crawler/main.go @@ -0,0 +1,103 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "strconv" + + "github.com/jsoneaday/derrick/workers/internal/contract" + "github.com/jsoneaday/derrick/workers/internal/crawler" +) + +func main() { + input, err := io.ReadAll(os.Stdin) + if err != nil { + writeResult(blockedResult("", "Crawler input must be a valid JSON object.")) + return + } + + var req crawler.Request + if err := json.Unmarshal(input, &req); err != nil { + writeResult(blockedResult("", "Crawler input must be a valid JSON object.")) + return + } + + validated, err := crawler.Validate(req) + if err != nil { + writeResult(blockedResult(req.StartURL, err.Error())) + return + } + + proxy := readProxy() + ctx := context.Background() + result := crawler.Run(ctx, validated, proxy) + if len(result.Pages) == 0 && result.StopReason == crawler.StopCompleted { + result.OK = false + } + if len(result.Pages) > 0 { + result.OK = true + } + writeResult(result) +} + +func readProxy() *crawler.ProxyConfig { + host := os.Getenv("DERRICK_EGRESS_PROXY_HOST") + portStr := os.Getenv("DERRICK_EGRESS_PROXY_PORT") + token := os.Getenv("DERRICK_EGRESS_PROXY_TOKEN") + if host == "" && portStr == "" && token == "" { + return nil + } + port, _ := strconv.Atoi(portStr) + return &crawler.ProxyConfig{Host: host, Port: port, Token: token} +} + +func blockedResult(startURL string, message string) crawler.Result { + return crawler.Result{ + OK: false, + StartURL: startURL, + Pages: []crawler.Page{}, + StopReason: crawler.StopBlocked, + Diagnostics: []string{message}, + } +} + +func writeResult(result crawler.Result) { + payload, err := encodedCrawlerResult(result) + if err != nil { + writeEncodedResult(blockedResult(result.StartURL, "Crawler output violated worker contract.")) + return + } + _, _ = os.Stdout.Write(payload) +} + +func encodedCrawlerResult(result crawler.Result) ([]byte, error) { + if result.Pages == nil { + result.Pages = []crawler.Page{} + } + if result.Diagnostics == nil { + result.Diagnostics = []string{} + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(result); err != nil { + return nil, err + } + data := bytes.TrimSpace(buf.Bytes()) + if err := contract.ValidateWebCrawlerResultJSON(data); err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +func writeEncodedResult(result crawler.Result) { + payload, err := encodedCrawlerResult(result) + if err != nil { + fallback := blockedResult("", "Crawler failed to produce schema-compliant output.") + payload, _ = encodedCrawlerResult(fallback) + } + _, _ = os.Stdout.Write(payload) +} diff --git a/workers/go/cmd/derrick-file-extractor/main.go b/workers/go/cmd/derrick-file-extractor/main.go new file mode 100644 index 0000000..8a93b09 --- /dev/null +++ b/workers/go/cmd/derrick-file-extractor/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + + "github.com/jsoneaday/derrick/workers/internal/contract" + "github.com/jsoneaday/derrick/workers/internal/extractor" +) + +func main() { + input, err := io.ReadAll(os.Stdin) + if err != nil { + writeResult(extractor.Result{ + OK: false, + Operation: extractor.OperationExtract, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor input must be a valid JSON object."}, + }, 1) + return + } + + var req extractor.Request + if err := json.Unmarshal(input, &req); err != nil { + writeResult(extractor.Result{ + OK: false, + Operation: extractor.OperationExtract, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor input must be a valid JSON object."}, + }, 1) + return + } + + result := extractor.Run(req, extractor.InputDirectory, extractor.OutputDirectory) + code := 0 + if !result.OK { + code = 1 + } + writeResult(result, code) +} + +func writeResult(result extractor.Result, code int) { + payload, err := encodedExtractorResult(result) + if err != nil { + payload, _ = encodedExtractorResult(extractor.Result{ + OK: false, + Operation: result.Operation, + Files: []extractor.FileResult{}, + Diagnostics: []string{"File extractor output violated worker contract."}, + }) + code = 1 + } + _, _ = os.Stdout.Write(payload) + os.Exit(code) +} + +func encodedExtractorResult(result extractor.Result) ([]byte, error) { + if result.Files == nil { + result.Files = []extractor.FileResult{} + } + if result.Diagnostics == nil { + result.Diagnostics = []string{} + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(result); err != nil { + return nil, err + } + data := bytes.TrimSpace(buf.Bytes()) + if err := contract.ValidateFileExtractorResultJSON(data); err != nil { + return nil, err + } + return append(data, '\n'), nil +} diff --git a/workers/go/go.mod b/workers/go/go.mod new file mode 100644 index 0000000..df29702 --- /dev/null +++ b/workers/go/go.mod @@ -0,0 +1,21 @@ +module github.com/jsoneaday/derrick/workers + +go 1.27.1 + +require ( + github.com/PuerkitoBio/goquery v1.10.3 + github.com/xuri/excelize/v2 v2.9.1 +) + +require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect + github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.1 // indirect + golang.org/x/crypto v0.38.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/text v0.25.0 // indirect +) diff --git a/workers/go/go.sum b/workers/go/go.sum new file mode 100644 index 0000000..80c0084 --- /dev/null +++ b/workers/go/go.sum @@ -0,0 +1,100 @@ +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= +github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= +github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= +github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= +github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= +golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/workers/go/internal/contract/contract.go b/workers/go/internal/contract/contract.go new file mode 100644 index 0000000..ba341c0 --- /dev/null +++ b/workers/go/internal/contract/contract.go @@ -0,0 +1,120 @@ +package contract + +import ( + "bytes" + "embed" + "encoding/json" + "fmt" + "io/fs" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +//go:embed schemas/*.json +var schemaFS embed.FS + +// ValidateHopEventJSON checks stdin against hop-event.schema.json. +func ValidateHopEventJSON(data []byte) error { + return validate("hop-event.schema.json", data) +} + +// ValidateEnvelopeListJSON checks stdout against envelope-list.schema.json. +func ValidateEnvelopeListJSON(data []byte) error { + return validate("envelope-list.schema.json", data) +} + +// ValidateWebCrawlerResultJSON checks crawler stdout against web-crawler-result.schema.json. +func ValidateWebCrawlerResultJSON(data []byte) error { + return validate("web-crawler-result.schema.json", data) +} + +// ValidateFileExtractorResultJSON checks extractor stdout against file-extractor-result.schema.json. +func ValidateFileExtractorResultJSON(data []byte) error { + return validate("file-extractor-result.schema.json", data) +} + +func validate(schemaName string, data []byte) error { + compiler := jsonschema.NewCompiler() + if err := loadSchemas(compiler); err != nil { + return err + } + schema, err := compiler.Compile(schemaCompileURL(schemaName)) + if err != nil { + return fmt.Errorf("compile schema %s: %w", schemaName, err) + } + var value any + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&value); err != nil { + return fmt.Errorf("invalid json: %w", err) + } + if err := schema.Validate(value); err != nil { + return fmt.Errorf("schema validation failed: %w", err) + } + return nil +} + +const schemaBase = "https://derrick.local/schemas/" + +func loadSchemas(compiler *jsonschema.Compiler) error { + return fs.WalkDir(schemaFS, "schemas", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + raw, err := schemaFS.ReadFile(path) + if err != nil { + return err + } + var doc any + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + name := strings.TrimPrefix(path, "schemas/") + ids := schemaResourceIDs(name, doc) + for _, id := range ids { + if err := compiler.AddResource(id, doc); err != nil { + return err + } + } + return nil + }) +} + +func schemaResourceIDs(filename string, doc any) []string { + seen := map[string]bool{} + add := func(id string) { + if id == "" || seen[id] { + return + } + seen[id] = true + } + add(schemaBase + filename) + if m, ok := doc.(map[string]any); ok { + if id, ok := m["$id"].(string); ok { + add(id) + } + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + return ids +} + +func schemaCompileURL(name string) string { + raw, err := schemaFS.ReadFile("schemas/" + name) + if err != nil { + return schemaBase + name + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err == nil { + if id, ok := doc["$id"].(string); ok && id != "" { + return id + } + } + return schemaBase + name +} diff --git a/workers/go/internal/contract/contract_test.go b/workers/go/internal/contract/contract_test.go new file mode 100644 index 0000000..bd5d229 --- /dev/null +++ b/workers/go/internal/contract/contract_test.go @@ -0,0 +1,49 @@ +package contract + +import "testing" + +func TestValidateHopEventAndEnvelopeList(t *testing.T) { + hop := []byte(`{"kind":"manual"}`) + if err := ValidateHopEventJSON(hop); err != nil { + t.Fatalf("hop event: %v", err) + } + env := []byte(`[{"verb":"result.emit","summary":"ok"}]`) + if err := ValidateEnvelopeListJSON(env); err != nil { + t.Fatalf("envelope list: %v", err) + } +} + +func TestValidateEnvelopeListRejectsNestedAlias(t *testing.T) { + env := []byte(`[{"verb":"result.emit","result":{"emit":{"content":"x"}}}]`) + if err := ValidateEnvelopeListJSON(env); err == nil { + t.Fatal("expected rejection") + } +} + +func TestValidateWebCrawlerResultAcceptsEmptyDiagnosticsArray(t *testing.T) { + payload := []byte(`{"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":[]}`) + if err := ValidateWebCrawlerResultJSON(payload); err != nil { + t.Fatalf("web crawler result: %v", err) + } +} + +func TestValidateWebCrawlerResultRejectsNullDiagnostics(t *testing.T) { + payload := []byte(`{"ok":true,"start_url":"https://example.com/","pages":[],"stop_reason":"completed","requests_made":0,"bytes_read":0,"truncated":false,"diagnostics":null}`) + if err := ValidateWebCrawlerResultJSON(payload); err == nil { + t.Fatal("expected rejection for null diagnostics") + } +} + +func TestValidateFileExtractorResultAcceptsEmptyFilesArray(t *testing.T) { + payload := []byte(`{"ok":true,"operation":"extract","files":[],"diagnostics":[]}`) + if err := ValidateFileExtractorResultJSON(payload); err != nil { + t.Fatalf("file extractor result: %v", err) + } +} + +func TestValidateFileExtractorResultRejectsNullFiles(t *testing.T) { + payload := []byte(`{"ok":false,"operation":"extract","files":null,"diagnostics":[]}`) + if err := ValidateFileExtractorResultJSON(payload); err == nil { + t.Fatal("expected rejection for null files") + } +} diff --git a/workers/go/internal/contract/schemas/connector-contract.schema.json b/workers/go/internal/contract/schemas/connector-contract.schema.json new file mode 100644 index 0000000..5ca6829 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-contract.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-contract.json", + "title": "Derrick connector protocol document", + "description": "Canonical host ops, emit shapes, allowed vendor-call ids, and success rules. Vendor HTTP bindings live in contracts/vendors/*.json.", + "type": "object", + "required": ["version", "ops", "scopes", "rules"], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "ops": { + "type": "object", + "required": ["sync_threads", "poll_inbox", "send_message"], + "additionalProperties": { "$ref": "#/$defs/op" } + }, + "scopes": { + "type": "object", + "required": ["full_sync"], + "additionalProperties": { "$ref": "#/$defs/scope" } + }, + "rules": { + "type": "object", + "required": [ + "sync_threads_lists_tabs_only", + "poll_loads_one_conversation", + "paginate_within_op", + "live_http_results_accumulate", + "direct_test_poll_requires_non_empty_messages", + "direct_test_threads_requires_non_empty", + "runtime_empty_messages_ok_if_vendor_ok" + ], + "properties": { + "sync_threads_lists_tabs_only": { "type": "boolean" }, + "poll_loads_one_conversation": { "type": "boolean" }, + "paginate_within_op": { "type": "boolean" }, + "live_http_results_accumulate": { "type": "boolean" }, + "direct_test_poll_requires_non_empty_messages": { "type": "boolean" }, + "direct_test_threads_requires_non_empty": { "type": "boolean" }, + "runtime_empty_messages_ok_if_vendor_ok": { "type": "boolean" } + }, + "additionalProperties": false + } + }, + "$defs": { + "op": { + "type": "object", + "required": ["kind", "emit"], + "properties": { + "kind": { "type": "string", "enum": ["manual", "message_in_room"] }, + "emit": { "type": "string", "enum": ["threads", "messages", "sent_message"] }, + "params": { "type": "array", "items": { "type": "string" } }, + "may_call": { "type": "array", "items": { "type": "string" } }, + "must_not_call": { "type": "array", "items": { "type": "string" } }, + "when_no_parent": { "$ref": "#/$defs/call_set" }, + "when_parent": { "$ref": "#/$defs/call_set" }, + "success": { "$ref": "#/$defs/success" }, + "failure": { "$ref": "#/$defs/failure" } + }, + "additionalProperties": false + }, + "call_set": { + "type": "object", + "properties": { + "may_call": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + }, + "success": { + "type": "object", + "properties": { + "empty_collection_ok": { "type": "boolean" }, + "empty_array_ok_if_vendor_ok": { "type": "boolean" }, + "requires_sent_message": { "type": "boolean" } + }, + "additionalProperties": false + }, + "failure": { + "type": "object", + "properties": { + "vendor_ok_false_empty_is_not_success": { "type": "boolean" }, + "codes": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + }, + "scope": { + "type": "object", + "required": ["ops", "include_reply_poll", "test_pagination"], + "properties": { + "ops": { "type": "array", "items": { "type": "string" } }, + "include_reply_poll": { "type": "boolean" }, + "test_pagination": { "type": "string", "enum": ["single_page", "follow_cursor"] }, + "poll_must_not_call": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + } + } +} diff --git a/workers/go/internal/contract/schemas/connector-params.schema.json b/workers/go/internal/contract/schemas/connector-params.schema.json new file mode 100644 index 0000000..32ba1a5 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-params.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-params.json", + "title": "Connector hop params", + "description": "params object on a connector plugin.invoke hop event.", + "type": "object", + "properties": { + "messaging_op": { + "type": "string", + "enum": ["sync_threads", "poll_inbox", "send_message"] + }, + "vendor_thread_id": { "type": "string" }, + "text": { "type": "string" }, + "since": { "type": "string" }, + "oldest": { "type": "string" }, + "parent_vendor_message_id": { "type": "string" }, + "thread_ts": { "type": "string" } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/connector-result-emit.schema.json b/workers/go/internal/contract/schemas/connector-result-emit.schema.json new file mode 100644 index 0000000..b67fcd1 --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-result-emit.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-result-emit.json", + "title": "Connector result.emit payload", + "description": "Structured fields the host persists from a connector terminal envelope.", + "type": "object", + "additionalProperties": false, + "properties": { + "threads": { + "type": "array", + "items": { + "type": "object", + "required": ["vendor_thread_id", "title"], + "additionalProperties": false, + "properties": { + "vendor_thread_id": { "type": "string" }, + "title": { "type": "string" }, + "is_member": { "type": "boolean" }, + "accessible": { "type": "boolean" } + } + } + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "required": ["vendor_thread_id", "vendor_message_id", "direction", "sender", "body", "created_at"], + "additionalProperties": false, + "properties": { + "vendor_thread_id": { "type": "string" }, + "vendor_message_id": { "type": "string" }, + "direction": { "type": "string", "enum": ["inbound", "outbound"] }, + "sender": { "type": "string" }, + "body": { "type": "string" }, + "created_at": { "type": ["string", "number"] }, + "parent_vendor_message_id": { "type": "string" }, + "thread_ts": { "type": "string" }, + "reply_count": { "type": "integer", "minimum": 0 } + } + } + }, + "sent_message": { + "type": "object", + "required": ["vendor_message_id", "created_at"], + "additionalProperties": false, + "properties": { + "vendor_message_id": { "type": ["string", "number"] }, + "created_at": { "type": ["string", "number"] } + } + }, + "title": { "type": "string" }, + "summary": { "type": "string" } + } +} diff --git a/workers/go/internal/contract/schemas/connector-vendor.schema.json b/workers/go/internal/contract/schemas/connector-vendor.schema.json new file mode 100644 index 0000000..5959d2a --- /dev/null +++ b/workers/go/internal/contract/schemas/connector-vendor.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/connector-vendor.schema.json", + "title": "Connector vendor profile", + "description": "HTTP bindings for one vendor. Call ids must match may_call / must_not_call in connector-contract.json.", + "type": "object", + "required": ["vendor", "vendor_ok_field", "calls"], + "additionalProperties": false, + "properties": { + "vendor": { "type": "string" }, + "vendor_ok_field": { "type": "string" }, + "membership_flag": { "type": "string" }, + "pagination_cursor": { "type": "string" }, + "calls": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "required": ["method", "url"], + "additionalProperties": false, + "properties": { + "method": { "type": "string", "minLength": 1 }, + "url": { "type": "string", "minLength": 1 } + } + } + } + } +} diff --git a/workers/go/internal/contract/schemas/envelope-list.schema.json b/workers/go/internal/contract/schemas/envelope-list.schema.json new file mode 100644 index 0000000..5fbef6d --- /dev/null +++ b/workers/go/internal/contract/schemas/envelope-list.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/envelope-list.json", + "title": "Guest stdout envelope list", + "description": "JSON array written to stdout by an offline guest (plugin or script). Canonical contract for all guest languages.", + "type": "array", + "minItems": 0, + "items": { + "type": "object", + "required": ["verb"], + "additionalProperties": false, + "properties": { + "verb": { + "type": "string", + "enum": [ + "http.request", + "result.emit", + "message.post", + "ui.present", + "secret.request", + "storage.read", + "storage.write", + "job.schedule", + "log" + ] + }, + "request_id": { "type": "string" }, + "method": { "type": "string" }, + "url": { "type": "string" }, + "title": { "type": "string" }, + "summary": { "type": "string" }, + "text": { "type": "string" }, + "content": { "type": "string" }, + "html": { "type": "string" }, + "markdown": { "type": "string" }, + "message": { "type": "string" }, + "schema_version": { "type": "integer" }, + "auth_ref": { "type": ["string", "null"] }, + "json": { + "description": "HTTP request body as a JSON value. HostHTTPRequest.json. The host sends this as the wire body." + }, + "headers": { "type": "object", "additionalProperties": { "type": "string" } }, + "widgets": { "type": "array", "items": { "type": "object" } }, + "secret_ref": { "type": "string" }, + "reason": { "type": "string" }, + "key": { "type": "string" }, + "value": true, + "interval_seconds": { "type": "integer" }, + "timezone": { "type": "string" }, + "threads": { "$ref": "connector-result-emit.schema.json#/properties/threads" }, + "messages": { "$ref": "connector-result-emit.schema.json#/properties/messages" }, + "sent_message": { "$ref": "connector-result-emit.schema.json#/properties/sent_message" } + } + } +} diff --git a/workers/go/internal/contract/schemas/execution-context-wire.schema.json b/workers/go/internal/contract/schemas/execution-context-wire.schema.json new file mode 100644 index 0000000..b1529b6 --- /dev/null +++ b/workers/go/internal/contract/schemas/execution-context-wire.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ExecutionContextWire", + "type": "object", + "required": ["schema_version", "session_id", "principal"], + "properties": { + "schema_version": { "const": 1 }, + "session_id": { "type": "string", "minLength": 1 }, + "turn_id": { "type": "string" }, + "agent_id": { "type": "string" }, + "principal": { "type": "string", "minLength": 1 }, + "workflow": { + "type": "object", + "properties": { + "workflow_id": { "type": "string" }, + "kind": { + "type": "string", + "enum": [ + "plugin_factory_create", + "plugin_factory_edit", + "connector_auth_discover", + "job_step", + "interactive_tool", + "none" + ] + }, + "step_id": { "type": "string" }, + "step_kind": { "type": "string" } + }, + "additionalProperties": false + }, + "delivery": { + "type": "string", + "enum": ["live_chat", "notification", "silent"] + }, + "capabilities": { + "type": "array", + "items": { + "type": "string", + "enum": ["sync_web_crawl", "host_review_retry"] + }, + "uniqueItems": true + } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/file-extractor-result.schema.json b/workers/go/internal/contract/schemas/file-extractor-result.schema.json new file mode 100644 index 0000000..c4e12bc --- /dev/null +++ b/workers/go/internal/contract/schemas/file-extractor-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/file-extractor-result.json", + "title": "File extractor worker stdout", + "description": "JSON object written to stdout by derrick-file-extractor.", + "$ref": "worker-product.schema.json#/$defs/file_extractor_result" +} diff --git a/workers/go/internal/contract/schemas/guest-runtime.schema.json b/workers/go/internal/contract/schemas/guest-runtime.schema.json new file mode 100644 index 0000000..03f4f62 --- /dev/null +++ b/workers/go/internal/contract/schemas/guest-runtime.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/guest-runtime.json", + "title": "Derrick offline guest runtime", + "description": "Canonical I/O contract for script_exec and plugin.invoke guests. Trusted worker stdout (crawler, extractor) is defined in worker-product.schema.json. Swift host and Go workers must match these schemas.", + "type": "object", + "required": ["language", "stdin", "stdout"], + "additionalProperties": false, + "properties": { + "language": { + "type": "string", + "const": "go", + "description": "Guest implementation language." + }, + "stdin": { + "description": "One hop event JSON object read from standard input.", + "$ref": "hop-event.schema.json" + }, + "stdout": { + "description": "Envelope list JSON array written to standard output.", + "$ref": "envelope-list.schema.json" + }, + "binary": { + "type": "string", + "const": "/tmp/guest", + "description": "Linux guest binary path inside the worker container." + } + } +} diff --git a/workers/go/internal/contract/schemas/hop-event.schema.json b/workers/go/internal/contract/schemas/hop-event.schema.json new file mode 100644 index 0000000..a183e28 --- /dev/null +++ b/workers/go/internal/contract/schemas/hop-event.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/hop-event.json", + "title": "Guest stdin hop event", + "description": "JSON object read from stdin when the host invokes an offline guest. Canonical contract for all guest languages.", + "type": "object", + "required": ["kind"], + "properties": { + "kind": { + "type": "string", + "enum": [ + "manual", + "schedule", + "message_in_room", + "http_results", + "ui_action", + "grant_ready", + "harness", + "script" + ] + }, + "http_results": { + "type": "array", + "items": { + "type": "object", + "required": ["request_id", "status"], + "additionalProperties": false, + "properties": { + "request_id": { "type": "string" }, + "status": { "type": "integer" }, + "headers": { "type": "object", "additionalProperties": { "type": "string" } }, + "body": { "type": "string" }, + "error": { "type": ["string", "null"] } + } + } + }, + "params": { + "$ref": "connector-params.schema.json" + } + }, + "additionalProperties": false +} diff --git a/workers/go/internal/contract/schemas/web-crawler-result.schema.json b/workers/go/internal/contract/schemas/web-crawler-result.schema.json new file mode 100644 index 0000000..9af3975 --- /dev/null +++ b/workers/go/internal/contract/schemas/web-crawler-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/web-crawler-result.json", + "title": "Web crawler worker stdout", + "description": "JSON object written to stdout by derrick-web-crawler.", + "$ref": "worker-product.schema.json#/$defs/web_crawler_result" +} diff --git a/workers/go/internal/contract/schemas/worker-product.schema.json b/workers/go/internal/contract/schemas/worker-product.schema.json new file mode 100644 index 0000000..874f6bf --- /dev/null +++ b/workers/go/internal/contract/schemas/worker-product.schema.json @@ -0,0 +1,97 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/worker-product.json", + "title": "Derrick trusted worker product contracts", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "$defs": { + "string_list": { + "type": "array", + "items": { "type": "string" } + }, + "web_crawler_stop_reason": { + "type": "string", + "enum": [ + "completed", + "max_pages", + "max_depth", + "timeout", + "total_bytes", + "queue_limit", + "cancelled", + "blocked" + ] + }, + "web_crawler_page": { + "type": "object", + "required": ["url", "depth", "status_code", "title", "text", "links_found"], + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "depth": { "type": "integer" }, + "status_code": { "type": "integer" }, + "content_type": { "type": "string" }, + "title": { "type": "string" }, + "text": { "type": "string" }, + "links_found": { "type": "integer" } + } + }, + "web_crawler_result": { + "type": "object", + "required": [ + "ok", + "start_url", + "pages", + "stop_reason", + "requests_made", + "bytes_read", + "truncated", + "diagnostics" + ], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "start_url": { "type": "string" }, + "pages": { + "type": "array", + "items": { "$ref": "#/$defs/web_crawler_page" } + }, + "stop_reason": { "$ref": "#/$defs/web_crawler_stop_reason" }, + "requests_made": { "type": "integer" }, + "bytes_read": { "type": "integer" }, + "truncated": { "type": "boolean" }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + }, + "file_extractor_operation": { + "type": "string", + "enum": ["extract", "convert"] + }, + "file_extractor_file_result": { + "type": "object", + "required": ["input_name", "kind", "byte_count"], + "additionalProperties": false, + "properties": { + "input_name": { "type": "string" }, + "output_name": { "type": "string" }, + "kind": { "type": "string" }, + "byte_count": { "type": "integer" }, + "preview": { "type": "string" }, + "error": { "type": "string" } + } + }, + "file_extractor_result": { + "type": "object", + "required": ["ok", "operation", "files", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "operation": { "$ref": "#/$defs/file_extractor_operation" }, + "files": { + "type": "array", + "items": { "$ref": "#/$defs/file_extractor_file_result" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } + } + } +} diff --git a/workers/go/internal/crawler/engine.go b/workers/go/internal/crawler/engine.go new file mode 100644 index 0000000..890ea22 --- /dev/null +++ b/workers/go/internal/crawler/engine.go @@ -0,0 +1,372 @@ +package crawler + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +type extractedPage struct { + title string + text string + isHTML bool +} + +func Run(ctx context.Context, req ValidatedRequest, proxy *ProxyConfig) Result { + deadline := time.Now().Add(time.Duration(req.TimeoutSeconds) * time.Second) + engine := &bfsEngine{ + req: req, + deadline: deadline, + proxy: proxy, + client: newHTTPClient(proxy), + } + return engine.run(ctx) +} + +type queueItem struct { + url *url.URL + depth int +} + +type bfsEngine struct { + req ValidatedRequest + deadline time.Time + proxy *ProxyConfig + client *http.Client + queue []queueItem + queuedKeys map[string]bool + visitedKeys map[string]bool + pageDepths map[string]int + pages []Page + pageIndexes map[string]int + diagnostics []string + stopReason StopReason + reachedMaxDepth bool + requestsMade int + bytesRead int + lastRequestAt time.Time +} + +func (e *bfsEngine) run(ctx context.Context) Result { + startKey := URLKey(e.req.StartURL) + e.queuedKeys = map[string]bool{startKey: true} + e.visitedKeys = map[string]bool{} + e.pageDepths = map[string]int{startKey: 0} + e.pageIndexes = map[string]int{} + e.queue = []queueItem{{url: e.req.StartURL, depth: 0}} + + for len(e.queue) > 0 && e.stopReason == "" { + if time.Now().After(e.deadline) { + e.stopReason = StopTimeout + break + } + if len(e.pages) >= e.req.MaxPages { + e.stopReason = StopMaxPages + break + } + if ctx.Err() != nil { + e.stopReason = StopCancelled + break + } + + next := e.queue[0] + e.queue = e.queue[1:] + key := URLKey(next.url) + delete(e.queuedKeys, key) + + if !HostAllowed(e.req.AllowedHosts, next.url.Hostname()) || !IsHTTP(next.url) { + continue + } + if e.visitedKeys[key] { + continue + } + e.visitedKeys[key] = true + if len(e.visitedKeys) > e.req.MaxPages { + e.stopReason = StopMaxPages + break + } + + e.visit(ctx, next.url, next.depth) + } + + if e.stopReason == "" { + if e.reachedMaxDepth { + e.stopReason = StopMaxDepth + } else { + e.stopReason = StopCompleted + } + } + + truncated := false + totalChars := 0 + for i, p := range e.pages { + totalChars += len(p.Text) + if totalChars > MaximumOutputChars { + truncated = true + e.pages = e.pages[:i] + break + } + } + + return Result{ + OK: e.stopReason != StopBlocked && len(e.pages) > 0, + StartURL: e.req.StartURL.String(), + Pages: e.pages, + StopReason: e.stopReason, + RequestsMade: e.requestsMade, + BytesRead: e.bytesRead, + Truncated: truncated, + Diagnostics: e.diagnostics, + } +} + +func (e *bfsEngine) visit(ctx context.Context, pageURL *url.URL, depth int) { + if e.stopReason != "" || time.Now().After(e.deadline) { + return + } + + normalized := NormalizeURL(pageURL) + e.requestsMade++ + body, status, contentType, finalURL, err := e.fetch(ctx, normalized) + if err != nil { + e.diagnostics = append(e.diagnostics, fmt.Sprintf("%s: %s", normalized.String(), err.Error())) + return + } + + e.bytesRead += len(body) + if e.bytesRead > MaximumTotalBytes { + e.stopReason = StopTotalBytes + return + } + + extracted := extractContent(body, contentType, finalURL) + ct := contentType + page := Page{ + URL: finalURL.String(), + Depth: depth, + StatusCode: status, + ContentType: func() *string { + if ct == "" { + return nil + } + return &ct + }(), + Title: extracted.title, + Text: clipText(extracted.text, MaximumExtractedText), + } + e.pages = append(e.pages, page) + sourceKey := URLKey(finalURL) + e.pageIndexes[sourceKey] = len(e.pages) - 1 + e.pageDepths[sourceKey] = depth + + if !extracted.isHTML { + return + } + + links := parseLinks(body, finalURL) + accepted := e.enqueueLinks(sourceKey, links) + idx := e.pageIndexes[sourceKey] + e.pages[idx].LinksFound = accepted +} + +func (e *bfsEngine) enqueueLinks(sourceKey string, links []*url.URL) int { + sourceDepth := e.pageDepths[sourceKey] + accepted := 0 + for _, link := range links { + if e.stopReason != "" { + break + } + if len(links) > MaximumLinksPerPage && accepted >= MaximumLinksPerPage { + break + } + normalized := NormalizeURL(link) + key := URLKey(normalized) + if !HostAllowed(e.req.AllowedHosts, normalized.Hostname()) || !IsHTTP(normalized) { + continue + } + if e.visitedKeys[key] || e.queuedKeys[key] { + continue + } + nextDepth := sourceDepth + 1 + if nextDepth > e.req.MaxDepth { + e.reachedMaxDepth = true + continue + } + if len(e.queue) >= MaximumQueuedURLs { + e.stopReason = StopQueueLimit + break + } + e.queuedKeys[key] = true + e.pageDepths[key] = nextDepth + e.queue = append(e.queue, queueItem{url: normalized, depth: nextDepth}) + accepted++ + } + return accepted +} + +func newHTTPClient(proxy *ProxyConfig) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + if proxy != nil && proxy.Host != "" { + proxyURL, _ := url.Parse(fmt.Sprintf("http://%s:%d", proxy.Host, proxy.Port)) + transport.Proxy = http.ProxyURL(proxyURL) + if proxy.Token != "" { + transport.ProxyConnectHeader = http.Header{ + "X-Derrick-Crawler-Token": []string{proxy.Token}, + } + } + } + return &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Timeout: 20 * time.Second, + } +} + +func (e *bfsEngine) fetch(ctx context.Context, start *url.URL) (body string, status int, contentType string, final *url.URL, err error) { + e.waitForSpacing() + current := start + redirected := map[string]bool{URLKey(start): true} + + for i := 0; i <= MaximumRedirectsPerPage; i++ { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, current.String(), nil) + if err != nil { + return "", 0, "", start, err + } + req.Header.Set("User-Agent", "DerrickWebCrawler/1") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Accept-Encoding", "identity") + if e.proxy != nil && e.proxy.Token != "" { + req.Header.Set("X-Derrick-Crawler-Token", e.proxy.Token) + } + + resp, err := e.client.Do(req) + if err != nil { + return "", 0, "", start, err + } + limited := io.LimitReader(resp.Body, int64(MaximumPageBytes)) + data, err := io.ReadAll(limited) + resp.Body.Close() + if err != nil { + return "", 0, "", start, err + } + text := string(data) + ct := resp.Header.Get("Content-Type") + + if resp.StatusCode < 300 || resp.StatusCode >= 400 { + return text, resp.StatusCode, ct, current, nil + } + + location := resp.Header.Get("Location") + if location == "" { + return text, resp.StatusCode, ct, current, nil + } + redirectURL, err := url.Parse(location) + if err != nil { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + redirectURL = current.ResolveReference(redirectURL) + if !IsHTTP(redirectURL) { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + host := strings.ToLower(strings.TrimSpace(redirectURL.Hostname())) + if host == "" { + return "", 0, "", start, fmt.Errorf("redirect left the start URL origin") + } + e.req.AllowedHosts[host] = true + normalized := NormalizeURL(redirectURL) + key := URLKey(normalized) + if redirected[key] { + return "", 0, "", start, fmt.Errorf("redirect loop detected") + } + redirected[key] = true + current = normalized + e.waitForSpacing() + } + return "", 0, "", start, fmt.Errorf("redirect limit reached") +} + +func (e *bfsEngine) waitForSpacing() { + if e.lastRequestAt.IsZero() { + e.lastRequestAt = time.Now() + return + } + elapsed := time.Since(e.lastRequestAt) + minimum := time.Duration(MinRequestDelayMS) * time.Millisecond + if elapsed < minimum { + time.Sleep(minimum - elapsed) + } + e.lastRequestAt = time.Now() +} + +func extractContent(body string, contentType string, base *url.URL) extractedPage { + lower := strings.ToLower(contentType) + if strings.Contains(lower, "html") || strings.Contains(lower, "xhtml") { + return extractHTML(body) + } + prefix := body + if len(prefix) > MaximumExtractedText { + prefix = prefix[:MaximumExtractedText] + } + return extractedPage{title: "", text: prefix, isHTML: false} +} + +func extractHTML(body string) extractedPage { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + return extractedPage{text: body, isHTML: true} + } + doc.Find("script, style, noscript, template, svg").Remove() + title := strings.TrimSpace(doc.Find("title").First().Text()) + text := strings.TrimSpace(doc.Find("body").Text()) + if text == "" { + text = strings.TrimSpace(doc.Text()) + } + return extractedPage{title: title, text: text, isHTML: true} +} + +func parseLinks(body string, base *url.URL) []*url.URL { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + return nil + } + var links []*url.URL + seen := map[string]bool{} + doc.Find("a[href]").Each(func(_ int, s *goquery.Selection) { + href, ok := s.Attr("href") + if !ok || strings.TrimSpace(href) == "" { + return + } + parsed, err := url.Parse(href) + if err != nil { + return + } + resolved := base.ResolveReference(parsed) + resolved.Fragment = "" + resolved.RawFragment = "" + resolved = NormalizeURL(resolved) + key := URLKey(resolved) + if seen[key] { + return + } + seen[key] = true + links = append(links, resolved) + }) + return links +} + +func clipText(text string, limit int) string { + if len(text) <= limit { + return text + } + return text[:limit] +} diff --git a/workers/go/internal/crawler/safety.go b/workers/go/internal/crawler/safety.go new file mode 100644 index 0000000..4a7f80b --- /dev/null +++ b/workers/go/internal/crawler/safety.go @@ -0,0 +1,33 @@ +package crawler + +import "strings" + +var blockedPatterns = []struct { + substr string + reason string +}{ + {"ddos", "distributed denial-of-service behavior is not allowed."}, + {"denial of service", "denial-of-service behavior is not allowed."}, + {"dos attack", "denial-of-service behavior is not allowed."}, + {"flood", "flooding a website is not allowed."}, + {"hammer", "repeatedly hammering a website is not allowed."}, + {"stress test", "load or stress testing a third-party website is not allowed."}, + {"load test", "load or stress testing a third-party website is not allowed."}, + {"port scan", "port scanning is not a web crawl."}, + {"brute force", "brute-force activity is not allowed."}, + {"infinite loop", "unbounded or infinite crawling is not allowed."}, + {"loop forever", "unbounded or infinite crawling is not allowed."}, + {"crawl forever", "unbounded or infinite crawling is not allowed."}, + {"never stop crawling", "unbounded or infinite crawling is not allowed."}, + {"unbounded crawl", "unbounded or infinite crawling is not allowed."}, +} + +func MaliciousGoalReason(goal string) string { + normalized := strings.ReplaceAll(strings.ReplaceAll(strings.ToLower(goal), "-", " "), "_", " ") + for _, p := range blockedPatterns { + if strings.Contains(normalized, p.substr) { + return p.reason + } + } + return "" +} diff --git a/workers/go/internal/crawler/types.go b/workers/go/internal/crawler/types.go new file mode 100644 index 0000000..98f33e5 --- /dev/null +++ b/workers/go/internal/crawler/types.go @@ -0,0 +1,69 @@ +package crawler + +// Wire types mirror worker-product.schema.json (web_crawler_result) in Structure Contract. + +const ( + DefaultMaxPages = 10 + MaximumMaxPages = 100 + DefaultMaxDepth = 2 + MaximumMaxDepth = 5 + DefaultTimeoutSeconds = 120 + MaximumTimeoutSeconds = 900 + MaximumPageBytes = 1_048_576 + MaximumTotalBytes = 10 * 1_048_576 + MaximumLinksPerPage = 200 + MaximumQueuedURLs = 400 + MaximumExtractedText = 12_000 + MaximumOutputChars = 500_000 + MaximumRedirectsPerPage = 5 + MinRequestDelayMS = 150 +) + +type Request struct { + StartURL string `json:"start_url"` + Goal string `json:"goal"` + MaxPages int `json:"max_pages"` + MaxDepth int `json:"max_depth"` + TimeoutSeconds int `json:"timeout_seconds"` + AllowedHosts []string `json:"allowed_hosts,omitempty"` +} + +type Page struct { + URL string `json:"url"` + Depth int `json:"depth"` + StatusCode int `json:"status_code"` + ContentType *string `json:"content_type,omitempty"` + Title string `json:"title"` + Text string `json:"text"` + LinksFound int `json:"links_found"` +} + +type StopReason string + +const ( + StopCompleted StopReason = "completed" + StopMaxPages StopReason = "max_pages" + StopMaxDepth StopReason = "max_depth" + StopTimeout StopReason = "timeout" + StopTotalBytes StopReason = "total_bytes" + StopQueueLimit StopReason = "queue_limit" + StopCancelled StopReason = "cancelled" + StopBlocked StopReason = "blocked" +) + +type Result struct { + OK bool `json:"ok"` + StartURL string `json:"start_url"` + Pages []Page `json:"pages"` + StopReason StopReason `json:"stop_reason"` + RequestsMade int `json:"requests_made"` + BytesRead int `json:"bytes_read"` + Truncated bool `json:"truncated"` + Diagnostics []string `json:"diagnostics"` +} + +type ProxyConfig struct { + Host string + Port int + Token string +} diff --git a/workers/go/internal/crawler/url.go b/workers/go/internal/crawler/url.go new file mode 100644 index 0000000..b3c1400 --- /dev/null +++ b/workers/go/internal/crawler/url.go @@ -0,0 +1,87 @@ +package crawler + +import ( + "net" + "net/url" + "strings" +) + +func IsHTTP(u *url.URL) bool { + scheme := strings.ToLower(u.Scheme) + return scheme == "http" || scheme == "https" +} + +func NormalizeURL(u *url.URL) *url.URL { + if u == nil { + return u + } + clone := *u + clone.Scheme = strings.ToLower(clone.Scheme) + clone.Host = strings.ToLower(clone.Host) + clone.Fragment = "" + if clone.Path == "" { + clone.Path = "/" + } + if clone.Port() != "" { + if (clone.Scheme == "http" && clone.Port() == "80") || + (clone.Scheme == "https" && clone.Port() == "443") { + clone.Host = clone.Hostname() + } + } + return &clone +} + +func URLKey(u *url.URL) string { + return NormalizeURL(u).String() +} + +func ParseStartURL(raw string) (*url.URL, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return nil, err + } + if !IsHTTP(u) { + return nil, errInvalidStartURL + } + host := strings.ToLower(strings.TrimSpace(u.Hostname())) + if host == "" { + return nil, errInvalidStartURL + } + if u.User != nil { + return nil, errInvalidStartURL + } + return NormalizeURL(u), nil +} + +func ResolvedAllowedHosts(startHost string, explicit []string) map[string]bool { + hosts := map[string]bool{strings.ToLower(startHost): true} + for _, h := range explicit { + n := strings.ToLower(strings.TrimSpace(h)) + if n != "" { + hosts[n] = true + } + } + return hosts +} + +func HostAllowed(hosts map[string]bool, host string) bool { + return hosts[strings.ToLower(strings.TrimSpace(host))] +} + +func IsPrivateHost(host string) bool { + host = strings.TrimSpace(strings.ToLower(host)) + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() +} + +var errInvalidStartURL = validationError("start_url must be an http or https URL without embedded credentials.") + +type validationError string + +func (e validationError) Error() string { return string(e) } diff --git a/workers/go/internal/crawler/validate.go b/workers/go/internal/crawler/validate.go new file mode 100644 index 0000000..ae08f38 --- /dev/null +++ b/workers/go/internal/crawler/validate.go @@ -0,0 +1,66 @@ +package crawler + +import ( + "net/url" + "strings" +) + +type ValidatedRequest struct { + StartURL *url.URL + Goal string + MaxPages int + MaxDepth int + TimeoutSeconds int + AllowedHosts map[string]bool +} + +func Validate(req Request) (ValidatedRequest, error) { + goal := strings.TrimSpace(req.Goal) + if goal == "" { + return ValidatedRequest{}, validationError("A crawl goal is required.") + } + if len(goal) > 2000 { + return ValidatedRequest{}, validationError("The crawl goal is too long.") + } + if reason := MaliciousGoalReason(goal); reason != "" { + return ValidatedRequest{}, validationError("Crawl blocked: " + reason) + } + + maxPages := req.MaxPages + if maxPages == 0 { + maxPages = DefaultMaxPages + } + if maxPages < 1 || maxPages > MaximumMaxPages { + return ValidatedRequest{}, validationError("max_pages must be between 1 and 100.") + } + + maxDepth := req.MaxDepth + if maxDepth == 0 { + maxDepth = DefaultMaxDepth + } + if maxDepth < 0 || maxDepth > MaximumMaxDepth { + return ValidatedRequest{}, validationError("max_depth must be between 0 and 5.") + } + + timeout := req.TimeoutSeconds + if timeout == 0 { + timeout = DefaultTimeoutSeconds + } + if timeout < 1 || timeout > MaximumTimeoutSeconds { + return ValidatedRequest{}, validationError("timeout_seconds must be between 1 and 900.") + } + + start, err := ParseStartURL(req.StartURL) + if err != nil { + return ValidatedRequest{}, err + } + + return ValidatedRequest{ + StartURL: start, + Goal: goal, + MaxPages: maxPages, + MaxDepth: maxDepth, + TimeoutSeconds: timeout, + AllowedHosts: ResolvedAllowedHosts(start.Hostname(), req.AllowedHosts), + }, nil +} diff --git a/workers/go/internal/extractor/engine.go b/workers/go/internal/extractor/engine.go new file mode 100644 index 0000000..6e86a4f --- /dev/null +++ b/workers/go/internal/extractor/engine.go @@ -0,0 +1,383 @@ +package extractor + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/xuri/excelize/v2" +) + +func Run(req Request, inputDir, outputDir string) Result { + if len(req.Files) == 0 { + return errorResult(req.Operation, "Choose at least one attached file.") + } + if len(req.Files) > MaximumFiles { + return errorResult(req.Operation, fmt.Sprintf("You can process at most %d files.", MaximumFiles)) + } + if req.OutputFormat == "" { + req.OutputFormat = FormatMarkdown + } + if req.Operation == "" { + req.Operation = OperationExtract + } + + var files []FileResult + var diagnostics []string + previewBudget := MaximumTotalPreviewChars + + for _, name := range req.Files { + safeName, err := validatedFilename(name) + if err != nil { + files = append(files, FileResult{ + InputName: name, + Kind: kindFor(name), + Error: strPtr(err.Error()), + }) + continue + } + inputPath := filepath.Join(inputDir, safeName) + if _, err := os.Stat(inputPath); err != nil { + msg := fmt.Sprintf("%s was not found in /data/in.", safeName) + files = append(files, FileResult{ + InputName: safeName, + Kind: kindFor(safeName), + Error: &msg, + }) + continue + } + + processed, err := process(inputPath, req.Operation, req.OutputFormat) + if err != nil { + msg := err.Error() + diagnostics = append(diagnostics, msg) + files = append(files, FileResult{ + InputName: safeName, + Kind: kindFor(safeName), + Error: &msg, + }) + continue + } + outPath := filepath.Join(outputDir, processed.OutputName) + if err := os.WriteFile(outPath, processed.Data, 0o644); err != nil { + msg := err.Error() + diagnostics = append(diagnostics, msg) + files = append(files, FileResult{ + InputName: safeName, + Kind: processed.Kind, + Error: &msg, + }) + continue + } + preview := clipPreview(processed.Preview, &previewBudget) + files = append(files, FileResult{ + InputName: safeName, + OutputName: &processed.OutputName, + Kind: processed.Kind, + ByteCount: len(processed.Data), + Preview: preview, + }) + } + + ok := false + for _, f := range files { + if f.Error == nil { + ok = true + break + } + } + return Result{ + OK: ok, + Operation: req.Operation, + Files: files, + Diagnostics: diagnostics, + } +} + +func errorResult(op Operation, message string) Result { + return Result{ + OK: false, + Operation: op, + Files: []FileResult{}, + Diagnostics: []string{message}, + } +} + +type processedFile struct { + OutputName string + Kind string + Data []byte + Preview string +} + +func validatedFilename(name string) (string, error) { + trimmed := strings.TrimSpace(name) + if trimmed == "" || strings.Contains(trimmed, "/") || strings.Contains(trimmed, "\\") || + strings.Contains(trimmed, "\x00") || trimmed == "." || trimmed == ".." { + return "", fmt.Errorf("%s is not a safe file name.", name) + } + base := filepath.Base(trimmed) + if base != trimmed { + return "", fmt.Errorf("%s is not a safe file name.", name) + } + return base, nil +} + +func kindFor(filename string) string { + return strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".") +} + +func process(path string, operation Operation, format OutputFormat) (processedFile, error) { + filename := filepath.Base(path) + fileKind := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".") + extracted, err := extractText(path, fileKind) + if err != nil { + return processedFile{}, err + } + + if operation == OperationExtract { + ext := "md" + body := extracted + if format == FormatTXT { + ext = "txt" + } else { + body = fmt.Sprintf("# %s\n\n%s\n", filename, extracted) + } + return processedFile{ + OutputName: replaceExt(filename, ext), + Kind: fileKind, + Data: []byte(body), + Preview: extracted, + }, nil + } + + switch format { + case FormatXLSX: + if fileKind != "csv" && fileKind != "tsv" && fileKind != "txt" { + return processedFile{}, fmt.Errorf("That conversion is not supported for %s files.", fileKind) + } + csv := extracted + if fileKind == "tsv" { + csv = strings.ReplaceAll(extracted, "\t", ",") + } + data, err := csvToXLSX(csv) + if err != nil { + return processedFile{}, err + } + return processedFile{ + OutputName: replaceExt(filename, "xlsx"), + Kind: fileKind, + Data: data, + Preview: extracted, + }, nil + case FormatCSV: + if fileKind == "xlsx" { + raw, err := os.ReadFile(path) + if err != nil { + return processedFile{}, err + } + csv, err := xlsxToCSV(raw) + if err != nil { + return processedFile{}, err + } + return processedFile{ + OutputName: replaceExt(filename, "csv"), + Kind: fileKind, + Data: []byte(csv), + Preview: csv, + }, nil + } + if fileKind != "csv" && fileKind != "tsv" && fileKind != "txt" { + return processedFile{}, fmt.Errorf("That conversion is not supported for %s files.", fileKind) + } + return processedFile{ + OutputName: replaceExt(filename, "csv"), + Kind: fileKind, + Data: []byte(extracted), + Preview: extracted, + }, nil + default: + ext := "md" + body := extracted + if format == FormatTXT { + ext = "txt" + } else { + body = fmt.Sprintf("# %s\n\n%s\n", filename, extracted) + } + return processedFile{ + OutputName: replaceExt(filename, ext), + Kind: fileKind, + Data: []byte(body), + Preview: extracted, + }, nil + } +} + +func extractText(path, kind string) (string, error) { + switch kind { + case "pdf": + return pdfToText(path) + case "docx": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return docxToText(raw) + case "xlsx": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return xlsxToCSV(raw) + case "html", "htm": + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return htmlToText(string(raw)), nil + default: + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(raw), nil + } +} + +func pdfToText(path string) (string, error) { + if _, err := exec.LookPath("pdftotext"); err != nil { + return "", fmt.Errorf("PDF text extraction is unavailable in this image.") + } + out, err := exec.Command("pdftotext", "-layout", path, "-").Output() + if err != nil { + return "", fmt.Errorf("PDF text extraction failed.") + } + return string(out), nil +} + +func docxToText(data []byte) (string, error) { + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", err + } + for _, f := range reader.File { + if f.Name != "word/document.xml" { + continue + } + rc, err := f.Open() + if err != nil { + return "", err + } + defer rc.Close() + return parseDocxXML(rc) + } + return "", fmt.Errorf("document.xml missing from docx") +} + +func parseDocxXML(r io.Reader) (string, error) { + decoder := xml.NewDecoder(r) + var parts []string + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "t" { + var text string + if err := decoder.DecodeElement(&text, &se); err != nil { + return "", err + } + if text != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, ""), nil +} + +func xlsxToCSV(data []byte) (string, error) { + book, err := excelize.OpenReader(bytes.NewReader(data)) + if err != nil { + return "", err + } + sheets := book.GetSheetList() + if len(sheets) == 0 { + return "", nil + } + rows, err := book.GetRows(sheets[0]) + if err != nil { + return "", err + } + var lines []string + for _, row := range rows { + lines = append(lines, strings.Join(row, ",")) + } + return strings.Join(lines, "\n"), nil +} + +func csvToXLSX(csv string) ([]byte, error) { + book := excelize.NewFile() + sheet := book.GetSheetName(0) + lines := strings.Split(csv, "\n") + for i, line := range lines { + if line == "" { + continue + } + cells := strings.Split(line, ",") + for j, cell := range cells { + cellName, _ := excelize.CoordinatesToCellName(j+1, i+1) + _ = book.SetCellValue(sheet, cellName, cell) + } + } + buf, err := book.WriteToBuffer() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func htmlToText(html string) string { + var out strings.Builder + inTag := false + for _, r := range html { + switch { + case r == '<': + inTag = true + case r == '>': + inTag = false + out.WriteRune(' ') + case !inTag: + out.WriteRune(r) + } + } + return strings.Join(strings.Fields(out.String()), " ") +} + +func replaceExt(filename, ext string) string { + base := strings.TrimSuffix(filename, filepath.Ext(filename)) + return base + "." + ext +} + +func clipPreview(text string, remaining *int) *string { + if *remaining <= 0 { + return nil + } + limit := min(MaximumPreviewCharacters, *remaining) + preview := text + if len(preview) > limit { + preview = preview[:limit] + "…" + } + *remaining -= len(preview) + return &preview +} + +func strPtr(s string) *string { return &s } diff --git a/workers/go/internal/extractor/types.go b/workers/go/internal/extractor/types.go new file mode 100644 index 0000000..6463359 --- /dev/null +++ b/workers/go/internal/extractor/types.go @@ -0,0 +1,48 @@ +// Wire types mirror worker-product.schema.json (file_extractor_result) in Structure Contract. +package extractor + +const ( + InputDirectory = "/data/in" + OutputDirectory = "/data/out" + MaximumFiles = 5 + MaximumPreviewCharacters = 8000 + MaximumTotalPreviewChars = 32000 +) + +type Operation string + +const ( + OperationExtract Operation = "extract" + OperationConvert Operation = "convert" +) + +type OutputFormat string + +const ( + FormatMarkdown OutputFormat = "markdown" + FormatTXT OutputFormat = "txt" + FormatCSV OutputFormat = "csv" + FormatXLSX OutputFormat = "xlsx" +) + +type Request struct { + Operation Operation `json:"operation"` + OutputFormat OutputFormat `json:"output_format"` + Files []string `json:"files"` +} + +type FileResult struct { + InputName string `json:"input_name"` + OutputName *string `json:"output_name,omitempty"` + Kind string `json:"kind"` + ByteCount int `json:"byte_count"` + Preview *string `json:"preview,omitempty"` + Error *string `json:"error,omitempty"` +} + +type Result struct { + OK bool `json:"ok"` + Operation Operation `json:"operation"` + Files []FileResult `json:"files"` + Diagnostics []string `json:"diagnostics"` +} From 6980ece2d18cadf7027b064a2ce2d1980ee9c76e Mon Sep 17 00:00:00 2001 From: David Choi Date: Wed, 9 Sep 2026 22:25:01 -0400 Subject: [PATCH 02/12] improve app init time --- .../DockerProductImagePrewarmer.swift | 20 +------ ui/JobKeepAlive/DaemonModuleBootstrap.swift | 13 ---- .../MCPServiceDockerHelperRunner.swift | 4 +- .../Support/AppBootstrapStatus.swift | 25 ++++---- .../DockerRunner/XPCDockerRunner.swift | 59 +++++++++++++++---- ui/ui/Services/AgentServiceClient.swift | 30 +++++++--- ui/ui/Views/ContentView.swift | 50 +++++++++------- ui/uiTests/AppBootstrapStatusTests.swift | 10 ++++ 8 files changed, 130 insertions(+), 81 deletions(-) diff --git a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift index fcff866..8eca7de 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift @@ -86,30 +86,14 @@ public enum DockerProductImagePrewarmer: Sendable { } } -/// One in-flight worker image build per process. +/// Legacy alias gate — delegates to `WorkerImageGate` so crawler/script paths share one build. public actor WebCrawlerImageGate { public static let shared = WebCrawlerImageGate() - private var inFlight: Task? - public init() {} public func ensureReady(executor: @escaping DockerCLIExecutor) async throws { - if let inFlight { - try await inFlight.value - return - } - let task = Task { - try await DockerProductImagePrewarmer.ensureWorkerImage(executor: executor) - } - inFlight = task - do { - try await task.value - inFlight = nil - } catch { - inFlight = nil - throw error - } + try await WorkerImageGate.shared.ensureReady(executor: executor) } } diff --git a/ui/JobKeepAlive/DaemonModuleBootstrap.swift b/ui/JobKeepAlive/DaemonModuleBootstrap.swift index dfae72f..638ec76 100644 --- a/ui/JobKeepAlive/DaemonModuleBootstrap.swift +++ b/ui/JobKeepAlive/DaemonModuleBootstrap.swift @@ -76,9 +76,6 @@ enum DaemonModuleBootstrap { Task { await prewarmGuestRuntimeImage() } - Task { - await startWebCrawlerImageInBackground() - } } catch { fputs("[derrickd] MCP module bootstrap failed: \(error.localizedDescription)\n", stderr) } @@ -127,14 +124,4 @@ enum DaemonModuleBootstrap { } } - /// Does not block jobs or chat. A crawl that arrives during this build waits on the same task. - private static func startWebCrawlerImageInBackground() async { - guard DerrickProcessRole.isDaemon else { return } - do { - try await MCPServiceDockerHelperRunner.shared.ensureWebCrawlerImage() - fputs("[derrickd] web crawler image ready\n", stderr) - } catch { - fputs("[derrickd] web crawler image background build skipped: \(error.localizedDescription)\n", stderr) - } - } } diff --git a/ui/MCPService/MCPServiceDockerHelperRunner.swift b/ui/MCPService/MCPServiceDockerHelperRunner.swift index 2106122..b623fac 100644 --- a/ui/MCPService/MCPServiceDockerHelperRunner.swift +++ b/ui/MCPService/MCPServiceDockerHelperRunner.swift @@ -68,9 +68,9 @@ final class MCPServiceDockerHelperRunner: @unchecked Sendable { try await WorkerImageGate.shared.ensureReady(executor: makeStdinCLIExecutor()) } - /// Build the crawler image in the background. Joins an in-flight build if one exists. + /// Legacy alias — same worker image as `prewarmGuestRuntime`. func ensureWebCrawlerImage() async throws { - try await WebCrawlerImageGate.shared.ensureReady(executor: makeStdinCLIExecutor()) + try await prewarmGuestRuntime() } var hasPeerEndpoint: Bool { diff --git a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift index 5be1566..c75b9b4 100644 --- a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift +++ b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift @@ -144,16 +144,11 @@ final class AppBootstrapStatus: ObservableObject { debugLog("[bootstrap] ignore phase=\(phase.rawValue) (already ready): \(message)") return } - // Parallel bootstrap: once we move past Docker prep, do not let guest-image - // prewarm overwrite daemon/database status in the modal. - if phase == .checkingDocker || phase == .preparingImage || phase == .verifyingEnvironment { - switch self.phase { - case .connectingHelper, .loadingSession: - debugLog("[bootstrap] ignore docker phase=\(phase.rawValue) while \(self.phase.rawValue): \(message)") - return - default: - break - } + // Parallel bootstrap: keep the highest-priority in-flight step visible (daemon connect + // beats "Opening local database…" while XPC is still retrying). + if isInitializing, Self.phasePriority(phase) < Self.phasePriority(self.phase) { + debugLog("[bootstrap] ignore lower-priority phase=\(phase.rawValue) while \(self.phase.rawValue): \(message)") + return } // Don't let a cancelled re-entrant task demote ready via failed paths above. self.phase = phase @@ -226,6 +221,16 @@ final class AppBootstrapStatus: ObservableObject { debugLog("[bootstrap] failure modal dismissed") } + private static func phasePriority(_ phase: Phase) -> Int { + switch phase { + case .connectingHelper: return 4 + case .checkingDocker: return 3 + case .preparingImage, .verifyingEnvironment: return 2 + case .loadingSession: return 1 + default: return 0 + } + } + enum FailureRecovery: Equatable, Sendable { case none case retryDaemon diff --git a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift index 6c5f93e..d1177e7 100644 --- a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift +++ b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift @@ -164,11 +164,13 @@ public final class XPCDockerRunner: @unchecked Sendable { public static let shared = XPCDockerRunner() private static let serviceName = "derrick.ui.DockerRunnerHelper" - private static let prewarmWaitCeilingSeconds: UInt64 = 1_200 + private static let dockerReachableWaitCeilingSeconds: UInt64 = 60 + private static let imagePrewarmWaitCeilingSeconds: UInt64 = 1_200 private let connection: NSXPCConnection private let appLogSink: XPCAppLogSink - private let prewarmState = PrewarmState() + private let dockerReachableState = PrewarmState() + private let imagePrewarmState = PrewarmState() public init() { let sink = XPCAppLogSink() @@ -206,11 +208,35 @@ public final class XPCDockerRunner: @unchecked Sendable { } } + /// Waits until Docker Desktop responds. Does not wait for the worker image build. + public func waitUntilDockerReachable() async throws { + if dockerReachableState.isCompleted() { + return + } + if let failure = dockerReachableState.failureIfCompleted() { + throw failure + } + let timeout = NSError( + domain: "XPCDockerRunner", + code: 504, + userInfo: [ + NSLocalizedDescriptionKey: + "Docker Desktop did not respond within \(Self.dockerReachableWaitCeilingSeconds)s." + ] + ) + try await dockerReachableState.wait( + timeoutNanoseconds: Self.dockerReachableWaitCeilingSeconds * 1_000_000_000, + timeoutError: timeout + ) + } + + /// Waits until the worker image is built or verified. Joins an in-flight background build. public func waitUntilPrewarmed() async throws { - if prewarmState.isCompleted() { + try await waitUntilDockerReachable() + if imagePrewarmState.isCompleted() { return } - if let failure = prewarmState.failureIfCompleted() { + if let failure = imagePrewarmState.failureIfCompleted() { throw failure } let timeout = NSError( @@ -218,11 +244,11 @@ public final class XPCDockerRunner: @unchecked Sendable { code: 504, userInfo: [ NSLocalizedDescriptionKey: - "Guest runtime setup timed out after \(Self.prewarmWaitCeilingSeconds)s." + "Worker image setup timed out after \(Self.imagePrewarmWaitCeilingSeconds)s." ] ) - try await prewarmState.wait( - timeoutNanoseconds: Self.prewarmWaitCeilingSeconds * 1_000_000_000, + try await imagePrewarmState.wait( + timeoutNanoseconds: Self.imagePrewarmWaitCeilingSeconds * 1_000_000_000, timeoutError: timeout ) } @@ -280,14 +306,27 @@ public final class XPCDockerRunner: @unchecked Sendable { ] ) } + dockerReachableState.markCompleted() + Task { + await prewarmWorkerImage() + } + } catch { + debugLog("Docker reachability check failed: \(error.localizedDescription)") + dockerReachableState.markFailed(error) + imagePrewarmState.markFailed(error) + } + } + + private func prewarmWorkerImage() async { + do { await reportBootstrap(phase: .preparingImage, message: "Preparing worker image…") let executor = makeDockerExecutor() try await WorkerImageGate.shared.ensureReady(executor: executor) - prewarmState.markCompleted() + imagePrewarmState.markCompleted() await reportBootstrap(phase: .verifyingEnvironment, message: "Worker image ready.") } catch { - debugLog("Guest runtime prewarming failed: \(error.localizedDescription)") - prewarmState.markFailed(error) + debugLog("Worker image prewarm failed: \(error.localizedDescription)") + imagePrewarmState.markFailed(error) } } diff --git a/ui/ui/Services/AgentServiceClient.swift b/ui/ui/Services/AgentServiceClient.swift index 339fb52..73b1afd 100644 --- a/ui/ui/Services/AgentServiceClient.swift +++ b/ui/ui/Services/AgentServiceClient.swift @@ -74,7 +74,10 @@ public final class AgentServiceClient: @unchecked Sendable { /// Connect (launch-on-demand), bootstrap DB/logs, return health. Retries a few times. /// Use at app startup (and when `ensureReadyForTurn` finds the link dead). - public func ensureUpAndHealth(retries: Int = 3) async throws -> ServiceHealthReport { + public func ensureUpAndHealth( + retries: Int = 3, + verifyHealth: Bool = true + ) async throws -> ServiceHealthReport { var lastError: Error? for attempt in 0.. NSXPCListenerEndpoint? { bootstrapStatus.update(phase: .checkingDocker, message: "Starting Docker runtime…") _ = XPCDockerRunner.shared - try await XPCDockerRunner.shared.waitUntilPrewarmed() + try await XPCDockerRunner.shared.waitUntilDockerReachable() do { return try await XPCDockerRunner.shared.fetchPeerListenerEndpoint() } catch { diff --git a/ui/uiTests/AppBootstrapStatusTests.swift b/ui/uiTests/AppBootstrapStatusTests.swift index 7fd7994..a91072d 100644 --- a/ui/uiTests/AppBootstrapStatusTests.swift +++ b/ui/uiTests/AppBootstrapStatusTests.swift @@ -174,6 +174,16 @@ import Testing #expect(status.phase == .ready) } + @MainActor + @Test func loadingSessionDoesNotOverwriteConnectingHelper() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.update(phase: .connectingHelper, message: "Connecting to Derrick daemon…") + status.update(phase: .loadingSession, message: "Opening local database…") + #expect(status.phase == .connectingHelper) + #expect(status.statusMessage == "Connecting to Derrick daemon…") + } + @MainActor @Test func cancelClearsInProgressModal() { let status = freshStatus() From aaca55223b1a6f53f07d692ba13ae0296192bbd7 Mon Sep 17 00:00:00 2001 From: David Choi Date: Wed, 9 Sep 2026 22:42:34 -0400 Subject: [PATCH 03/12] consolidate api key resolver --- ui/JobService/MessagingAgentTurnClient.swift | 10 +----- ui/MCPService/MCPServiceScriptReviewer.swift | 23 +++++++++---- ui/MCPService/MCPServiceToolHost.swift | 4 --- .../Conversation/ConversationModel.swift | 23 +++---------- .../Conversation/ProfileDelegateRunner.swift | 2 +- .../LLMProviderCredentialGate.swift | 32 +++++++++++++++-- .../Services/XPCConversationToolClient.swift | 5 +-- .../Support/LLM/ConfigureScriptReviewer.swift | 17 ++-------- .../Support/LLM/SummarizerConfig.swift | 10 +----- .../Support/PluginFactoryModels.swift | 34 ++----------------- ui/ui/Views/ContentView.swift | 10 ++---- ui/uiTests/uiTests.swift | 2 ++ 12 files changed, 64 insertions(+), 108 deletions(-) diff --git a/ui/JobService/MessagingAgentTurnClient.swift b/ui/JobService/MessagingAgentTurnClient.swift index 461a1e4..12e260b 100644 --- a/ui/JobService/MessagingAgentTurnClient.swift +++ b/ui/JobService/MessagingAgentTurnClient.swift @@ -11,7 +11,7 @@ enum MessagingAgentTurnClient { let profile = try await resolveProfile(handle: route.profileHandle, repository: repository) let model = (try? JSONDecoder().decode(LLMModelChoice.self, from: profile.modelJSON)) ?? .defaultHelperModel - let apiKey = await resolveAPIKey(for: model) ?? "" + let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) ?? "" let profileContextJSON = try JSONEncoder().encode(AgentProfileTurnContext(profile: profile)) let sessionID = MessagingAgentSessionID.make( pluginID: route.pluginID, @@ -60,14 +60,6 @@ enum MessagingAgentTurnClient { throw MessagingAgentTurnClientError.profileUnavailable(handle) } - @MainActor - private static func resolveAPIKey(for model: LLMModelChoice) -> String? { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - } - private static func sendConnectorMessage( route: MessagingAgentRoute, text: String, diff --git a/ui/MCPService/MCPServiceScriptReviewer.swift b/ui/MCPService/MCPServiceScriptReviewer.swift index 9a1e498..7da7ba4 100644 --- a/ui/MCPService/MCPServiceScriptReviewer.swift +++ b/ui/MCPService/MCPServiceScriptReviewer.swift @@ -20,9 +20,8 @@ struct MCPServiceScriptReviewer: ScriptReviewer { } func review(_ args: ScriptExecutionArguments) async throws -> ScriptReviewOutcome { - guard let apiKey = MCPServiceCallContext.shared.helperAPIKey, - !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - else { + let selected = resolveSelectedModel() + guard let apiKey = await resolveAPIKey(for: selected) else { throw NSError( domain: "MCPService", code: 404, @@ -30,7 +29,6 @@ struct MCPServiceScriptReviewer: ScriptReviewer { ) } - let selected = resolveSelectedModel() do { return try await review(args, model: selected, apiKey: apiKey) } catch { @@ -38,7 +36,7 @@ struct MCPServiceScriptReviewer: ScriptReviewer { "[MCPService] reviewer model \(selected.label) failed: \(error.localizedDescription); trying defaults\n", stderr ) - if let fallback = await fallbackReview(args: args, apiKey: apiKey, excluding: selected) { + if let fallback = await fallbackReview(args: args, excluding: selected) { return fallback } throw error @@ -108,12 +106,12 @@ struct MCPServiceScriptReviewer: ScriptReviewer { private func fallbackReview( args: ScriptExecutionArguments, - apiKey: String, excluding: ReviewerModel ) async -> ScriptReviewOutcome? { var candidates: [ReviewerModel] = [Self.defaultModel, Self.secondaryDefault] candidates.removeAll { $0 == excluding } for candidate in candidates { + guard let apiKey = await resolveAPIKey(for: candidate) else { continue } do { let outcome = try await review(args, model: candidate, apiKey: apiKey) fputs("[MCPService] fallback reviewer succeeded model=\(candidate.label)\n", stderr) @@ -127,6 +125,19 @@ struct MCPServiceScriptReviewer: ScriptReviewer { } return nil } + + private func resolveAPIKey(for model: ReviewerModel) async -> String? { + await LLMProviderCredentialGate.resolveAPIKey(for: llmModelChoice(from: model)) + } + + private func llmModelChoice(from model: ReviewerModel) -> LLMModelChoice { + switch model { + case .openai(let openAIModel): + return .openai(openAIModel) + case .gemini(let geminiModel): + return .gemini(geminiModel) + } + } } /// Process-wide slots for the current MCPService tool call (handlers may not inherit TaskLocal). diff --git a/ui/MCPService/MCPServiceToolHost.swift b/ui/MCPService/MCPServiceToolHost.swift index 8808e4c..d3aa0af 100644 --- a/ui/MCPService/MCPServiceToolHost.swift +++ b/ui/MCPService/MCPServiceToolHost.swift @@ -67,10 +67,6 @@ actor MCPServiceToolHost { await WorkflowProgressPublisher.publish(stage: "factory", message: progress) } }, - apiKeyProvider: { - MCPServiceCallContext.shared.helperAPIKey - ?? TurnProcessContext.effectiveAPIKey - } ) let made = try await MCPLocalBridge.make { server in await server.registerScriptExecutionTool( diff --git a/ui/SharedAgentRuntime/Conversation/ConversationModel.swift b/ui/SharedAgentRuntime/Conversation/ConversationModel.swift index 95e02e8..e2ed82c 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationModel.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationModel.swift @@ -266,7 +266,7 @@ final class ConversationModel { let interceptor = makeContentPolicyInterceptor() let orchestrator = self.orchestrator let workerModel = helperModelSettings.workerAgentModel - let workerApiKey = resolveAPIKey(for: workerModel, turnFallback: apiKey) ?? apiKey + let workerApiKey = await LLMProviderCredentialGate.resolveAPIKey(for: workerModel) ?? apiKey let effectiveModel: LLMModelChoice let effectiveThinking: ModelThinkingOption? @@ -593,21 +593,6 @@ final class ConversationModel { return DefaultPolicyInterceptor(policy: policy) } - /// API key for a helper/worker model: keychain/env for its provider, else the active turn key. - private func resolveAPIKey(for model: LLMModelChoice, turnFallback: String) -> String? { - if let key = AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ), !key.isEmpty { - return key - } - if let turnKey = TurnProcessContext.effectiveAPIKey, !turnKey.isEmpty { - return turnKey - } - let trimmed = turnFallback.trimmingCharacters(in: .whitespacesAndNewlines) - return trimmed.isEmpty ? nil : trimmed - } - /// In-process host for orchestration tools (`agents_*`, `jobs_*`). Not used for MCP effectors. /// `nonisolated`: tool handlers must not hop to MainActor while the turn awaits the MCP local bridge /// (that pattern deadlocks when runTurn is MainActor-isolated). @@ -727,13 +712,14 @@ final class ConversationModel { wakePrompt: wakePrompt, description: description ) + let providerAPIKey = await LLMProviderCredentialGate.resolveAPIKey(for: .defaultHelperModel) let request = try JobOrderBuilder.createJobRequest( from: input, principal: principal, source: .agent, sessionID: sessionID, agentID: agentID, - helperAPIKey: TurnProcessContext.effectiveAPIKey, + helperAPIKey: providerAPIKey, helperReviewerModelJSON: reviewerJSON ) debugLog("[jobs_create] calling JobService createJob…") @@ -787,13 +773,14 @@ final class ConversationModel { wakePrompt: wakePrompt, enabled: true ) + let providerAPIKey = await LLMProviderCredentialGate.resolveAPIKey(for: .defaultHelperModel) let request = try JobOrderBuilder.createScheduleRequest( from: input, principal: principal, source: .agent, sessionID: sessionID, agentID: agentID, - helperAPIKey: TurnProcessContext.effectiveAPIKey, + helperAPIKey: providerAPIKey, helperReviewerModelJSON: reviewerJSON ) let schedule = try await placer.createSchedule(request) diff --git a/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift b/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift index 3a9f866..38f040b 100644 --- a/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift +++ b/ui/SharedAgentRuntime/Conversation/ProfileDelegateRunner.swift @@ -40,7 +40,7 @@ enum ProfileDelegateRunner { let thinking = profileContext.thinkingJSON.flatMap { try? JSONDecoder().decode(ModelThinkingOption.self, from: $0) } - let apiKey = TurnProcessContext.effectiveAPIKey ?? "" + let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) ?? "" let delegateSessionKey = MemorySessionKey( sessionID: sessionKey.sessionID, diff --git a/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift b/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift index 4dfecd8..58cb31d 100644 --- a/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift +++ b/ui/SharedAgentRuntime/LLMProviderCredentialGate.swift @@ -1,14 +1,40 @@ import Foundation import Structure -/// Whether an LLM provider has a usable API key (Keychain or `.env`, per `AppSecretResolver`). +/// Provider credential checks and API key resolution (Keychain or `.env`, per `AppSecretResolver`). @MainActor enum LLMProviderCredentialGate { - static func hasAPIKey(for provider: LLMProviderChoice, resolver: AppSecretResolver) -> Bool { + /// One key per provider — the single resolution path for chat, reviewers, workers, and jobs. + static func resolveAPIKey( + for provider: LLMProviderChoice, + resolver: AppSecretResolver = AppSecretResolver() + ) -> String? { resolver.resolve( account: provider.secretAccount, environmentKeys: provider.apiKeyEnvironmentKeys - ) != nil + ) + } + + static func resolveAPIKey( + for model: LLMModelChoice, + resolver: AppSecretResolver = AppSecretResolver() + ) -> String? { + resolveAPIKey(for: model.provider, resolver: resolver) + } + + /// Off-main callers (AgentService, MCPService, jobs). + static func resolveAPIKey(for provider: LLMProviderChoice) async -> String? { + await MainActor.run { + resolveAPIKey(for: provider) + } + } + + static func resolveAPIKey(for model: LLMModelChoice) async -> String? { + await resolveAPIKey(for: model.provider) + } + + static func hasAPIKey(for provider: LLMProviderChoice, resolver: AppSecretResolver) -> Bool { + resolveAPIKey(for: provider, resolver: resolver) != nil } static func configuredProviders(resolver: AppSecretResolver) -> [LLMProviderChoice] { diff --git a/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift b/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift index e02829a..dd09786 100644 --- a/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift +++ b/ui/SharedAgentRuntime/Services/XPCConversationToolClient.swift @@ -9,19 +9,16 @@ import Structure public struct XPCConversationToolClient: ConversationToolClient, Sendable { private let principal: ServicePrincipal private let agentsClient: MCPClient? - private let helperAPIKeyProvider: @Sendable () -> String? /// JSON `HelperModelWire` for MCP script security reviewer (from `LLMModelSettings`). private let helperReviewerModelJSONProvider: @Sendable () async -> String? public init( principal: ServicePrincipal, agentsClient: MCPClient? = nil, - helperAPIKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, helperReviewerModelJSONProvider: @escaping @Sendable () async -> String? = { nil } ) { self.principal = principal self.agentsClient = agentsClient - self.helperAPIKeyProvider = helperAPIKeyProvider self.helperReviewerModelJSONProvider = helperReviewerModelJSONProvider } @@ -85,7 +82,7 @@ public struct XPCConversationToolClient: ConversationToolClient, Sendable { principal: activePrincipal, toolName: name, argumentsJSON: argumentsJSON, - helperAPIKey: helperAPIKeyProvider(), + helperAPIKey: nil, helperReviewerModelJSON: reviewerModelJSON, pluginFactoryCreationActive: executionContextJSON != nil && TurnProcessContext.effectivePluginFactoryCreationActive, diff --git a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift index 3b9552b..035572d 100644 --- a/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift +++ b/ui/SharedAgentRuntime/Support/LLM/ConfigureScriptReviewer.swift @@ -21,7 +21,7 @@ actor ConfiguredScriptReviewer: ScriptReviewer { func review(_ args: ScriptExecutionArguments) async throws -> ScriptReviewOutcome { let selectedModel = await MainActor.run { settings.scriptReviewerModel } - guard let apiKey = await resolveAPIKey(for: selectedModel) else { + guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: selectedModel) else { await MainActor.run { debugLog( "Helper reviewer model \(selectedModel.helperDisplayName) unavailable; trying default helper reviewer." @@ -83,7 +83,7 @@ actor ConfiguredScriptReviewer: ScriptReviewer { if selectedModel == defaultModel { return nil } - guard let apiKey = await resolveAPIKey(for: defaultModel) else { + guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: defaultModel) else { return nil } @@ -111,17 +111,4 @@ actor ConfiguredScriptReviewer: ScriptReviewer { return try await reviewer.review(args) } } - - private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - // AgentService XPC process cannot read the UI keychain; use the turn-supplied key. - return TurnProcessContext.effectiveAPIKey - } } diff --git a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift index fe516fd..8697e4d 100644 --- a/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift +++ b/ui/SharedAgentRuntime/Support/LLM/SummarizerConfig.swift @@ -111,15 +111,7 @@ actor ConfiguredMemorySummarizer: MemorySummarizer { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return TurnProcessContext.effectiveAPIKey + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private static func makeSummary(text: String, keywords: [String], sourceTokenCount: Int) -> MemorySummary { diff --git a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift index e487ee7..9153314 100644 --- a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift +++ b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift @@ -13,22 +13,18 @@ actor ConfiguredPluginFactoryService { private let thinkingSettings: LLMModelThinkingSettings private let executor: any PluginFactoryExecutor private let logger: PluginFactoryLogger - private let apiKeyProvider: @Sendable () -> String? - init( repository: DBRepository, settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, executor: any PluginFactoryExecutor, - logger: @escaping PluginFactoryLogger = { _ in }, - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey } + logger: @escaping PluginFactoryLogger = { _ in } ) { self.repository = repository self.settings = settings self.thinkingSettings = thinkingSettings self.executor = executor self.logger = logger - self.apiKeyProvider = apiKeyProvider } func build( @@ -43,7 +39,6 @@ actor ConfiguredPluginFactoryService { settings: settings, thinkingSettings: thinkingSettings, existingReleases: existingReleases, - apiKeyProvider: apiKeyProvider, logger: logger ), executor: executor, @@ -51,7 +46,6 @@ actor ConfiguredPluginFactoryService { inner: ConfiguredPluginSafetyReviewer( settings: settings, thinkingSettings: thinkingSettings, - apiKeyProvider: apiKeyProvider, logger: logger ) ), @@ -68,20 +62,17 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { private let settings: LLMModelSettings private let thinkingSettings: LLMModelThinkingSettings private let existingReleases: [PluginFactoryReleaseSummary] - private let apiKeyProvider: @Sendable () -> String? private let logger: PluginFactoryLogger init( settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, existingReleases: [PluginFactoryReleaseSummary] = [], - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, logger: @escaping PluginFactoryLogger = { _ in } ) { self.settings = settings self.thinkingSettings = thinkingSettings self.existingReleases = existingReleases - self.apiKeyProvider = apiKeyProvider self.logger = logger } @@ -107,15 +98,7 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return apiKeyProvider() + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private func stream( @@ -315,18 +298,15 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private let settings: LLMModelSettings private let thinkingSettings: LLMModelThinkingSettings - private let apiKeyProvider: @Sendable () -> String? private let logger: PluginFactoryLogger init( settings: LLMModelSettings, thinkingSettings: LLMModelThinkingSettings, - apiKeyProvider: @escaping @Sendable () -> String? = { TurnProcessContext.effectiveAPIKey }, logger: @escaping PluginFactoryLogger = { _ in } ) { self.settings = settings self.thinkingSettings = thinkingSettings - self.apiKeyProvider = apiKeyProvider self.logger = logger } @@ -354,15 +334,7 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { } private func resolveAPIKey(for model: LLMModelChoice) async -> String? { - if let key = await MainActor.run(body: { - AppSecretResolver().resolve( - account: model.provider.secretAccount, - environmentKeys: model.provider.apiKeyEnvironmentKeys - ) - }), !key.isEmpty { - return key - } - return apiKeyProvider() + await LLMProviderCredentialGate.resolveAPIKey(for: model) } private func stream( diff --git a/ui/ui/Views/ContentView.swift b/ui/ui/Views/ContentView.swift index 8a24fb7..f4e2783 100644 --- a/ui/ui/Views/ContentView.swift +++ b/ui/ui/Views/ContentView.swift @@ -275,10 +275,7 @@ struct ContentView: View { } private var currentHelperAPIKey: String? { - secretResolver.resolve( - account: selectedProvider.secretAccount, - environmentKeys: selectedProvider.apiKeyEnvironmentKeys - ) + LLMProviderCredentialGate.resolveAPIKey(for: selectedProvider, resolver: secretResolver) } private var currentHelperReviewerModelJSON: String? { @@ -1496,10 +1493,7 @@ struct ContentView: View { } private func resolveAPIKey() -> String? { - secretResolver.resolve( - account: selectedModel.provider.secretAccount, - environmentKeys: selectedModel.provider.apiKeyEnvironmentKeys - ) + LLMProviderCredentialGate.resolveAPIKey(for: selectedModel.provider, resolver: secretResolver) } @ViewBuilder diff --git a/ui/uiTests/uiTests.swift b/ui/uiTests/uiTests.swift index c2687c8..a3f64c3 100644 --- a/ui/uiTests/uiTests.swift +++ b/ui/uiTests/uiTests.swift @@ -205,6 +205,8 @@ import DBRepository #expect(LLMProviderCredentialGate.hasAPIKey(for: .openai, resolver: resolver)) #expect(!LLMProviderCredentialGate.hasAPIKey(for: .google, resolver: resolver)) #expect(LLMProviderCredentialGate.configuredProviders(resolver: resolver) == [.openai]) + #expect(LLMProviderCredentialGate.resolveAPIKey(for: .openai, resolver: resolver) == "test") + #expect(LLMProviderCredentialGate.resolveAPIKey(for: .google, resolver: resolver) == nil) } @Test func llmFailureClassifierDetectsCreditErrors() { From db7458a51771d20ad342d92fd5f60138f72dd053 Mon Sep 17 00:00:00 2001 From: David Choi Date: Wed, 9 Sep 2026 23:17:02 -0400 Subject: [PATCH 04/12] fix script use consolidated schema --- .githooks/pre-commit | 1 + .../LiveFactoryModels.swift | 43 +-- .../MCPServer/Script/GuestHopLoop.swift | 1 + .../MCPServer/Script/HostHTTPClient.swift | 2 +- .../MCPServer/Script/SystemInstruction.swift | 2 +- .../MCPServer/WebCrawlerToolModule.swift | 13 +- .../Factory/PluginFactoryImplementation.swift | 27 +- .../AppLayerServices/News/NewsSourceURL.swift | 66 +++- .../SharedAgentRuntime/PromptResources.swift | 4 - .../ScriptExecContract.generated.swift | 13 + .../Sources/Contract/GuestContract.swift | 1 + .../contracts/script-exec-contract.json | 173 +++++++++ .../schemas/script-exec-contract.schema.json | 346 ++++++++++++++++++ .../Contract/ScriptExecContractDocument.swift | 250 +++++++++++++ .../ScriptExecContractIntegrity.swift | 93 +++++ .../Contract/ScriptExecContractPrompts.swift | 56 +++ .../Contract/ScriptExecContractStore.swift | 64 ++++ .../DockerRunnerXPC/DockerWorkerRuntime.swift | 2 +- .../Plugin/Envelope/DerrickGuestGo.swift | 58 +-- .../AppLayerServicesWireTests.swift | 10 +- .../StructureTests/GuestContractTests.swift | 1 + .../StructureTests/NewsReaderTests.swift | 19 + .../ScriptExecContractTests.swift | 50 +++ readme.md | 2 +- scripts/generate-script-exec-contract.swift | 121 ++++++ .../Conversation/ConversationPipeline.swift | 5 - .../ConversationPipelinePolicy.swift | 2 +- .../conversation_rag_instructions.md | 12 - .../Resources/mcp_tool_instructions.md | 28 +- .../Resources/script_reviewer_instructions.md | 37 -- .../Support/PluginFactoryModels.swift | 58 +-- ui/ui/Views/PluginHTMLResultExtractor.swift | 2 + ui/uiTests/PromptResourcesTests.swift | 2 +- .../schemas/script-exec-contract.schema.json | 346 ++++++++++++++++++ 34 files changed, 1627 insertions(+), 283 deletions(-) create mode 100644 packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift create mode 100644 packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json create mode 100644 packages/Structure/Sources/Contract/ScriptExecContractDocument.swift create mode 100644 packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift create mode 100644 packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift create mode 100644 packages/Structure/Sources/Contract/ScriptExecContractStore.swift create mode 100644 packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift create mode 100755 scripts/generate-script-exec-contract.swift delete mode 100644 ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md create mode 100644 workers/go/internal/contract/schemas/script-exec-contract.schema.json diff --git a/.githooks/pre-commit b/.githooks/pre-commit index b920297..1cfc9f3 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -3,3 +3,4 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" "$ROOT/scripts/verify-no-secrets.sh" --staged "$ROOT/scripts/generate-connector-contract.swift" --check +"$ROOT/scripts/generate-script-exec-contract.swift" --check diff --git a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift index af2a46c..e721d2c 100644 --- a/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift +++ b/packages/MCPServer/Sources/FactoryHarnessSupport/LiveFactoryModels.swift @@ -75,41 +75,8 @@ public actor LiveFactoryBuilder: PluginFactoryBuilder { private static func builderSystemPrompt(for userGoal: String) -> String { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. - Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), go_source (string), - test_input_json (string containing valid JSON — a serialized object, not prose), - skill_files (array of objects with path and body), - secrets (array of objects with id, label, and kind; required for connector plugins), - role (string, optional: "connector" or "standard"), - messaging_ops (array of strings, required for connector role). - plugin_id must use lowercase letters, numbers, hyphens, and dots only - (for example my-connector). Never use underscores in plugin_id. - If the plugin needs a username, password, token, or API key, declare them in secrets. - kind must be username, password, token, or api_key. id is a stable Keychain key - such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in go_source. - Set role to "connector" when the plugin sends and receives messages with an external - messaging service (any chat or mail connector). Omit role or use "standard" otherwise. - For role connector, include messaging_ops: an array of implemented ops - (send_message, poll_inbox, sync_threads). It must match the user goal scope and test_input_json. - Do not return manifest_json. The host creates the canonical Agent Plugin manifest. - If skill_files is not needed, return an empty array. Every skill file path must be exactly - skills//SKILL.md. Never use manifest.json or other paths in skill_files. - \(DerrickGuestGo.modelContract) + \(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) - Before returning the draft, self-check the implementation: - - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - - Match host responses by the emitted request_id. - - Use only the Go standard library (no net/http, os/exec, or filesystem access). - - The direct test input must exercise the terminal result path with matching http_results fixtures. - For messaging connector plugins (role connector) that call a vendor HTTP API: - - Declare secrets in the manifest only. Never hard-code credentials. - - Parse each http_results body as JSON when the vendor returns JSON. - - When scope includes send_message, the final result.emit must include sent_message. - - Direct tests for poll_inbox must include a non-empty messages array; runtime empty messages with vendor success is success. - - When scope includes sync_threads, the final result.emit must include a non-empty threads array. - Each thread needs vendor_thread_id (opaque vendor ID) and title (human label for the host channel picker). - Emit only conversations the saved secret can access; for Slack skip channels where is_member is false. When vendor documentation is supplied in the user prompt, use it only to fill may_call HTTP details. """ } @@ -199,12 +166,8 @@ public actor LiveFactoryReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ - You are Derrick's independent plugin alignment and safety reviewer. - Return exactly one JSON object: - {"decision":"approved|rejected","summary":"...","findings":[ - {"severity":"info|warning|blocking","category":"alignment|safety|correctness|privacy|supplyChain","message":"..."} - ]} - Reject unsafe or non-deterministic code. Approve when direct test output matches fixtures through result.emit. + Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. + \(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) """ } diff --git a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift index 943248a..ed9aae6 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/GuestHopLoop.swift @@ -62,6 +62,7 @@ public enum GuestHopLoop: Sendable { } lastSummary = envelope.payload["summary"]?.stringValue ?? envelope.payload["content"]?.stringValue + ?? envelope.payload["markdown"]?.stringValue ?? envelope.payload["html"]?.stringValue ?? envelope.payload["text"]?.stringValue ?? envelope.payload["title"]?.stringValue diff --git a/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift b/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift index 629c803..b706963 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/HostHTTPClient.swift @@ -62,7 +62,7 @@ public actor HostHTTPClient { guard !trimmed.isEmpty, let url = URL(string: wireURL), url.scheme != nil, url.host != nil else { return HostHTTPFetch(status: 0, headers: [:], body: "", error: "invalid_url") } - var currentURL = url + var currentURL = NewsSourceURL.canonicalFetchURL(url) var currentMethod = request.method var currentBody = wire.body let envelopeHeaders = wire.headers diff --git a/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift b/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift index 6ea9c13..a05bfb7 100644 --- a/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift +++ b/packages/MCPServer/Sources/MCPServer/Script/SystemInstruction.swift @@ -9,5 +9,5 @@ import Foundation import Structure public var ReviewerSystemPrompt: String { - DerrickBundledText.mustLoad("script_reviewer_instructions.md") + ScriptExecContractPrompts.reviewerGuide() } diff --git a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift index 29d8d17..bfa60b2 100644 --- a/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift +++ b/packages/MCPServer/Sources/MCPServer/WebCrawlerToolModule.swift @@ -149,15 +149,16 @@ public enum WebCrawlerToolModule: MCPToolModule { let timeoutSeconds = intValue(arguments["timeout_seconds"]) ?? 120 guard !startURL.isEmpty else { throw WebCrawlerToolError.invalidStartURL } - guard let url = URL(string: startURL), - let scheme = url.scheme?.lowercased(), + guard let rawURL = URL(string: startURL), + let scheme = rawURL.scheme?.lowercased(), scheme == "http" || scheme == "https", - url.host?.isEmpty == false, - url.user == nil, - url.password == nil + rawURL.host?.isEmpty == false, + rawURL.user == nil, + rawURL.password == nil else { throw WebCrawlerToolError.invalidStartURL } + let url = NewsSourceURL.canonicalFetchURL(rawURL, contextHint: goal) guard !goal.isEmpty else { throw WebCrawlerToolError.emptyGoal } guard goal.count <= 2_000 else { throw WebCrawlerToolError.goalTooLong } if let reason = maliciousGoalReason(goal) { @@ -174,7 +175,7 @@ public enum WebCrawlerToolModule: MCPToolModule { } return WebCrawlerWireRequest( - startURL: startURL, + startURL: url.absoluteString, goal: goal, maxPages: maxPages, maxDepth: maxDepth, diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift index fb7cdfd..cee3ef6 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift @@ -76,12 +76,8 @@ public struct PluginFactorySession: Sendable { "Fix every item below in your next JSON draft response:", ] parts.append(contentsOf: findings.map { "- \($0)" }) - parts.append( - """ - Connector test_input_json must use a hops array replayed by the factory. Match each http.request \ - request_id to an http_results fixture. Declare the same ops in messaging_ops and params.messaging_op. - """ - ) + parts.append(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) + parts.append(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) return parts.joined(separator: "\n") case .reviewRejected(let summary, let findings): var parts = [ @@ -92,23 +88,8 @@ public struct PluginFactorySession: Sendable { parts.append("Findings:") parts.append(contentsOf: findings.map { "- \($0)" }) } - parts.append( - """ - Connector protocol (do not add rules): - \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) - """ - ) - parts.append( - """ - Before returning the next draft, update test_input_json to a hops array replayed by the factory: - {"hops":[{"kind":"message_in_room","params":{"messaging_op":"send_message",...}},\ - {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ - "params":{...}}]} - Include http_results fixtures for every messaging_op you implement. Match request_id values \ - in fixtures to the http.request envelopes your go_source emits. De-duplicate http_results \ - by request_id using stable sorting — do not overwrite duplicates by response order. - """ - ) + parts.append(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) + parts.append(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) return parts.joined(separator: "\n") default: return error.localizedDescription diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift index a07e64a..7317b3e 100644 --- a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift +++ b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift @@ -2,30 +2,82 @@ import Foundation /// Rewrites well-known news homepages to a public RSS/Atom endpoint the host can parse. public enum NewsSourceURL { - public static func canonicalFetchURL(_ url: URL) -> URL { + /// Rewrites Google News HTML/topic URLs to RSS feeds the host can fetch reliably. + /// `contextHint` may carry a crawl goal or user prompt (for example "tech news"). + public static func canonicalFetchURL(_ url: URL, contextHint: String? = nil) -> URL { let host = (url.host ?? "").lowercased() guard isGoogleNewsHost(host) else { return url } let path = url.path.lowercased() if path.contains("/rss") || path.hasSuffix(".xml") { return url } + if path.contains("/topics/") { + if let section = googleNewsSection(from: contextHint) { + return googleNewsSectionRSS(section: section) + } + return googleNewsGeneralRSS() + } var parts = URLComponents(url: url, resolvingAgainstBaseURL: false) ?? URLComponents() parts.scheme = "https" parts.host = "news.google.com" parts.path = "/rss" if parts.queryItems == nil || parts.queryItems?.isEmpty == true { - parts.queryItems = [ - URLQueryItem(name: "hl", value: "en-US"), - URLQueryItem(name: "gl", value: "US"), - URLQueryItem(name: "ceid", value: "US:en"), - ] + parts.queryItems = defaultLocaleQueryItems } parts.fragment = nil - return parts.url ?? URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + return parts.url ?? googleNewsGeneralRSS() } public static func isGoogleNewsHost(_ host: String) -> Bool { let value = host.lowercased() return value == "news.google.com" || value.hasSuffix(".news.google.com") } + + private static let defaultLocaleQueryItems = [ + URLQueryItem(name: "hl", value: "en-US"), + URLQueryItem(name: "gl", value: "US"), + URLQueryItem(name: "ceid", value: "US:en"), + ] + + private static func googleNewsGeneralRSS() -> URL { + URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + } + + private static func googleNewsSectionRSS(section: String) -> URL { + var parts = URLComponents() + parts.scheme = "https" + parts.host = "news.google.com" + parts.path = "/rss/headlines/section/topic/\(section)" + parts.queryItems = defaultLocaleQueryItems + return parts.url ?? googleNewsGeneralRSS() + } + + private static func googleNewsSection(from contextHint: String?) -> String? { + let hint = (contextHint ?? "") + .lowercased() + .replacingOccurrences(of: "-", with: " ") + .replacingOccurrences(of: "_", with: " ") + if hint.contains("tech") { + return "TECHNOLOGY" + } + if hint.contains("business") || hint.contains("finance") || hint.contains("market") { + return "BUSINESS" + } + if hint.contains("science") { + return "SCIENCE" + } + if hint.contains("sport") { + return "SPORTS" + } + if hint.contains("health") { + return "HEALTH" + } + if hint.contains("entertainment") { + return "ENTERTAINMENT" + } + if hint.contains("world") { + return "WORLD" + } + return nil + } } diff --git a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift index 9f83297..cf769fa 100644 --- a/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift +++ b/packages/Structure/Sources/AppLayerServices/SharedAgentRuntime/PromptResources.swift @@ -31,10 +31,6 @@ public enum PromptResources { try load(named: "files_extract_skill", from: resourceRoot) } - public static func scriptReviewerInstructions(from resourceRoot: URL? = nil) throws -> String { - try DerrickBundledText.load("script_reviewer_instructions.md", from: resourceRoot) - } - public static func workerOverlay(from resourceRoot: URL? = nil) throws -> String { try DerrickBundledText.load("worker_overlay.md", from: resourceRoot) } diff --git a/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift b/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift new file mode 100644 index 0000000..e64934f --- /dev/null +++ b/packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift @@ -0,0 +1,13 @@ +// Automatically generated by scripts/generate-script-exec-contract.swift. DO NOT EDIT. + +/// SHA-256 of script_exec protocol JSON and schemas. `swift test` fails when this is stale. +public enum ScriptExecContractFingerprint: Sendable { + public static let sha256 = "3a543a59f64b9c72a1fe94b8845af719740c17dfbc5ea81f5849c9554dcc8c89" + public static let sourceFiles: [String] = [ + "schemas/script-exec-contract.schema.json", + "schemas/guest-runtime.schema.json", + "schemas/hop-event.schema.json", + "schemas/envelope-list.schema.json", + "contracts/script-exec-contract.json", + ] +} diff --git a/packages/Structure/Sources/Contract/GuestContract.swift b/packages/Structure/Sources/Contract/GuestContract.swift index 25568ca..fdf034a 100644 --- a/packages/Structure/Sources/Contract/GuestContract.swift +++ b/packages/Structure/Sources/Contract/GuestContract.swift @@ -15,6 +15,7 @@ public enum GuestContract: Sendable { case workerProduct = "worker-product.schema.json" case webCrawlerResult = "web-crawler-result.schema.json" case fileExtractorResult = "file-extractor-result.schema.json" + case scriptExecContract = "script-exec-contract.schema.json" } public static func loadSchemaText(_ schema: Schema) throws -> String { diff --git a/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json b/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json new file mode 100644 index 0000000..99f30c6 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json @@ -0,0 +1,173 @@ +{ + "version": 1, + "runtime": { + "language": "go", + "package": "main", + "binary": "/tmp/guest", + "stdlib_only": true, + "forbidden_imports": ["net/http", "net", "os/exec"], + "allowed_os_usage": ["os.Stdin", "os.Stdout"], + "example_go": "package main\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n)\n\nfunc main() {\n\tvar event map[string]any\n\tif err := json.NewDecoder(os.Stdin).Decode(&event); err != nil {\n\t\treturn\n\t}\n\temit([]map[string]any{{\"verb\": \"result.emit\", \"title\": \"Result\", \"summary\": \"done\"}})\n}\n\nfunc emit(envelopes []map[string]any) {\n\tenc := json.NewEncoder(os.Stdout)\n\tenc.SetEscapeHTML(false)\n\t_ = enc.Encode(envelopes)\n}" + }, + "io": { + "stdin_schema": "hop-event.schema.json", + "stdout_schema": "envelope-list.schema.json", + "guest_runtime_schema": "guest-runtime.schema.json" + }, + "workflow": { + "first_hop_verbs": ["http.request"], + "http_results_event_kind": "http_results", + "terminal_verbs": ["result.emit", "message.post"], + "match_http_by": "request_id", + "http_results_accumulate": true, + "post_body_field": "json" + }, + "output": { + "fields": { + "content": { + "purpose": "parsed plain-text summaries from fetched HTML, XML, or RSS", + "host_strips_incidental_markup": true + }, + "summary": { + "purpose": "short plain-text headline or one-line result", + "host_strips_incidental_markup": true + }, + "html": { + "purpose": "only when the user explicitly requested HTML", + "host_allowlist_sanitizes": true + }, + "markdown": { + "purpose": "intentional Markdown formatting only", + "prefer_content_for_extracted_text": true + } + }, + "forbidden_patterns": ["repr(http_results)"], + "raw_body_in_content_requires_explicit_user_request": true + }, + "agent": { + "prefer_direct_urls_over_serp_html": true, + "retry_with_different_urls_on_empty_fetch": true, + "max_correction_attempts_after_block": 1 + }, + "review": { + "fail_fast": true, + "response_schema": { + "alignedWithRequest": "boolean", + "confidence": "number 0.0-1.0", + "suggestedAction": "allow|deny", + "concerns": "string[]", + "summary": "string" + }, + "checks": [ + { + "id": "intent_alignment", + "order": 1, + "description": "Script, description, reason, and user prompt are consistent.", + "notes": [ + "Derrick has a job scheduler (jobs_create, run_after_seconds, cron). Timing is applied by JobService before this script runs.", + "The script must do the work immediately when invoked.", + "Words like delayed, scheduled, in 7 seconds, later, or run_after refer to the scheduler, not sleep inside the script.", + "Do not deny a script that performs the requested work just because it has no delay." + ] + }, + { + "id": "no_secret_literals", + "order": 2, + "description": "No tokens, API keys, passwords, or other secret literals in the source." + }, + { + "id": "terminal_result_not_fetch_only", + "order": 3, + "description": "The script implements the requested terminal result, not just the fetch.", + "notes": [ + "For summarize, list, inspect, or extract requests, the http_results branch must parse the response body and emit the requested data.", + "Do not allow repr(http_results), a fetch-only confirmation, or an entire raw body copied to content unless the user explicitly requested the raw source.", + "If raw HTML is requested, html is allowed because the host sanitizes it before rendering." + ] + }, + { + "id": "untrusted_remote_content", + "order": 4, + "description": "Fetched HTML or XML is untrusted input; output must use envelope-list output fields correctly.", + "notes": [ + "Prefer content or summary for extracted plain-text lists and headlines.", + "Use html only when the user explicitly asked for HTML.", + "Use markdown only for intentional Markdown formatting.", + "Approve scripts that parse RSS, XML, or HTML and emit normalized plain text in content or summary.", + "Do not deny solely because the source was XML or HTML or because the guest does not re-validate http or https on plain-text lines." + ] + } + ], + "do_not_deny_for": [ + "go_style", + "envelope_construction", + "destination_urls", + "absence_of_dependencies" + ], + "enforced_elsewhere": [ + "static_go_verifier_enforces_forbidden_imports", + "guest_has_no_network_host_performs_http", + "host_applies_ssrf_on_http" + ] + }, + "rules": { + "guest_has_no_network": true, + "host_performs_http": true, + "host_applies_ssrf": true, + "static_verifier_enforces_imports": true, + "scheduler_timing_not_in_script": true, + "deterministic_output_required": true, + "stable_sort_and_dedupe_collections": true, + "no_time_random_uuid_for_visible_output": true, + "match_http_results_by_request_id": true + }, + "plugin_factory": { + "manifest": { + "host_creates_manifest": true, + "do_not_return_manifest_json": true, + "agent_plugin_schema": "Agent Plugin 1.0", + "entrypoint": "./app.derrick/plugin.go", + "plugin_id_allowed_chars": "lowercase letters, numbers, hyphens, dots", + "plugin_id_forbidden_chars": ["underscore"], + "roles": ["connector", "standard"], + "connector_messaging_ops": ["send_message", "poll_inbox", "sync_threads"], + "secret_kinds": ["username", "password", "token", "api_key"], + "never_embed_credentials_in_go_source": true, + "host_lists_connectors_under_messaging": true + }, + "builder": { + "response_keys": [ + "plugin_id", + "version", + "description", + "go_source", + "test_input_json", + "skill_files", + "secrets", + "role", + "messaging_ops" + ], + "skill_files_path_pattern": "skills//SKILL.md", + "empty_skill_files_when_unused": true, + "test_input_is_serialized_json_object": true, + "test_input_must_not_be_empty": true, + "test_input_exercises_terminal_result": true, + "connector_test_input_uses_hops_array": true, + "connector_parse_json_http_results_when_vendor_returns_json": true + }, + "review": { + "compilation_success_not_approval": true, + "response_schema": { + "decision": "approved|rejected", + "summary": "string", + "findings": "[{severity: info|warning|blocking, category: alignment|safety|correctness|privacy|supplyChain, message: string}]" + }, + "reject_non_go_source": true, + "reject_raw_network_outside_http_request_envelopes": true, + "reject_missing_stdin_read": true, + "source_derived_titles_may_be_fragments": true, + "reject_unsupported_direct_test_claims": true, + "connector_rules_document": "connector-contract.json" + } + } +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json new file mode 100644 index 0000000..2e36901 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/script-exec-contract.schema.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/script-exec-contract.json", + "title": "Derrick script_exec protocol document", + "description": "Canonical guest runtime, workflow, output, agent, and reviewer rules for script_exec and offline Go guests.", + "type": "object", + "required": [ + "version", + "runtime", + "io", + "workflow", + "output", + "agent", + "review", + "rules", + "plugin_factory" + ], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "runtime": { "$ref": "#/$defs/runtime" }, + "io": { "$ref": "#/$defs/io" }, + "workflow": { "$ref": "#/$defs/workflow" }, + "output": { "$ref": "#/$defs/output" }, + "agent": { "$ref": "#/$defs/agent" }, + "review": { "$ref": "#/$defs/review" }, + "rules": { "$ref": "#/$defs/rules" }, + "plugin_factory": { "$ref": "#/$defs/plugin_factory" } + }, + "$defs": { + "runtime": { + "type": "object", + "required": [ + "language", + "package", + "binary", + "stdlib_only", + "forbidden_imports", + "allowed_os_usage", + "example_go" + ], + "additionalProperties": false, + "properties": { + "language": { "type": "string", "const": "go" }, + "package": { "type": "string", "const": "main" }, + "binary": { "type": "string", "const": "/tmp/guest" }, + "stdlib_only": { "type": "boolean" }, + "forbidden_imports": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "allowed_os_usage": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "example_go": { "type": "string", "minLength": 1 } + } + }, + "io": { + "type": "object", + "required": ["stdin_schema", "stdout_schema", "guest_runtime_schema"], + "additionalProperties": false, + "properties": { + "stdin_schema": { "type": "string", "const": "hop-event.schema.json" }, + "stdout_schema": { "type": "string", "const": "envelope-list.schema.json" }, + "guest_runtime_schema": { "type": "string", "const": "guest-runtime.schema.json" } + } + }, + "workflow": { + "type": "object", + "required": [ + "first_hop_verbs", + "http_results_event_kind", + "terminal_verbs", + "match_http_by", + "http_results_accumulate", + "post_body_field" + ], + "additionalProperties": false, + "properties": { + "first_hop_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "http_results_event_kind": { "type": "string" }, + "terminal_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "match_http_by": { "type": "string", "const": "request_id" }, + "http_results_accumulate": { "type": "boolean" }, + "post_body_field": { "type": "string", "const": "json" } + } + }, + "output": { + "type": "object", + "required": ["fields", "forbidden_patterns", "raw_body_in_content_requires_explicit_user_request"], + "additionalProperties": false, + "properties": { + "fields": { + "type": "object", + "required": ["content", "summary", "html", "markdown"], + "additionalProperties": false, + "properties": { + "content": { "$ref": "#/$defs/output_field" }, + "summary": { "$ref": "#/$defs/output_field" }, + "html": { "$ref": "#/$defs/output_field" }, + "markdown": { "$ref": "#/$defs/output_field" } + } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" } + }, + "raw_body_in_content_requires_explicit_user_request": { "type": "boolean" } + } + }, + "output_field": { + "type": "object", + "required": ["purpose"], + "additionalProperties": true, + "properties": { + "purpose": { "type": "string" }, + "host_strips_incidental_markup": { "type": "boolean" }, + "host_allowlist_sanitizes": { "type": "boolean" }, + "prefer_content_for_extracted_text": { "type": "boolean" } + } + }, + "agent": { + "type": "object", + "required": [ + "prefer_direct_urls_over_serp_html", + "retry_with_different_urls_on_empty_fetch", + "max_correction_attempts_after_block" + ], + "additionalProperties": false, + "properties": { + "prefer_direct_urls_over_serp_html": { "type": "boolean" }, + "retry_with_different_urls_on_empty_fetch": { "type": "boolean" }, + "max_correction_attempts_after_block": { "type": "integer", "minimum": 0 } + } + }, + "review": { + "type": "object", + "required": [ + "fail_fast", + "response_schema", + "checks", + "do_not_deny_for", + "enforced_elsewhere" + ], + "additionalProperties": false, + "properties": { + "fail_fast": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": [ + "alignedWithRequest", + "confidence", + "suggestedAction", + "concerns", + "summary" + ], + "additionalProperties": false, + "properties": { + "alignedWithRequest": { "type": "string" }, + "confidence": { "type": "string" }, + "suggestedAction": { "type": "string" }, + "concerns": { "type": "string" }, + "summary": { "type": "string" } + } + }, + "checks": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/review_check" } + }, + "do_not_deny_for": { + "type": "array", + "items": { "type": "string" } + }, + "enforced_elsewhere": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "review_check": { + "type": "object", + "required": ["id", "order", "description"], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "order": { "type": "integer", "minimum": 1 }, + "description": { "type": "string" }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "rules": { + "type": "object", + "required": [ + "guest_has_no_network", + "host_performs_http", + "host_applies_ssrf", + "static_verifier_enforces_imports", + "scheduler_timing_not_in_script", + "deterministic_output_required", + "stable_sort_and_dedupe_collections", + "no_time_random_uuid_for_visible_output", + "match_http_results_by_request_id" + ], + "additionalProperties": false, + "properties": { + "guest_has_no_network": { "type": "boolean" }, + "host_performs_http": { "type": "boolean" }, + "host_applies_ssrf": { "type": "boolean" }, + "static_verifier_enforces_imports": { "type": "boolean" }, + "scheduler_timing_not_in_script": { "type": "boolean" }, + "deterministic_output_required": { "type": "boolean" }, + "stable_sort_and_dedupe_collections": { "type": "boolean" }, + "no_time_random_uuid_for_visible_output": { "type": "boolean" }, + "match_http_results_by_request_id": { "type": "boolean" } + } + }, + "plugin_factory": { + "type": "object", + "required": ["manifest", "builder", "review"], + "additionalProperties": false, + "properties": { + "manifest": { "$ref": "#/$defs/plugin_factory_manifest" }, + "builder": { "$ref": "#/$defs/plugin_factory_builder" }, + "review": { "$ref": "#/$defs/plugin_factory_review" } + } + }, + "plugin_factory_manifest": { + "type": "object", + "required": [ + "host_creates_manifest", + "do_not_return_manifest_json", + "agent_plugin_schema", + "entrypoint", + "plugin_id_allowed_chars", + "plugin_id_forbidden_chars", + "roles", + "connector_messaging_ops", + "secret_kinds", + "never_embed_credentials_in_go_source", + "host_lists_connectors_under_messaging" + ], + "additionalProperties": false, + "properties": { + "host_creates_manifest": { "type": "boolean" }, + "do_not_return_manifest_json": { "type": "boolean" }, + "agent_plugin_schema": { "type": "string" }, + "entrypoint": { "type": "string" }, + "plugin_id_allowed_chars": { "type": "string" }, + "plugin_id_forbidden_chars": { + "type": "array", + "items": { "type": "string" } + }, + "roles": { + "type": "array", + "items": { "type": "string" } + }, + "connector_messaging_ops": { + "type": "array", + "items": { "type": "string" } + }, + "secret_kinds": { + "type": "array", + "items": { "type": "string" } + }, + "never_embed_credentials_in_go_source": { "type": "boolean" }, + "host_lists_connectors_under_messaging": { "type": "boolean" } + } + }, + "plugin_factory_builder": { + "type": "object", + "required": [ + "response_keys", + "skill_files_path_pattern", + "empty_skill_files_when_unused", + "test_input_is_serialized_json_object", + "test_input_must_not_be_empty", + "test_input_exercises_terminal_result", + "connector_test_input_uses_hops_array", + "connector_parse_json_http_results_when_vendor_returns_json" + ], + "additionalProperties": false, + "properties": { + "response_keys": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "skill_files_path_pattern": { "type": "string" }, + "empty_skill_files_when_unused": { "type": "boolean" }, + "test_input_is_serialized_json_object": { "type": "boolean" }, + "test_input_must_not_be_empty": { "type": "boolean" }, + "test_input_exercises_terminal_result": { "type": "boolean" }, + "connector_test_input_uses_hops_array": { "type": "boolean" }, + "connector_parse_json_http_results_when_vendor_returns_json": { "type": "boolean" } + } + }, + "plugin_factory_review": { + "type": "object", + "required": [ + "compilation_success_not_approval", + "response_schema", + "reject_non_go_source", + "reject_raw_network_outside_http_request_envelopes", + "reject_missing_stdin_read", + "source_derived_titles_may_be_fragments", + "reject_unsupported_direct_test_claims", + "connector_rules_document" + ], + "additionalProperties": false, + "properties": { + "compilation_success_not_approval": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": ["decision", "summary", "findings"], + "additionalProperties": false, + "properties": { + "decision": { "type": "string" }, + "summary": { "type": "string" }, + "findings": { "type": "string" } + } + }, + "reject_non_go_source": { "type": "boolean" }, + "reject_raw_network_outside_http_request_envelopes": { "type": "boolean" }, + "reject_missing_stdin_read": { "type": "boolean" }, + "source_derived_titles_may_be_fragments": { "type": "boolean" }, + "reject_unsupported_direct_test_claims": { "type": "boolean" }, + "connector_rules_document": { "type": "string" } + } + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift b/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift new file mode 100644 index 0000000..83a308d --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractDocument.swift @@ -0,0 +1,250 @@ +import Foundation + +/// Canonical Derrick script_exec protocol decoded from `script-exec-contract.json`. +public struct ScriptExecContractDocument: Codable, Sendable, Hashable { + public var version: Int + public var runtime: ScriptExecRuntimeSpec + public var io: ScriptExecIOSpec + public var workflow: ScriptExecWorkflowSpec + public var output: ScriptExecOutputSpec + public var agent: ScriptExecAgentSpec + public var review: ScriptExecReviewSpec + public var rules: ScriptExecProtocolRules + public var pluginFactory: ScriptExecPluginFactorySpec + + enum CodingKeys: String, CodingKey { + case version, runtime, io, workflow, output, agent, review, rules + case pluginFactory = "plugin_factory" + } +} + +public struct ScriptExecRuntimeSpec: Codable, Sendable, Hashable { + public var language: String + public var package: String + public var binary: String + public var stdlibOnly: Bool + public var forbiddenImports: [String] + public var allowedOSUsage: [String] + public var exampleGo: String + + enum CodingKeys: String, CodingKey { + case language, package, binary + case stdlibOnly = "stdlib_only" + case forbiddenImports = "forbidden_imports" + case allowedOSUsage = "allowed_os_usage" + case exampleGo = "example_go" + } +} + +public struct ScriptExecIOSpec: Codable, Sendable, Hashable { + public var stdinSchema: String + public var stdoutSchema: String + public var guestRuntimeSchema: String + + enum CodingKeys: String, CodingKey { + case stdinSchema = "stdin_schema" + case stdoutSchema = "stdout_schema" + case guestRuntimeSchema = "guest_runtime_schema" + } +} + +public struct ScriptExecWorkflowSpec: Codable, Sendable, Hashable { + public var firstHopVerbs: [String] + public var httpResultsEventKind: String + public var terminalVerbs: [String] + public var matchHTTPBy: String + public var httpResultsAccumulate: Bool + public var postBodyField: String + + enum CodingKeys: String, CodingKey { + case firstHopVerbs = "first_hop_verbs" + case httpResultsEventKind = "http_results_event_kind" + case terminalVerbs = "terminal_verbs" + case matchHTTPBy = "match_http_by" + case httpResultsAccumulate = "http_results_accumulate" + case postBodyField = "post_body_field" + } +} + +public struct ScriptExecOutputSpec: Codable, Sendable, Hashable { + public var fields: [String: ScriptExecOutputFieldSpec] + public var forbiddenPatterns: [String] + public var rawBodyInContentRequiresExplicitUserRequest: Bool + + enum CodingKeys: String, CodingKey { + case fields + case forbiddenPatterns = "forbidden_patterns" + case rawBodyInContentRequiresExplicitUserRequest = + "raw_body_in_content_requires_explicit_user_request" + } +} + +public struct ScriptExecOutputFieldSpec: Codable, Sendable, Hashable { + public var purpose: String + public var hostStripsIncidentalMarkup: Bool? + public var hostAllowlistSanitizes: Bool? + public var preferContentForExtractedText: Bool? + + enum CodingKeys: String, CodingKey { + case purpose + case hostStripsIncidentalMarkup = "host_strips_incidental_markup" + case hostAllowlistSanitizes = "host_allowlist_sanitizes" + case preferContentForExtractedText = "prefer_content_for_extracted_text" + } +} + +public struct ScriptExecAgentSpec: Codable, Sendable, Hashable { + public var preferDirectURLsOverSERPHTML: Bool + public var retryWithDifferentURLsOnEmptyFetch: Bool + public var maxCorrectionAttemptsAfterBlock: Int + + enum CodingKeys: String, CodingKey { + case preferDirectURLsOverSERPHTML = "prefer_direct_urls_over_serp_html" + case retryWithDifferentURLsOnEmptyFetch = "retry_with_different_urls_on_empty_fetch" + case maxCorrectionAttemptsAfterBlock = "max_correction_attempts_after_block" + } +} + +public struct ScriptExecReviewSpec: Codable, Sendable, Hashable { + public var failFast: Bool + public var responseSchema: [String: String] + public var checks: [ScriptExecReviewCheck] + public var doNotDenyFor: [String] + public var enforcedElsewhere: [String] + + enum CodingKeys: String, CodingKey { + case failFast = "fail_fast" + case responseSchema = "response_schema" + case checks + case doNotDenyFor = "do_not_deny_for" + case enforcedElsewhere = "enforced_elsewhere" + } +} + +public struct ScriptExecReviewCheck: Codable, Sendable, Hashable { + public var id: String + public var order: Int + public var description: String + public var notes: [String]? +} + +public struct ScriptExecProtocolRules: Codable, Sendable, Hashable { + public var guestHasNoNetwork: Bool + public var hostPerformsHTTP: Bool + public var hostAppliesSSRF: Bool + public var staticVerifierEnforcesImports: Bool + public var schedulerTimingNotInScript: Bool + public var deterministicOutputRequired: Bool + public var stableSortAndDedupeCollections: Bool + public var noTimeRandomUUIDForVisibleOutput: Bool + public var matchHTTPResultsByRequestID: Bool + + enum CodingKeys: String, CodingKey { + case guestHasNoNetwork = "guest_has_no_network" + case hostPerformsHTTP = "host_performs_http" + case hostAppliesSSRF = "host_applies_ssrf" + case staticVerifierEnforcesImports = "static_verifier_enforces_imports" + case schedulerTimingNotInScript = "scheduler_timing_not_in_script" + case deterministicOutputRequired = "deterministic_output_required" + case stableSortAndDedupeCollections = "stable_sort_and_dedupe_collections" + case noTimeRandomUUIDForVisibleOutput = "no_time_random_uuid_for_visible_output" + case matchHTTPResultsByRequestID = "match_http_results_by_request_id" + } +} + +public struct ScriptExecPluginFactorySpec: Codable, Sendable, Hashable { + public var manifest: ScriptExecPluginFactoryManifestSpec + public var builder: ScriptExecPluginFactoryBuilderSpec + public var review: ScriptExecPluginFactoryReviewSpec +} + +public struct ScriptExecPluginFactoryManifestSpec: Codable, Sendable, Hashable { + public var hostCreatesManifest: Bool + public var doNotReturnManifestJSON: Bool + public var agentPluginSchema: String + public var entrypoint: String + public var pluginIDAllowedChars: String + public var pluginIDForbiddenChars: [String] + public var roles: [String] + public var connectorMessagingOps: [String] + public var secretKinds: [String] + public var neverEmbedCredentialsInGoSource: Bool + public var hostListsConnectorsUnderMessaging: Bool + + enum CodingKeys: String, CodingKey { + case hostCreatesManifest = "host_creates_manifest" + case doNotReturnManifestJSON = "do_not_return_manifest_json" + case agentPluginSchema = "agent_plugin_schema" + case entrypoint + case pluginIDAllowedChars = "plugin_id_allowed_chars" + case pluginIDForbiddenChars = "plugin_id_forbidden_chars" + case roles + case connectorMessagingOps = "connector_messaging_ops" + case secretKinds = "secret_kinds" + case neverEmbedCredentialsInGoSource = "never_embed_credentials_in_go_source" + case hostListsConnectorsUnderMessaging = "host_lists_connectors_under_messaging" + } +} + +public struct ScriptExecPluginFactoryBuilderSpec: Codable, Sendable, Hashable { + public var responseKeys: [String] + public var skillFilesPathPattern: String + public var emptySkillFilesWhenUnused: Bool + public var testInputIsSerializedJSONObject: Bool + public var testInputMustNotBeEmpty: Bool + public var testInputExercisesTerminalResult: Bool + public var connectorTestInputUsesHopsArray: Bool + public var connectorParseJSONHTTPResultsWhenVendorReturnsJSON: Bool + + enum CodingKeys: String, CodingKey { + case responseKeys = "response_keys" + case skillFilesPathPattern = "skill_files_path_pattern" + case emptySkillFilesWhenUnused = "empty_skill_files_when_unused" + case testInputIsSerializedJSONObject = "test_input_is_serialized_json_object" + case testInputMustNotBeEmpty = "test_input_must_not_be_empty" + case testInputExercisesTerminalResult = "test_input_exercises_terminal_result" + case connectorTestInputUsesHopsArray = "connector_test_input_uses_hops_array" + case connectorParseJSONHTTPResultsWhenVendorReturnsJSON = + "connector_parse_json_http_results_when_vendor_returns_json" + } +} + +public struct ScriptExecPluginFactoryReviewSpec: Codable, Sendable, Hashable { + public var compilationSuccessNotApproval: Bool + public var responseSchema: [String: String] + public var rejectNonGoSource: Bool + public var rejectRawNetworkOutsideHTTPRequestEnvelopes: Bool + public var rejectMissingStdinRead: Bool + public var sourceDerivedTitlesMayBeFragments: Bool + public var rejectUnsupportedDirectTestClaims: Bool + public var connectorRulesDocument: String + + enum CodingKeys: String, CodingKey { + case compilationSuccessNotApproval = "compilation_success_not_approval" + case responseSchema = "response_schema" + case rejectNonGoSource = "reject_non_go_source" + case rejectRawNetworkOutsideHTTPRequestEnvelopes = + "reject_raw_network_outside_http_request_envelopes" + case rejectMissingStdinRead = "reject_missing_stdin_read" + case sourceDerivedTitlesMayBeFragments = "source_derived_titles_may_be_fragments" + case rejectUnsupportedDirectTestClaims = "reject_unsupported_direct_test_claims" + case connectorRulesDocument = "connector_rules_document" + } +} + +public enum ScriptExecContractError: Error, Equatable, LocalizedError, Sendable { + case missingResource(String) + case invalidJSON(String) + case integrityFailed(String) + + public var errorDescription: String? { + switch self { + case .missingResource(let name): + return "Missing bundled script_exec contract resource \(name)." + case .invalidJSON(let name): + return "Script_exec contract resource \(name) is not valid JSON." + case .integrityFailed(let detail): + return "Script_exec contract JSON failed schema checks: \(detail)" + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift b/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift new file mode 100644 index 0000000..81face7 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractIntegrity.swift @@ -0,0 +1,93 @@ +import Foundation + +/// Host checks that the bundled script_exec JSON still matches its schema and wire schemas. +public enum ScriptExecContractIntegrity: Sendable { + public static func validateBundledGraph() throws { + let contractData = try ScriptExecContractStore.resourceData( + name: ScriptExecContractStore.protocolResource, + subdirectory: "contracts" + ) + let contractJSON: [String: Any] + do { + contractJSON = try JSONSchema.object( + from: contractData, + name: ScriptExecContractStore.protocolResource + ) + } catch { + throw ScriptExecContractError.invalidJSON(ScriptExecContractStore.protocolResource) + } + try validateAgainstSchema(contractJSON, schema: .scriptExecContract) + try validateWireSchemaRefs(contractJSON) + try validateOutputFields(contractJSON) + try validateReviewChecks(contractJSON) + } + + private static func validateAgainstSchema(_ instance: Any, schema: GuestContract.Schema) throws { + do { + try GuestContract.validate(instance, against: schema) + } catch let error as GuestContractError { + throw ScriptExecContractError.integrityFailed(error.localizedDescription) + } + } + + private static func validateWireSchemaRefs(_ contract: [String: Any]) throws { + guard let io = contract["io"] as? [String: Any] else { return } + for key in ["stdin_schema", "stdout_schema", "guest_runtime_schema"] { + guard let fileName = io[key] as? String, + GuestContract.Schema(rawValue: fileName) != nil else { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json io.\(key) must reference a bundled guest schema." + ) + } + } + } + + private static func validateOutputFields(_ contract: [String: Any]) throws { + let envelope = try GuestContract.loadSchemaObject(.envelopeList) + guard let properties = (envelope["items"] as? [String: Any])?["properties"] as? [String: Any] else { + throw ScriptExecContractError.integrityFailed( + "envelope-list.schema.json is missing items.properties." + ) + } + let envelopeKeys = Set(properties.keys) + guard let output = contract["output"] as? [String: Any], + let fields = output["fields"] as? [String: Any] else { + return + } + let missing = Set(fields.keys).subtracting(envelopeKeys) + if !missing.isEmpty { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json output.fields references unknown envelope fields: \(missing.sorted().joined(separator: ", "))." + ) + } + } + + private static func validateReviewChecks(_ contract: [String: Any]) throws { + guard let review = contract["review"] as? [String: Any], + let checks = review["checks"] as? [[String: Any]] else { + return + } + var seenIDs: Set = [] + var seenOrders: Set = [] + for check in checks { + guard let id = check["id"] as? String, + let order = check["order"] as? Int else { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks entries require id and order." + ) + } + if seenIDs.contains(id) { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks has duplicate id \(id)." + ) + } + if seenOrders.contains(order) { + throw ScriptExecContractError.integrityFailed( + "script-exec-contract.json review.checks has duplicate order \(order)." + ) + } + seenIDs.insert(id) + seenOrders.insert(order) + } + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift b/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift new file mode 100644 index 0000000..82278b3 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractPrompts.swift @@ -0,0 +1,56 @@ +import Foundation + +/// Renders the bundled script_exec protocol for agent, builder, and reviewer prompts. +public enum ScriptExecContractPrompts: Sendable { + public static func builderGuide() -> String { + dumpOrUnavailable(preamble: """ + Offline Go guest rules come only from this protocol JSON and the wire schemas below. \ + Do not invent requirements that are not in the JSON. + """) + } + + public static func reviewerGuide() -> String { + dumpOrUnavailable(preamble: """ + You are a reviewer for script_exec declarations. Review against script-exec-contract.json only. \ + If a rule is not in the JSON, do not require it. Apply review.checks in order; when review.fail_fast \ + is true, return on the first failing check. Return only valid JSON matching review.response_schema. + """) + } + + public static func pluginFactoryBuilderGuide() -> String { + dumpOrUnavailable(preamble: """ + Plugin factory builder rules come only from script-exec-contract.json (plugin_factory, runtime, \ + workflow, output, rules) and connector-contract.json when building a connector. Do not invent \ + requirements that are not in those JSON documents. + """) + } + + public static func pluginFactoryReviewerGuide() -> String { + dumpOrUnavailable(preamble: """ + You are Derrick's independent plugin alignment and safety reviewer. Review guest go_source against \ + script-exec-contract.json (runtime, workflow, output, rules, plugin_factory.review). If a guest rule \ + is not in the JSON, do not require it. For connector plugins also apply connector-contract.json below. \ + Return exactly one JSON object matching plugin_factory.review.response_schema. + """) + } + + public static func dump(preamble: String) throws -> String { + _ = try ScriptExecContractStore.loadProtocol() + var lines: [String] = [preamble, "", "--- script-exec-contract.json ---"] + lines.append(try ScriptExecContractStore.loadProtocolText()) + lines.append("") + lines.append("--- \(GuestContract.Schema.guestRuntime.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.guestRuntime)) + lines.append("") + lines.append("--- \(GuestContract.Schema.hopEvent.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.hopEvent)) + lines.append("") + lines.append("--- \(GuestContract.Schema.envelopeList.rawValue) ---") + lines.append(try GuestContract.loadSchemaText(.envelopeList)) + return lines.joined(separator: "\n") + } + + private static func dumpOrUnavailable(preamble: String) -> String { + (try? dump(preamble: preamble)) ?? "Script_exec contract JSON failed to load." + } +} diff --git a/packages/Structure/Sources/Contract/ScriptExecContractStore.swift b/packages/Structure/Sources/Contract/ScriptExecContractStore.swift new file mode 100644 index 0000000..063fe59 --- /dev/null +++ b/packages/Structure/Sources/Contract/ScriptExecContractStore.swift @@ -0,0 +1,64 @@ +import CryptoKit +import Foundation + +/// Loads the bundled script_exec protocol JSON. +public enum ScriptExecContractStore: Sendable { + public static let protocolResource = "script-exec-contract.json" + public static let fingerprintEntries: [(subdirectory: String, file: String)] = [ + ("schemas", "script-exec-contract.schema.json"), + ("schemas", "guest-runtime.schema.json"), + ("schemas", "hop-event.schema.json"), + ("schemas", "envelope-list.schema.json"), + ("contracts", "script-exec-contract.json"), + ] + + public static var fingerprintSources: [String] { + fingerprintEntries.map { "\($0.subdirectory)/\($0.file)" } + } + + public static func loadProtocol() throws -> ScriptExecContractDocument { + try ScriptExecContractIntegrity.validateBundledGraph() + let data = try resourceData(name: protocolResource, subdirectory: "contracts") + do { + return try JSONDecoder().decode(ScriptExecContractDocument.self, from: data) + } catch { + throw ScriptExecContractError.invalidJSON(protocolResource) + } + } + + public static func loadProtocolText() throws -> String { + try utf8Text(resourceData(name: protocolResource, subdirectory: "contracts")) + } + + public static func computeFingerprint() throws -> String { + var joined = Data() + for entry in fingerprintEntries { + let relative = "\(entry.subdirectory)/\(entry.file)" + let data = try resourceData(name: entry.file, subdirectory: entry.subdirectory) + joined.append(Data(relative.utf8)) + joined.append(0) + joined.append(data) + joined.append(0) + } + let digest = SHA256.hash(data: joined) + return digest.map { String(format: "%02x", $0) }.joined() + } + + static func resourceData(name: String, subdirectory: String) throws -> Data { + guard let url = Bundle.module.url( + forResource: name, + withExtension: nil, + subdirectory: subdirectory + ) else { + throw ScriptExecContractError.missingResource("\(subdirectory)/\(name)") + } + return try Data(contentsOf: url) + } + + private static func utf8Text(_ data: Data) throws -> String { + guard let text = String(data: data, encoding: .utf8) else { + throw ScriptExecContractError.invalidJSON(protocolResource) + } + return text + } +} diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift index dca7e12..7a8f1f3 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift @@ -15,7 +15,7 @@ public enum DockerWorkerRuntime: Sendable { /// Allowed `docker exec … sh -c` payloads for guest source I/O and in-container compile. public static let guestWriteSourceShell = "cat > /tmp/plugin.go" public static let guestCompileShell = - "cd /tmp && /usr/local/go/bin/go build -trimpath -ldflags=-s -w -o guest plugin.go && chmod +x guest" + "cd /tmp && /usr/local/go/bin/go build -trimpath -ldflags=\"-s -w\" -o guest plugin.go && chmod +x guest" public static let guestReadBinaryShell = "cat /tmp/guest" public static let pinnedDigest = DockerProductImageDigests.worker diff --git a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift index 175d314..033bd9e 100644 --- a/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift +++ b/packages/Structure/Sources/Plugin/Envelope/DerrickGuestGo.swift @@ -2,64 +2,8 @@ import Foundation /// Go contract shown to models that generate Derrick plugin guest programs. public enum DerrickGuestGo: Sendable { - public static let modelContract = """ - Go guest contract: - - The program is a standalone `package main` built to a Linux binary and run as `/tmp/guest`. - - Read one JSON object from standard input matching the hop-event schema. - - Write one JSON array of envelope objects to standard output matching the envelope-list schema. - - Every envelope object needs `verb` from the envelope-list schema. - - POST bodies go in `json`. The host decodes `http.request` as HostHTTPRequest and sends `json` as the HTTP body. - - On the first event, emit `http.request` envelopes for host HTTP. - - On an `http_results` event, emit `result.emit` or `message.post`. - - The host, not the guest container, performs HTTP and supplies response bodies. - - Do not import net/http, net, os/exec, os (except os.Stdin/os.Stdout), or perform filesystem access. - - Use only the Go standard library. - - For repeatable output, match HTTP responses by request_id, sort and de-duplicate collections - by stable keys, and never use current time, randomness, UUIDs, response arrival order, or - map iteration order for user-visible output. - - Later http_results events include earlier responses plus the newest ones. Match by request_id. - - Minimal output pattern: - ```go - package main - - import ( - "encoding/json" - "os" - ) - - func main() { - var event map[string]any - if err := json.NewDecoder(os.Stdin).Decode(&event); err != nil { - return - } - emit([]map[string]any{{"verb": "result.emit", "title": "Result", "summary": "done"}}) - } - - func emit(envelopes []map[string]any) { - enc := json.NewEncoder(os.Stdout) - enc.SetEscapeHTML(false) - _ = enc.Encode(envelopes) - } - ``` - Inspect `event["kind"]` and `event["http_results"]` to choose the next envelopes. - """ - public static func source(for spec: PluginSpec? = nil) throws -> String { - var sections = [modelContract] - sections.append( - """ - Canonical JSON schemas (Swift host and Go guest must match exactly): - --- \(GuestContract.Schema.guestRuntime.rawValue) --- - \(try GuestContract.loadSchemaText(.guestRuntime)) - - --- \(GuestContract.Schema.hopEvent.rawValue) --- - \(try GuestContract.loadSchemaText(.hopEvent)) - - --- \(GuestContract.Schema.envelopeList.rawValue) --- - \(try GuestContract.loadSchemaText(.envelopeList)) - """ - ) + var sections = [ScriptExecContractPrompts.builderGuide()] if let spec { sections.append( """ diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index e594bbf..f79aa32 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -84,11 +84,11 @@ import Testing #expect(decoded.executableFingerprint == nil) } - @Test func bundledScriptReviewerInstructionsLoadFromSourceTree() throws { - let scriptReviewer = try DerrickBundledText.load("script_reviewer_instructions.md") - #expect(scriptReviewer.contains("intent alignment")) - #expect(scriptReviewer.contains("secret literals")) - #expect(scriptReviewer.contains("Go verifier")) + @Test func scriptExecReviewerPromptLoadsFromBundledContract() { + let scriptReviewer = ScriptExecContractPrompts.reviewerGuide() + #expect(scriptReviewer.contains("script-exec-contract.json")) + #expect(scriptReviewer.contains("intent_alignment")) + #expect(scriptReviewer.contains("If a rule is not in the JSON")) } @Test func healthDecodesLegacyPayloadWithoutGuestRuntime() throws { diff --git a/packages/Structure/Tests/StructureTests/GuestContractTests.swift b/packages/Structure/Tests/StructureTests/GuestContractTests.swift index 430d134..58d1a06 100644 --- a/packages/Structure/Tests/StructureTests/GuestContractTests.swift +++ b/packages/Structure/Tests/StructureTests/GuestContractTests.swift @@ -142,4 +142,5 @@ import Testing try GuestContractValidation.validateFileExtractorResultJSON(Data(json.utf8)) } } + } diff --git a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift index 7b0f12a..56235d1 100644 --- a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift +++ b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift @@ -129,6 +129,25 @@ import Structure #expect(already.path.contains("rss")) } + @Test func googleNewsTopicPageMapsToSectionRSSWhenHintMentionsTech() { + let topic = URL( + string: "https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB" + )! + let mapped = NewsSourceURL.canonicalFetchURL( + topic, + contextHint: "today's technology headlines" + ) + #expect(mapped.path == "/rss/headlines/section/topic/TECHNOLOGY") + } + + @Test func googleNewsTopicPageFallsBackToGeneralRSSWithoutHint() { + let topic = URL( + string: "https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB" + )! + let mapped = NewsSourceURL.canonicalFetchURL(topic) + #expect(mapped.path == "/rss") + } + @Test func refreshParsesGoogleNewsHomepageViaRSS() async throws { let rss = """ diff --git a/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift b/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift new file mode 100644 index 0000000..c71db5c --- /dev/null +++ b/packages/Structure/Tests/StructureTests/ScriptExecContractTests.swift @@ -0,0 +1,50 @@ +import Foundation +import Structure +import Testing + +@Suite struct ScriptExecContractTests { + @Test func protocolJSONLoadsAndMatchesRules() throws { + let document = try ScriptExecContractStore.loadProtocol() + #expect(document.version == 1) + #expect(document.runtime.language == "go") + #expect(document.review.failFast) + #expect(document.review.checks.count == 4) + #expect(document.rules.guestHasNoNetwork) + #expect(document.output.fields["content"]?.purpose.contains("plain-text") == true) + #expect(document.pluginFactory.manifest.hostCreatesManifest) + #expect(document.pluginFactory.builder.connectorTestInputUsesHopsArray) + #expect(document.pluginFactory.review.compilationSuccessNotApproval) + } + + @Test func pluginFactoryPromptsReferenceContractJSON() { + let builder = ScriptExecContractPrompts.pluginFactoryBuilderGuide() + let reviewer = ScriptExecContractPrompts.pluginFactoryReviewerGuide() + #expect(builder.contains("plugin_factory")) + #expect(builder.contains("--- script-exec-contract.json ---")) + #expect(reviewer.contains("plugin_factory.review")) + #expect(reviewer.contains("If a guest rule is not in the JSON")) + } + + @Test func fingerprintMatchesGeneratedFile() throws { + #expect(try ScriptExecContractStore.computeFingerprint() == ScriptExecContractFingerprint.sha256) + #expect(ScriptExecContractFingerprint.sourceFiles == ScriptExecContractStore.fingerprintSources) + } + + @Test func bundledContractSatisfiesItsSchema() throws { + try ScriptExecContractIntegrity.validateBundledGraph() + } + + @Test func reviewerGuideDumpsCanonicalJSON() throws { + let guide = ScriptExecContractPrompts.reviewerGuide() + #expect(guide.contains("If a rule is not in the JSON")) + #expect(guide.contains("--- script-exec-contract.json ---")) + #expect(guide.contains(try ScriptExecContractStore.loadProtocolText())) + #expect(guide.contains("--- \(GuestContract.Schema.envelopeList.rawValue) ---")) + } + + @Test func builderGuideDumpsWireSchemas() throws { + let guide = ScriptExecContractPrompts.builderGuide() + #expect(guide.contains("--- \(GuestContract.Schema.hopEvent.rawValue) ---")) + #expect(guide.contains("--- \(GuestContract.Schema.guestRuntime.rawValue) ---")) + } +} diff --git a/readme.md b/readme.md index dbf7d9f..b8b1262 100644 --- a/readme.md +++ b/readme.md @@ -67,7 +67,7 @@ Before `script_exec` writes to disk, a **configured LLM reviewer** checks: - No secret literals in source - Safe handling of fetched content (no raw HTML leakage unless requested) -Instructions live in `ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md`. A static **Go verifier** also rejects forbidden APIs. +Rules live in `packages/Structure/Sources/Contract/Resources/contracts/script-exec-contract.json` (same pattern as connector plugins). A static **Go verifier** also rejects forbidden APIs. ### Egress & network diff --git a/scripts/generate-script-exec-contract.swift b/scripts/generate-script-exec-contract.swift new file mode 100755 index 0000000..4c1995c --- /dev/null +++ b/scripts/generate-script-exec-contract.swift @@ -0,0 +1,121 @@ +#!/usr/bin/env swift +import CryptoKit +import Foundation + +/// Regenerates `ScriptExecContract.generated.swift` from bundled script_exec JSON. +/// `--check` exits 1 when the committed fingerprint does not match the JSON files +/// or when the JSON no longer satisfies the bundled schemas. + +let repoRoot = URL(fileURLWithPath: CommandLine.arguments[0]) + .resolvingSymlinksInPath() + .deletingLastPathComponent() + .deletingLastPathComponent() + +let resources = repoRoot + .appendingPathComponent("packages/Structure/Sources/Contract/Resources") + +let sources: [(relative: String, url: URL)] = [ + "schemas/script-exec-contract.schema.json", + "schemas/guest-runtime.schema.json", + "schemas/hop-event.schema.json", + "schemas/envelope-list.schema.json", + "contracts/script-exec-contract.json", +].map { relative in + (relative, resources.appendingPathComponent(relative)) +} + +let generatedURL = repoRoot + .appendingPathComponent("packages/Structure/Sources/Contract/Generated/ScriptExecContract.generated.swift") + +func jsonObject(_ url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + guard let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ScriptError("\(url.lastPathComponent) is not a JSON object.") + } + return object +} + +func requireKeys(_ object: [String: Any], _ keys: [String], file: String) throws { + for key in keys where object[key] == nil { + throw ScriptError("\(file) is missing required key \(key).") + } +} + +func validateGraph() throws { + let contractSchema = try jsonObject(resources.appendingPathComponent("schemas/script-exec-contract.schema.json")) + let contract = try jsonObject(resources.appendingPathComponent("contracts/script-exec-contract.json")) + try requireKeys(contract, contractSchema["required"] as? [String] ?? [], file: "script-exec-contract.json") + let rulesSchema = (contractSchema["properties"] as? [String: Any])?["rules"] as? [String: Any] + try requireKeys( + contract["rules"] as? [String: Any] ?? [:], + rulesSchema?["required"] as? [String] ?? [], + file: "script-exec-contract.json rules" + ) + let checks = (contract["review"] as? [String: Any])?["checks"] as? [[String: Any]] ?? [] + if checks.isEmpty { + throw ScriptError("script-exec-contract.json review.checks must not be empty.") + } + let pluginFactorySchema = (contractSchema["properties"] as? [String: Any])?["plugin_factory"] as? [String: Any] + try requireKeys( + contract["plugin_factory"] as? [String: Any] ?? [:], + pluginFactorySchema?["required"] as? [String] ?? [], + file: "script-exec-contract.json plugin_factory" + ) +} + +func fingerprint() throws -> String { + var joined = Data() + for source in sources { + let data = try Data(contentsOf: source.url) + joined.append(Data(source.relative.utf8)) + joined.append(0) + joined.append(data) + joined.append(0) + } + return SHA256.hash(data: joined).map { String(format: "%02x", $0) }.joined() +} + +func generatedSource(hash: String) -> String { + """ + // Automatically generated by scripts/generate-script-exec-contract.swift. DO NOT EDIT. + + /// SHA-256 of script_exec protocol JSON and schemas. `swift test` fails when this is stale. + public enum ScriptExecContractFingerprint: Sendable { + public static let sha256 = "\(hash)" + public static let sourceFiles: [String] = [ + \(sources.map { " \"\($0.relative)\"," }.joined(separator: "\n")) + ] + } + + """ +} + +struct ScriptError: Error, CustomStringConvertible { + let description: String + init(_ description: String) { self.description = description } +} + +do { + try validateGraph() +} catch { + fputs("\(error)\n", stderr) + exit(1) +} + +let hash = try fingerprint() +let check = CommandLine.arguments.contains("--check") +if check { + let existing = try String(contentsOf: generatedURL, encoding: .utf8) + if !existing.contains("public static let sha256 = \"\(hash)\"") { + fputs("script_exec contract fingerprint is stale. Run scripts/generate-script-exec-contract.swift\n", stderr) + exit(1) + } + exit(0) +} + +try FileManager.default.createDirectory( + at: generatedURL.deletingLastPathComponent(), + withIntermediateDirectories: true +) +try generatedSource(hash: hash).write(to: generatedURL, atomically: true, encoding: .utf8) +print("Wrote \(generatedURL.path) sha256=\(hash)") diff --git a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift index de55fe9..8a658c8 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationPipeline.swift @@ -122,11 +122,6 @@ struct ConversationPipeline: Sen if !toolInstructions.isEmpty { sections.append(toolInstructions) } - if !toolInstructions.contains("Go guest contract:") { - if let sdk = try? PromptResources.guestSDKForModel() { - sections.append(sdk) - } - } return sections.joined(separator: "\n\n") } diff --git a/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift b/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift index 8c8c64b..406d88b 100644 --- a/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift +++ b/ui/SharedAgentRuntime/Conversation/ConversationPipelinePolicy.swift @@ -877,7 +877,7 @@ extension ConversationPipeline { private static func longRunningToolProgressMessage(for toolName: String) -> String? { switch toolName { case AllowedMCPTool.webCrawl.rawValue: - return "Crawling vendor documentation. This may take a few minutes…" + return "Fetching web content. This may take a few minutes…" case AllowedMCPTool.pluginFactoryBuild.rawValue: return "Building the plugin (code generation, Docker tests, and safety review). This may take several minutes…" default: diff --git a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md index 2e186cf..7b381d6 100644 --- a/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md +++ b/ui/SharedAgentRuntime/Resources/conversation_rag_instructions.md @@ -13,26 +13,14 @@ Use a tool when the user asks for any of: - search, look up, browse, fetch, scrape, or “from the web / online” - site-specific retail or catalog data (e.g. Amazon, “best sellers”, “top 10 … being sold”, prices, availability) -Use `script_exec` for scripting, automation, and live web access through the host HTTP bridge. - For those requests: 1. Prefer calling the tool **on the first turn** with reasonable defaults. 2. Do **not** answer with only clarifying questions when a sensible default exists (e.g. US site, general category, bestseller or top search results). State the default you used in the final answer after the tool runs. 3. Ask a clarifying question only when the request is impossible to execute without a critical missing fact (not for optional polish). 4. Never invent live rankings, prices, stock, market moves, or “what’s selling now” from training data. -5. For current events / market turmoil: fetch real articles from news or finance sites (not Google search results pages). If a scrape returns no usable content, retry with other sites before concluding data is unavailable. - -## Response format - -Always respond using the required JSON schema (`thinking` / `tool_call` / `tool_batch` / `complete`). Never reply as free-form plain text outside that schema. When presenting a list of choices, options, steps, items, or alternative paths to the user, ALWAYS format them as a clean Markdown bulleted list (using `-` or `*`) or a numbered list (using `1.`, `2.`), instead of writing them as plain paragraphs. -## Website crawling (`web.crawl`) - -- Call `web.crawl` **directly** in live chat for typical same-origin crawls; wait for the result in the same turn. -- Use `jobs_create` with `tool_name` `web.crawl` only when you judge the crawl is likely to take **more than about one minute** (large `max_pages`, deep `max_depth`, or long `timeout_seconds`). Set `wake_after: true` and a short `wake_prompt` so the user gets a notification when it finishes. - ## Scheduled jobs (`jobs_create`) - `wake_after` (default **true**): when **true**, the agent is woken after a **successful** run to summarize the tool result and the user gets a notification + result panel. When **false**, success is silent (no wake, no notification). diff --git a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md index 6fdc036..1175324 100644 --- a/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md +++ b/ui/SharedAgentRuntime/Resources/mcp_tool_instructions.md @@ -9,31 +9,9 @@ - Set `status` to "complete" when you have finished and are responding directly to the user, and populate the `assistant_response` field with your Markdown reply. - Pass tool `arguments` as a **stringified JSON object** under the `arguments` key (schema requirement). Prefer short Go source and avoid embedding unescaped double quotes in the script body. 6. Users should not have to name tools. Choose tools autonomously from intent. -7. Use `files.extract` for attached PDFs, Office documents, HTML, CSV, and Excel. Use `script_exec` for other scripting/automation. Use `web.crawl` for live website access. - 1. For `script_exec`, use standalone **Go** (`package main`) only. Read one hop-event JSON object from standard input and write an envelope-list JSON **array** to standard output. Do not import net/http, net, os/exec, or perform filesystem access. No third-party modules. - 2. The container has no network. Emit `http.request` envelopes; the host performs HTTP and invokes the guest program again with an `http_results` event. - 3. On the first hop emit `{"verb":"http.request","request_id":"…","method":"GET","url":"…"}`. On `http_results`, parse the supplied UTF-8 body and emit `result.emit` or `message.post`. - 4. The script must complete the user's requested extraction or summary, not only prove that a fetch happened. Never emit `repr(http_results)` or copy an entire fetched body into `content` unless the user explicitly requested the raw source. For HTML/XML, remove scripts and styles, extract the relevant visible fields, normalize the text, and cap the result. If raw HTML is explicitly requested, emit it in `html`; the host sanitizes that field. - 5. Prefer content sites. Do **not** scrape Google/Bing/Yahoo SERP HTML. - 6. Keep scripts short. Use `timeout_seconds` on the tool args if needed. - 7. If the first fetch is empty, try another `script_exec` with different URLs before answering. - 8. If `script_exec` returns `blocked` or `failed` with implementation findings, treat those - findings as correction feedback and make at most one corrected `script_exec` call before - answering. Do not repeat the same script unchanged. If the reviewer identifies a security - refusal or the corrected call also fails, report the exact finding instead of claiming success. -9. Use `web.crawl` for website crawling instead of generating a crawler script. Call it directly - in live chat so the result returns in the same turn. Only submit a crawl through `jobs_create` - when you judge it is likely to take more than about one minute (for example a large `max_pages`, - deep `max_depth`, many start URLs, or a long `timeout_seconds`). For background crawls use - `tool_name` `web.crawl`, `wake_after` true, and a short `wake_prompt`; tell the user the crawl - was submitted and that the result will arrive in a notification banner. Never request more than - 900 seconds. -10. A web crawl goal must describe the requested result. Never use it for DDoS, flooding, - load/stress testing, port scanning, brute force, or other high-volume behavior. Keep the - crawl same-origin and rely on the tool's page, depth, byte, rate, and timeout limits. -11. Use `files.extract` for attached files instead of generating an extractor script. Call it directly; do not submit it through `jobs_create`. Omit `filenames` to process every attached file in this chat. Never request more than 180 seconds. -12. After tool execution, respond with clean user-facing output only (Markdown/JSON/CSV as requested); do not include raw tool-call JSON, escaped script source, or internal control payloads. -13. Multi-agent tools (when listed in the catalog): +7. Use `files.extract` for attached files, `script_exec` for other scripting/automation, and `web.crawl` for live website access. Obey the bundled guest SDK (`script-exec-contract.json`) for `script_exec`; obey the web crawler and file extractor skills for those tools. +8. After tool execution, respond with clean user-facing output only (Markdown/JSON/CSV as requested); do not include raw tool-call JSON, escaped script source, or internal control payloads. +9. Multi-agent tools (when listed in the catalog): 1. If the user names a multi-agent tool or asks to spawn/list/send/cancel agents, issue that `tool_call` (or `tool_batch`) **before** any `complete` answer. Do not invent tool results. 2. `agents_spawn` — required args: `goal` (short), `task` (concrete). Blocks until the worker finishes; use the returned `result` in your next step. Optional `agent_id` slug. 3. Workers never talk to the user; you synthesize worker results into the final `assistant_response`. diff --git a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md b/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md deleted file mode 100644 index 10b3640..0000000 --- a/ui/SharedAgentRuntime/Resources/script_reviewer_instructions.md +++ /dev/null @@ -1,37 +0,0 @@ -You are a reviewer for script_exec declarations. - -FAIL-FAST (mandatory): -- Apply checks in order. As soon as ANY single check fails, return with failure JSON immediately. -- Do NOT continue scanning for more issues after the first failure. -- On first failure: return suggestedAction "deny", alignedWithRequest false, concerns with only that one failing reason, and a short summary. No essays. -- Only if every check passes: return suggestedAction "allow" with a brief summary (1-2 sentences). concerns may be empty or at most one minor operational note. - -Checks (if any fail return failure JSON immediately): -1) Script, description, reason, and user prompt are consistent (intent alignment). - Derrick has a job scheduler (jobs_create / run_after_seconds / cron). Timing is applied by - JobService before this script runs. The script must do the work immediately when invoked. - Words like delayed, scheduled, in 7 seconds, later, or run_after in the reason or user_prompt - refer to that scheduler — not to sleep/setTimeout inside the script. Do not deny a script - that performs the requested work (e.g. netFetch the URL) just because it has no delay. -2) No tokens, API keys, passwords, or other secret literals in the source. - -3) The script implements the requested terminal result, not just the fetch. - For requests to summarize, list, inspect, or extract fetched content, the `http_results` branch - must parse the relevant response body and emit the requested data. Do not allow - `repr(http_results)`, a fetch-only confirmation, or an entire raw body copied to - `content` unless the user explicitly requested the raw source. If raw HTML is requested, `html` - is allowed because the host sanitizes it before rendering. -4) Fetched HTML/XML is treated as untrusted data. Text and Markdown results must remove unsafe - markup and validate generated links as http or https. Do not reject ordinary safe HTML in the - `html` field solely because the host performs the final sanitization. - -Do not deny for Go style, envelope construction, destination URLs, or the absence of dependencies. The static Go verifier enforces direct network and process restrictions. The guest has no network; the host performs HTTP and applies SSRF there. - -Return only valid JSON with this exact schema: -{ - "alignedWithRequest": true|false, - "confidence": 0.0-1.0, - "suggestedAction": "allow"|"deny", - "concerns": ["..."], - "summary": "short explanation" -} diff --git a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift index 9153314..225f429 100644 --- a/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift +++ b/ui/SharedAgentRuntime/Support/PluginFactoryModels.swift @@ -131,49 +131,8 @@ actor ConfiguredPluginFactoryBuilder: PluginFactoryBuilder { private static func builderSystemPrompt(for userGoal: String) -> String { """ You are the Derrick plugin builder. Convert the user's goal into one complete Agent Plugin draft. - Return exactly one JSON object with these keys: - plugin_id (string), version (string), description (string), go_source (string), - test_input_json (string containing valid JSON — a serialized object, not prose), - skill_files (array of objects with path and body), - secrets (array of objects with id, label, and kind; required for connector plugins), - role (string, optional: "connector" or "standard"). - plugin_id must use lowercase letters, numbers, hyphens, and dots only - (for example my-connector). Never use underscores in plugin_id. - If the plugin needs a username, password, token, or API key, declare them in secrets. - kind must be username, password, token, or api_key. id is a stable Keychain key - such as username or bot_token. label is the text shown when the user saves the value. - Never put real credentials in go_source. - Set role to "connector" when the plugin sends and receives messages with an external - messaging service (any chat or mail connector). Omit role or use "standard" otherwise. - For role connector, include messaging_ops: an array of implemented ops - (send_message, poll_inbox, sync_threads). It must match the user goal scope and test_input_json. - The host writes messaging_ops into extensions.app.derrick in plugin.json. - The host lists connector plugins under Messaging. Do not guess this from the plugin_id. - Do not return manifest_json. The host creates the canonical Agent Plugin manifest, - including the exact `$schema` field for Agent Plugin 1.0 and the fixed - extensions.app.derrick.entrypoint ./app.derrick/plugin.go. - \(DerrickGuestGo.modelContract) + \(ScriptExecContractPrompts.pluginFactoryBuilderGuide()) \(ConnectorContractPrompts.builderGuide(forUserGoal: userGoal)) - Before returning the draft, self-check the implementation: - - Sort every returned collection by an explicit stable key after parsing and de-duplicate it. - - Match host responses by the emitted request_id. - - Use only the Go standard library (no third-party modules, raw sockets, or subprocess). - - The direct test input must exercise the terminal result path with matching http_results fixtures. - If skill_files is not needed, return an empty array. Every skill file path must be exactly - skills//SKILL.md. - For messaging connector plugins (role connector) that call a vendor HTTP API: - - Declare secrets in the manifest only. Never hard-code credentials. - - Parse each http_results body as JSON when the vendor returns JSON. - - test_input_json http_results must exercise success paths for every messaging_op in scope. - - test_input_json must be a single JSON object serialized as a string (valid JSON.parse input). - - test_input_json must not be empty or "{}". - - For connector plugins, test_input_json must use a hops array: - {"hops":[{"kind":"message_in_room","params":{"messaging_op":"send_message",...}},\ - {"kind":"http_results","http_results":[{"request_id":"...","status":200,"body":"..."}],\ - "params":{...}}]} - Repeat additional hop pairs for each messaging_op in scope. request_id values in fixtures must \ - match the http.request envelopes your go_source emits. - - Match http_results by request_id and de-duplicate with stable sorting; never depend on response order. When vendor documentation is supplied in the user prompt, use it only to fill may_call HTTP details. """ } @@ -366,22 +325,9 @@ actor ConfiguredPluginSafetyReviewer: PluginFactoryReviewer { private static func reviewerSystemPrompt(for userGoal: String?) -> String { """ - You are Derrick's independent plugin alignment and safety reviewer. Review the user's goal, manifest, test_input_json, exact Go source, and direct test output. - Return exactly one JSON object: - {"decision":"approved|rejected","summary":"...","findings":[ - {"severity":"info|warning|blocking","category":"alignment|safety|correctness|privacy|supplyChain","message":"..."} - ]} - Reject unsafe, misleading, unrelated, non-deterministic, credential-seeking, or policy-bypassing code. - Apply these checks from observable evidence: - - A deterministic result uses stable sorting and de-duplication and does not depend on response order, current time, randomness, or UUIDs. - - Source-derived headline titles may be fragments; only generated explanatory summaries must be complete sentences when the manifest requires prose. - - `result.emit.html` is an allowed output format. Derrick sanitizes it with an allowlist before rendering. Reject executable script behavior or a deliberate sanitizer bypass, not ordinary safe HTML tags. - - Reject missing source-grounded parsing or claims that the direct test output does not support. - - For connector plugins, obey the connector protocol JSON below. If a rule is not in that JSON, do not require it. + \(ScriptExecContractPrompts.pluginFactoryReviewerGuide()) \(ConnectorContractPrompts.reviewerGuide(forUserGoal: userGoal)) - Compilation success is not approval. Do not rewrite the code or approve a draft that fails these checks. - Reject non-Go source, raw network usage outside http.request envelopes, or missing stdin reads. """ } diff --git a/ui/ui/Views/PluginHTMLResultExtractor.swift b/ui/ui/Views/PluginHTMLResultExtractor.swift index 77e7d82..cbe041a 100644 --- a/ui/ui/Views/PluginHTMLResultExtractor.swift +++ b/ui/ui/Views/PluginHTMLResultExtractor.swift @@ -57,6 +57,7 @@ enum PluginResultExtractor { let format = envelope.payload["format"]?.stringValue?.lowercased() let candidate = envelope.payload["content"]?.stringValue ?? envelope.payload["summary"]?.stringValue + ?? envelope.payload["markdown"]?.stringValue ?? envelope.payload["text"]?.stringValue ?? "" guard !candidate.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { @@ -320,6 +321,7 @@ enum PluginResultExtractor { let candidate = item["content"] as? String ?? item["summary"] as? String + ?? item["markdown"] as? String ?? item["text"] as? String ?? "" guard !candidate.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { diff --git a/ui/uiTests/PromptResourcesTests.swift b/ui/uiTests/PromptResourcesTests.swift index c63e47a..0c9990a 100644 --- a/ui/uiTests/PromptResourcesTests.swift +++ b/ui/uiTests/PromptResourcesTests.swift @@ -77,7 +77,7 @@ import Testing #expect(source.contains("package main")) let wrapped = try PromptResources.guestSDKForModel() #expect(wrapped.contains("```go")) - #expect(wrapped.contains("Go guest contract")) + #expect(wrapped.contains("script-exec-contract.json")) } @Test func throwsWhenConversationRAGInstructionsAreMissing() throws { diff --git a/workers/go/internal/contract/schemas/script-exec-contract.schema.json b/workers/go/internal/contract/schemas/script-exec-contract.schema.json new file mode 100644 index 0000000..2e36901 --- /dev/null +++ b/workers/go/internal/contract/schemas/script-exec-contract.schema.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/script-exec-contract.json", + "title": "Derrick script_exec protocol document", + "description": "Canonical guest runtime, workflow, output, agent, and reviewer rules for script_exec and offline Go guests.", + "type": "object", + "required": [ + "version", + "runtime", + "io", + "workflow", + "output", + "agent", + "review", + "rules", + "plugin_factory" + ], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "const": 1 }, + "runtime": { "$ref": "#/$defs/runtime" }, + "io": { "$ref": "#/$defs/io" }, + "workflow": { "$ref": "#/$defs/workflow" }, + "output": { "$ref": "#/$defs/output" }, + "agent": { "$ref": "#/$defs/agent" }, + "review": { "$ref": "#/$defs/review" }, + "rules": { "$ref": "#/$defs/rules" }, + "plugin_factory": { "$ref": "#/$defs/plugin_factory" } + }, + "$defs": { + "runtime": { + "type": "object", + "required": [ + "language", + "package", + "binary", + "stdlib_only", + "forbidden_imports", + "allowed_os_usage", + "example_go" + ], + "additionalProperties": false, + "properties": { + "language": { "type": "string", "const": "go" }, + "package": { "type": "string", "const": "main" }, + "binary": { "type": "string", "const": "/tmp/guest" }, + "stdlib_only": { "type": "boolean" }, + "forbidden_imports": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "allowed_os_usage": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "example_go": { "type": "string", "minLength": 1 } + } + }, + "io": { + "type": "object", + "required": ["stdin_schema", "stdout_schema", "guest_runtime_schema"], + "additionalProperties": false, + "properties": { + "stdin_schema": { "type": "string", "const": "hop-event.schema.json" }, + "stdout_schema": { "type": "string", "const": "envelope-list.schema.json" }, + "guest_runtime_schema": { "type": "string", "const": "guest-runtime.schema.json" } + } + }, + "workflow": { + "type": "object", + "required": [ + "first_hop_verbs", + "http_results_event_kind", + "terminal_verbs", + "match_http_by", + "http_results_accumulate", + "post_body_field" + ], + "additionalProperties": false, + "properties": { + "first_hop_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "http_results_event_kind": { "type": "string" }, + "terminal_verbs": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "match_http_by": { "type": "string", "const": "request_id" }, + "http_results_accumulate": { "type": "boolean" }, + "post_body_field": { "type": "string", "const": "json" } + } + }, + "output": { + "type": "object", + "required": ["fields", "forbidden_patterns", "raw_body_in_content_requires_explicit_user_request"], + "additionalProperties": false, + "properties": { + "fields": { + "type": "object", + "required": ["content", "summary", "html", "markdown"], + "additionalProperties": false, + "properties": { + "content": { "$ref": "#/$defs/output_field" }, + "summary": { "$ref": "#/$defs/output_field" }, + "html": { "$ref": "#/$defs/output_field" }, + "markdown": { "$ref": "#/$defs/output_field" } + } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" } + }, + "raw_body_in_content_requires_explicit_user_request": { "type": "boolean" } + } + }, + "output_field": { + "type": "object", + "required": ["purpose"], + "additionalProperties": true, + "properties": { + "purpose": { "type": "string" }, + "host_strips_incidental_markup": { "type": "boolean" }, + "host_allowlist_sanitizes": { "type": "boolean" }, + "prefer_content_for_extracted_text": { "type": "boolean" } + } + }, + "agent": { + "type": "object", + "required": [ + "prefer_direct_urls_over_serp_html", + "retry_with_different_urls_on_empty_fetch", + "max_correction_attempts_after_block" + ], + "additionalProperties": false, + "properties": { + "prefer_direct_urls_over_serp_html": { "type": "boolean" }, + "retry_with_different_urls_on_empty_fetch": { "type": "boolean" }, + "max_correction_attempts_after_block": { "type": "integer", "minimum": 0 } + } + }, + "review": { + "type": "object", + "required": [ + "fail_fast", + "response_schema", + "checks", + "do_not_deny_for", + "enforced_elsewhere" + ], + "additionalProperties": false, + "properties": { + "fail_fast": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": [ + "alignedWithRequest", + "confidence", + "suggestedAction", + "concerns", + "summary" + ], + "additionalProperties": false, + "properties": { + "alignedWithRequest": { "type": "string" }, + "confidence": { "type": "string" }, + "suggestedAction": { "type": "string" }, + "concerns": { "type": "string" }, + "summary": { "type": "string" } + } + }, + "checks": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/review_check" } + }, + "do_not_deny_for": { + "type": "array", + "items": { "type": "string" } + }, + "enforced_elsewhere": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "review_check": { + "type": "object", + "required": ["id", "order", "description"], + "additionalProperties": false, + "properties": { + "id": { "type": "string" }, + "order": { "type": "integer", "minimum": 1 }, + "description": { "type": "string" }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "rules": { + "type": "object", + "required": [ + "guest_has_no_network", + "host_performs_http", + "host_applies_ssrf", + "static_verifier_enforces_imports", + "scheduler_timing_not_in_script", + "deterministic_output_required", + "stable_sort_and_dedupe_collections", + "no_time_random_uuid_for_visible_output", + "match_http_results_by_request_id" + ], + "additionalProperties": false, + "properties": { + "guest_has_no_network": { "type": "boolean" }, + "host_performs_http": { "type": "boolean" }, + "host_applies_ssrf": { "type": "boolean" }, + "static_verifier_enforces_imports": { "type": "boolean" }, + "scheduler_timing_not_in_script": { "type": "boolean" }, + "deterministic_output_required": { "type": "boolean" }, + "stable_sort_and_dedupe_collections": { "type": "boolean" }, + "no_time_random_uuid_for_visible_output": { "type": "boolean" }, + "match_http_results_by_request_id": { "type": "boolean" } + } + }, + "plugin_factory": { + "type": "object", + "required": ["manifest", "builder", "review"], + "additionalProperties": false, + "properties": { + "manifest": { "$ref": "#/$defs/plugin_factory_manifest" }, + "builder": { "$ref": "#/$defs/plugin_factory_builder" }, + "review": { "$ref": "#/$defs/plugin_factory_review" } + } + }, + "plugin_factory_manifest": { + "type": "object", + "required": [ + "host_creates_manifest", + "do_not_return_manifest_json", + "agent_plugin_schema", + "entrypoint", + "plugin_id_allowed_chars", + "plugin_id_forbidden_chars", + "roles", + "connector_messaging_ops", + "secret_kinds", + "never_embed_credentials_in_go_source", + "host_lists_connectors_under_messaging" + ], + "additionalProperties": false, + "properties": { + "host_creates_manifest": { "type": "boolean" }, + "do_not_return_manifest_json": { "type": "boolean" }, + "agent_plugin_schema": { "type": "string" }, + "entrypoint": { "type": "string" }, + "plugin_id_allowed_chars": { "type": "string" }, + "plugin_id_forbidden_chars": { + "type": "array", + "items": { "type": "string" } + }, + "roles": { + "type": "array", + "items": { "type": "string" } + }, + "connector_messaging_ops": { + "type": "array", + "items": { "type": "string" } + }, + "secret_kinds": { + "type": "array", + "items": { "type": "string" } + }, + "never_embed_credentials_in_go_source": { "type": "boolean" }, + "host_lists_connectors_under_messaging": { "type": "boolean" } + } + }, + "plugin_factory_builder": { + "type": "object", + "required": [ + "response_keys", + "skill_files_path_pattern", + "empty_skill_files_when_unused", + "test_input_is_serialized_json_object", + "test_input_must_not_be_empty", + "test_input_exercises_terminal_result", + "connector_test_input_uses_hops_array", + "connector_parse_json_http_results_when_vendor_returns_json" + ], + "additionalProperties": false, + "properties": { + "response_keys": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "skill_files_path_pattern": { "type": "string" }, + "empty_skill_files_when_unused": { "type": "boolean" }, + "test_input_is_serialized_json_object": { "type": "boolean" }, + "test_input_must_not_be_empty": { "type": "boolean" }, + "test_input_exercises_terminal_result": { "type": "boolean" }, + "connector_test_input_uses_hops_array": { "type": "boolean" }, + "connector_parse_json_http_results_when_vendor_returns_json": { "type": "boolean" } + } + }, + "plugin_factory_review": { + "type": "object", + "required": [ + "compilation_success_not_approval", + "response_schema", + "reject_non_go_source", + "reject_raw_network_outside_http_request_envelopes", + "reject_missing_stdin_read", + "source_derived_titles_may_be_fragments", + "reject_unsupported_direct_test_claims", + "connector_rules_document" + ], + "additionalProperties": false, + "properties": { + "compilation_success_not_approval": { "type": "boolean" }, + "response_schema": { + "type": "object", + "required": ["decision", "summary", "findings"], + "additionalProperties": false, + "properties": { + "decision": { "type": "string" }, + "summary": { "type": "string" }, + "findings": { "type": "string" } + } + }, + "reject_non_go_source": { "type": "boolean" }, + "reject_raw_network_outside_http_request_envelopes": { "type": "boolean" }, + "reject_missing_stdin_read": { "type": "boolean" }, + "source_derived_titles_may_be_fragments": { "type": "boolean" }, + "reject_unsupported_direct_test_claims": { "type": "boolean" }, + "connector_rules_document": { "type": "string" } + } + } + } +} From 1bad7a397e6db372598f02b19b14cffc87eb589f Mon Sep 17 00:00:00 2001 From: David Choi Date: Wed, 9 Sep 2026 23:27:06 -0400 Subject: [PATCH 05/12] update init modal --- .../Support/AppBootstrapStatus.swift | 100 +++++++++++++++++- .../DockerRunner/XPCDockerRunner.swift | 9 ++ ui/ui/Views/ContentView.swift | 71 ++++++------- ui/uiTests/AppBootstrapStatusTests.swift | 29 +++++ 4 files changed, 169 insertions(+), 40 deletions(-) diff --git a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift index c75b9b4..e5016da 100644 --- a/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift +++ b/ui/SharedAgentRuntime/Support/AppBootstrapStatus.swift @@ -18,8 +18,44 @@ final class AppBootstrapStatus: ObservableObject { case failed } + /// Parallel bootstrap steps shown in the init modal. Completed steps are removed from the list. + enum TaskID: String, Sendable, CaseIterable, Equatable { + case daemon + case database + case docker + case workerImage + + var sortOrder: Int { + switch self { + case .daemon: return 0 + case .database: return 1 + case .docker: return 2 + case .workerImage: return 3 + } + } + + var defaultMessage: String { + switch self { + case .daemon: + return "Connecting to Derrick daemon…" + case .database: + return "Opening local database…" + case .docker: + return "Checking Docker Desktop…" + case .workerImage: + return "Preparing worker image…" + } + } + } + + struct LoadingTask: Identifiable, Equatable, Sendable { + let id: TaskID + var message: String + } + @Published private(set) var phase: Phase = .idle @Published private(set) var statusMessage: String = "Starting…" + @Published private(set) var activeLoadingTasks: [LoadingTask] = [] @Published private(set) var failureTitle: String? @Published private(set) var failureMessage: String? /// Extra recovery control on the failure modal (for example Open Login Items). @@ -43,6 +79,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = false phase = .idle statusMessage = "Starting…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -108,6 +145,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = deferModal phase = .loadingSession statusMessage = "Loading session store…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -138,6 +176,37 @@ final class AppBootstrapStatus: ObservableObject { } } + func beginTask(_ id: TaskID, message: String? = nil) { + guard isInitializing else { return } + let label = message ?? id.defaultMessage + if let index = activeLoadingTasks.firstIndex(where: { $0.id == id }) { + activeLoadingTasks[index].message = label + } else { + activeLoadingTasks.append(LoadingTask(id: id, message: label)) + activeLoadingTasks.sort { $0.id.sortOrder < $1.id.sortOrder } + } + if !deferModalPresentation { + isModalPresented = true + } + debugLog("[bootstrap] task begin \(id.rawValue): \(label)") + } + + func updateTask(_ id: TaskID, message: String) { + guard isInitializing else { return } + if let index = activeLoadingTasks.firstIndex(where: { $0.id == id }) { + activeLoadingTasks[index].message = message + } else { + beginTask(id, message: message) + } + debugLog("[bootstrap] task update \(id.rawValue): \(message)") + } + + func completeTask(_ id: TaskID) { + guard activeLoadingTasks.contains(where: { $0.id == id }) else { return } + activeLoadingTasks.removeAll { $0.id == id } + debugLog("[bootstrap] task complete \(id.rawValue)") + } + func update(phase: Phase, message: String) { // Never re-open the modal after ready (parallel service ensure-up must not reflash it). if self.phase == .ready, phase != .failed, phase != .ready { @@ -148,23 +217,45 @@ final class AppBootstrapStatus: ObservableObject { // beats "Opening local database…" while XPC is still retrying). if isInitializing, Self.phasePriority(phase) < Self.phasePriority(self.phase) { debugLog("[bootstrap] ignore lower-priority phase=\(phase.rawValue) while \(self.phase.rawValue): \(message)") - return + } else { + // Don't let a cancelled re-entrant task demote ready via failed paths above. + self.phase = phase + self.statusMessage = message } - // Don't let a cancelled re-entrant task demote ready via failed paths above. - self.phase = phase - self.statusMessage = message + syncLoadingTask(for: phase, message: message) if !deferModalPresentation { isModalPresented = true } debugLog("[bootstrap] phase=\(phase.rawValue) \(message)") } + private func syncLoadingTask(for phase: Phase, message: String) { + guard isInitializing else { return } + switch phase { + case .connectingHelper: + beginTask(.daemon, message: message) + case .loadingSession: + if message.localizedCaseInsensitiveContains("database") { + beginTask(.database, message: message) + } + case .checkingDocker: + beginTask(.docker, message: message) + case .preparingImage: + beginTask(.workerImage, message: message) + case .verifyingEnvironment: + completeTask(.workerImage) + default: + break + } + } + func markReady() { deferredModalRevealTask?.cancel() deferredModalRevealTask = nil deferModalPresentation = false phase = .ready statusMessage = "Ready" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none @@ -208,6 +299,7 @@ final class AppBootstrapStatus: ObservableObject { deferModalPresentation = false phase = .idle statusMessage = "Starting…" + activeLoadingTasks = [] failureTitle = nil failureMessage = nil failureRecovery = .none diff --git a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift index d1177e7..b08dedf 100644 --- a/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift +++ b/ui/SharedAgentRuntime/Support/DockerRunner/XPCDockerRunner.swift @@ -307,6 +307,7 @@ public final class XPCDockerRunner: @unchecked Sendable { ) } dockerReachableState.markCompleted() + await reportBootstrapTaskCompleted(.docker) Task { await prewarmWorkerImage() } @@ -413,6 +414,14 @@ public final class XPCDockerRunner: @unchecked Sendable { } } + private func reportBootstrapTaskCompleted(_ id: AppBootstrapStatus.TaskID) async { + await MainActor.run { + let status = AppBootstrapStatus.shared + guard status.isInitializing else { return } + status.completeTask(id) + } + } + deinit { connection.invalidate() } diff --git a/ui/ui/Views/ContentView.swift b/ui/ui/Views/ContentView.swift index f4e2783..1380c2d 100644 --- a/ui/ui/Views/ContentView.swift +++ b/ui/ui/Views/ContentView.swift @@ -596,7 +596,7 @@ struct ContentView: View { minWidth: 380, minHeight: 0, maxWidth: 440, - maxHeight: bootstrapStatus.phase == .failed ? 420 : 280, + maxHeight: bootstrapModalMaxHeight, onBackdropDismiss: bootstrapStatus.phase == .failed ? { bootstrapStatus.dismissFailure() } : nil, @@ -605,10 +605,7 @@ struct ContentView: View { : nil, header: { HStack(spacing: 10) { - if bootstrapStatus.showsProgressIndicator { - ProgressView() - .controlSize(.small) - } else if bootstrapStatus.phase == .failed { + if bootstrapStatus.phase == .failed { Image(systemName: ModalChrome.bootstrapFailureSymbol) .font(ModalChrome.symbolFont) .symbolRenderingMode(.hierarchical) @@ -626,23 +623,29 @@ struct ContentView: View { }, body: { VStack(alignment: .leading, spacing: 12) { - Text(bootstrapStatus.statusMessage) - .font(.body) - .foregroundStyle(.primary) - .fixedSize(horizontal: false, vertical: true) - - if bootstrapStatus.phase == .failed, let detail = bootstrapStatus.failureMessage, - detail != bootstrapStatus.statusMessage { - Text(detail) - .font(.subheadline) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) + if bootstrapStatus.isInitializing { + VStack(alignment: .leading, spacing: 10) { + ForEach(bootstrapStatus.activeLoadingTasks) { task in + HStack(alignment: .top, spacing: 10) { + ProgressView() + .controlSize(.small) + .frame(width: 16, height: 16) + .padding(.top, 2) + Text(task.message) + .font(.body) + .foregroundStyle(.primary) + .fixedSize(horizontal: false, vertical: true) + } + } + } + .animation(.easeOut(duration: 0.2), value: bootstrapStatus.activeLoadingTasks) } - if bootstrapStatus.isInitializing { - Text(bootstrapProgressHint) - .font(.caption) - .foregroundStyle(.secondary) + if bootstrapStatus.phase == .failed { + Text(bootstrapStatus.failureMessage ?? bootstrapStatus.statusMessage) + .font(.body) + .foregroundStyle(.primary) + .fixedSize(horizontal: false, vertical: true) } } .padding(.horizontal, 20) @@ -720,17 +723,12 @@ struct ContentView: View { .background(WindowConfigurator()) } - private var bootstrapProgressHint: String { - switch bootstrapStatus.phase { - case .preparingImage, .checkingDocker, .verifyingEnvironment: - return "Keep Docker Desktop running. The worker image may finish building in the background." - case .connectingHelper: - return "Starting the background helper…" - case .loadingSession: - return "Loading your local workspace…" - default: - return "Starting…" + private var bootstrapModalMaxHeight: CGFloat { + if bootstrapStatus.phase == .failed { + return 420 } + let rowCount = max(bootstrapStatus.activeLoadingTasks.count, 1) + return CGFloat(120 + rowCount * 34) } @MainActor @@ -761,23 +759,23 @@ struct ContentView: View { do { bootstrapStatus.update(phase: .loadingSession, message: "Starting Derrick…") + bootstrapStatus.beginTask(.daemon) + bootstrapStatus.beginTask(.database) + bootstrapStatus.beginTask(.docker) // Docker reachability, daemon, and DB are independent — run in parallel. let dockerPeerTask = Task { try await prewarmLaunchDockerPeer() } async let health = connectLaunchDaemon() async let repo = loadLaunchRepository() - bootstrapStatus.update( - phase: .connectingHelper, - message: "Connecting to Derrick daemon…" - ) let healthResult = try await health + bootstrapStatus.completeTask(.daemon) debugLog( "Daemon ensure-up ok status=\(healthResult.status.rawValue) pid=\(healthResult.pid) runtime=\(healthResult.guestRuntimeImage ?? "?") detail=\(healthResult.detail ?? "")" ) - bootstrapStatus.update(phase: .loadingSession, message: "Opening local database…") let repoResult = try await repo + bootstrapStatus.completeTask(.database) sessionReady = true bootstrapStatus.markReady() @@ -856,9 +854,10 @@ struct ContentView: View { /// Prewarm Docker in parallel with daemon + DB. Only peer handoff needs both daemon XPC and Docker. @MainActor private func prewarmLaunchDockerPeer() async throws -> NSXPCListenerEndpoint? { - bootstrapStatus.update(phase: .checkingDocker, message: "Starting Docker runtime…") + bootstrapStatus.updateTask(.docker, message: "Starting Docker runtime…") _ = XPCDockerRunner.shared try await XPCDockerRunner.shared.waitUntilDockerReachable() + bootstrapStatus.completeTask(.docker) do { return try await XPCDockerRunner.shared.fetchPeerListenerEndpoint() } catch { diff --git a/ui/uiTests/AppBootstrapStatusTests.swift b/ui/uiTests/AppBootstrapStatusTests.swift index a91072d..007d27a 100644 --- a/ui/uiTests/AppBootstrapStatusTests.swift +++ b/ui/uiTests/AppBootstrapStatusTests.swift @@ -184,6 +184,35 @@ import Testing #expect(status.statusMessage == "Connecting to Derrick daemon…") } + @MainActor + @Test func parallelLoadingTasksTrackIndependently() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.beginTask(.daemon) + status.beginTask(.database) + status.beginTask(.docker) + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + status.completeTask(.database) + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .docker]) + status.completeTask(.daemon) + status.completeTask(.docker) + #expect(status.activeLoadingTasks.isEmpty) + } + + @MainActor + @Test func phaseUpdateAddsMatchingLoadingTasks() { + let status = freshStatus() + #expect(status.beginLoadingSession()) + status.update(phase: .connectingHelper, message: "Connecting to Derrick daemon…") + status.update(phase: .loadingSession, message: "Opening local database…") + status.update(phase: .checkingDocker, message: "Checking Docker Desktop…") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + status.update(phase: .preparingImage, message: "Preparing worker image…") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker, .workerImage]) + status.update(phase: .verifyingEnvironment, message: "Worker image ready.") + #expect(status.activeLoadingTasks.map(\.id) == [.daemon, .database, .docker]) + } + @MainActor @Test func cancelClearsInProgressModal() { let status = freshStatus() From aff3459e2f5ef7bd806ceb41190de5591e57ba51 Mon Sep 17 00:00:00 2001 From: David Choi Date: Thu, 10 Sep 2026 23:52:28 -0400 Subject: [PATCH 06/12] fix news reader --- CONTRIBUTING.md | 3 +- THIRD_PARTY_NOTICES.md | 2 +- docker/guest-runtime/Dockerfile | 15 - docker/worker/Dockerfile | 4 + master-todo.md | 2 +- .../DBRepository/DBRepositoryNews.swift | 19 +- .../Migrations/0005_news_readers.up.sql | 1 + .../DBRepositoryTests/DBNewsReaderTests.swift | 6 +- .../PluginFactoryCreateWorkflow.swift | 127 ++- .../DockerRunRequestValidator.swift | 1 + .../DockerRunnerXPCTests.swift | 3 +- .../MCPServer/DockerImageInspector.swift | 21 + .../DockerProductImagePrewarmer.swift | 13 +- .../MCPServer/NewsReaderDockerExecutor.swift | 115 +++ .../NewsReaderDockerInputPreparer.swift | 43 + .../Tests/MCPServerTests/MCPServerTests.swift | 62 +- .../Factory/PluginFactoryImplementation.swift | 6 +- .../PluginTests/PluginFactoryTests.swift | 111 ++- .../AppServices/DerrickBundledText.swift | 2 +- .../AppServices/ServiceHealth.swift | 3 - .../InProcessServiceBridges.swift | 3 + .../MCPService/MCPServiceXPC.swift | 10 + .../MCPService/MCPToolCallTimeouts.swift | 3 + .../MCPService/PluginFactoryCreateInput.swift | 95 +- .../News/NewsFeedParser.swift | 154 --- .../News/NewsReaderModels.swift | 76 +- .../News/NewsReaderRefresh.swift | 195 ++-- .../News/NewsReaderWorkerTypes.swift | 72 ++ .../News/NewsWorkerBridge.swift | 30 + .../Plugin/PluginSkillDraft.swift | 447 +++++++++ .../Sources/Contract/GuestContract.swift | 1 + .../schemas/news-reader-result.schema.json | 7 + .../schemas/worker-product.schema.json | 31 +- .../DerrickDockerRuntimeIdentity.swift | 1 + .../DockerProductImageDigests.generated.swift | 2 +- .../DockerRunnerXPC/DockerWorkerRuntime.swift | 12 + .../Factory/PluginFactoryRuntimeTypes.swift | 28 + .../Plugin/Factory/PluginFactoryTypes.swift | 8 +- .../Plugin/Manifest/DerrickRuntime.swift | 4 +- .../Plugin/Manifest/PluginManifestError.swift | 2 +- .../AppLayerServicesWireTests.swift | 24 +- .../StructureTests/NewsReaderTests.swift | 193 +--- .../PluginSkillDraftTests.swift | 118 +++ scripts/reset-local-state.sh | 48 + ui/JobKeepAlive/DaemonModuleBootstrap.swift | 5 +- ui/JobKeepAlive/DaemonUnifiedXPC.swift | 4 + ui/MCPService/MCPServiceExportedObject.swift | 17 + ui/MCPService/MCPServiceToolHost.swift | 42 + .../Services/MCPServiceClient.swift | 19 + ui/ui/News/MCPServiceNewsWorker.swift | 23 + ui/ui/News/NewsReaderStore.swift | 44 +- ui/ui/News/NewsReaderSummarizer.swift | 65 ++ ui/ui/News/NewsWorkspaceView.swift | 80 +- ui/ui/Plugins/PluginCreationController.swift | 596 ++++++------ ui/ui/Plugins/PluginsWorkspaceView.swift | 890 +++++++++++------- ui/ui/Views/ContentView.swift | 5 +- ui/uiTests/NewsReaderFlowTests.swift | 58 ++ workers/go/cmd/derrick-news-reader/main.go | 97 ++ workers/go/internal/contract/contract.go | 5 + .../schemas/news-reader-result.schema.json | 7 + .../schemas/worker-product.schema.json | 31 +- workers/go/internal/newsreader/engine.go | 34 + workers/go/internal/newsreader/fetch.go | 110 +++ workers/go/internal/newsreader/list.go | 88 ++ .../go/internal/newsreader/newsreader_test.go | 30 + workers/go/internal/newsreader/rss.go | 207 ++++ workers/go/internal/newsreader/text.go | 76 ++ workers/go/internal/newsreader/types.go | 49 + workers/go/internal/newsreader/urls.go | 57 ++ workers/go/internal/newsreader/validate.go | 38 + 70 files changed, 3531 insertions(+), 1269 deletions(-) delete mode 100644 docker/guest-runtime/Dockerfile create mode 100644 packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift create mode 100644 packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift delete mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift create mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift create mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift create mode 100644 packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift create mode 100644 packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json create mode 100644 packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift create mode 100755 scripts/reset-local-state.sh create mode 100644 ui/ui/News/MCPServiceNewsWorker.swift create mode 100644 ui/ui/News/NewsReaderSummarizer.swift create mode 100644 ui/uiTests/NewsReaderFlowTests.swift create mode 100644 workers/go/cmd/derrick-news-reader/main.go create mode 100644 workers/go/internal/contract/schemas/news-reader-result.schema.json create mode 100644 workers/go/internal/newsreader/engine.go create mode 100644 workers/go/internal/newsreader/fetch.go create mode 100644 workers/go/internal/newsreader/list.go create mode 100644 workers/go/internal/newsreader/newsreader_test.go create mode 100644 workers/go/internal/newsreader/rss.go create mode 100644 workers/go/internal/newsreader/text.go create mode 100644 workers/go/internal/newsreader/types.go create mode 100644 workers/go/internal/newsreader/urls.go create mode 100644 workers/go/internal/newsreader/validate.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 755f87f..96feb39 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,8 @@ Thank you for your interest in contributing. Please read the [Code of Conduct](C ## Requirements - **macOS** with **Xcode 27** (Swift 6.4+) -- **Docker Desktop** (Python guest runtime, web crawler, and file extractor images) +- **Docker Desktop** (Go worker image for guests, web crawler, and file extractor) +- **Go 1.27.1+** (`brew install go`) for local diagnostics; guest compile runs in Docker - Apple Developer account for code signing ## Getting started diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index d7da35c..8d30ce1 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -23,7 +23,7 @@ Derrick integrates with user-configured APIs. You supply your own keys and are s ## Runtime -- **Docker Desktop** — script and plugin execution use the `python:3.14.7` guest image (see `docs/adr-swift-script-runtime.md` for superseded Swift guest notes). +- **Docker Desktop** — script and plugin execution use the `derrick-worker:go-v1` guest image. ## Project license diff --git a/docker/guest-runtime/Dockerfile b/docker/guest-runtime/Dockerfile deleted file mode 100644 index 5c99ce1..0000000 --- a/docker/guest-runtime/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Offline guest runtime for script_exec and plugin.invoke (Python primary). -# Build: docker build -f docker/guest-runtime/Dockerfile -t derrick-guest-runtime:python-v1 . -FROM python:3.14.7 - -RUN useradd --create-home --uid 10001 guest - -# uv — fast, lockfile-friendly dependency installs inside the guest image. -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv - -WORKDIR /home/guest -USER guest - -# Guests run with --network none; deps are baked into the image or installed at build time. -ENV UV_LINK_MODE=copy \ - PYTHONUNBUFFERED=1 diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index bab3814..7098c07 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -9,9 +9,12 @@ COPY workers/go ./ RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-web-crawler ./cmd/derrick-crawler RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-file-extractor ./cmd/derrick-file-extractor +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-news-reader ./cmd/derrick-news-reader FROM debian:bookworm-slim +LABEL derrick.worker.binaries="crawler,extractor,news-reader" + RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils ca-certificates \ && rm -rf /var/lib/apt/lists/* \ @@ -20,6 +23,7 @@ RUN apt-get update \ COPY --from=build /usr/local/go /usr/local/go COPY --from=build /out/derrick-web-crawler /usr/local/bin/derrick-web-crawler COPY --from=build /out/derrick-file-extractor /usr/local/bin/derrick-file-extractor +COPY --from=build /out/derrick-news-reader /usr/local/bin/derrick-news-reader RUN mkdir -p /data/in /data/out && chown -R worker:worker /data ENV PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/master-todo.md b/master-todo.md index c6e09ec..9cd98b9 100644 --- a/master-todo.md +++ b/master-todo.md @@ -77,7 +77,7 @@ Open: binary convert (xlsx) has no “here is your file” UI yet. Do not delete Each `script_exec` / `plugin.invoke`: 1. Wait for the offline queue (max 1). -2. `docker create` a unique `derrick-guest-runtime-` from `python:3.14.7`. +2. `docker create` a unique `derrick-guest-runtime-` from `derrick-worker:go-v1`. 3. Run until the host hop loop is done (terminal envelope, error, or in-use lease TTL). 4. `docker rm -f` immediately — that is “I’m done.” 5. Release the queue slot so the next script can create at once. diff --git a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift b/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift index 1490c86..65b5595 100644 --- a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift +++ b/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift @@ -12,7 +12,7 @@ public extension DBRepository { try Self.execute(""" INSERT INTO news_readers ( id, name, topics_json, sources_json, mode, max_count, schedule, - last_error, last_fetched_at, created_at, updated_at + summary_text, last_error, last_fetched_at, created_at, updated_at ) VALUES ( \(quoted(spec.id)), \(quoted(spec.name)), @@ -21,6 +21,7 @@ public extension DBRepository { \(quoted(spec.mode.rawValue)), \(spec.maxCount), \(quoted(spec.schedule.rawValue)), + \(sqlValue(spec.summaryText)), \(sqlValue(spec.lastError)), \(sqlValue(spec.lastFetchedAt.map { Self.iso8601Formatter().string(from: $0) })), \(quoted(Self.iso8601Formatter().string(from: spec.createdAt))), @@ -33,6 +34,7 @@ public extension DBRepository { mode = excluded.mode, max_count = excluded.max_count, schedule = excluded.schedule, + summary_text = excluded.summary_text, last_error = excluded.last_error, last_fetched_at = excluded.last_fetched_at, updated_at = excluded.updated_at; @@ -44,7 +46,7 @@ public extension DBRepository { try withDatabaseHandle { handle in let sql = """ SELECT id, name, topics_json, sources_json, mode, max_count, schedule, - last_error, last_fetched_at, created_at, updated_at + summary_text, last_error, last_fetched_at, created_at, updated_at FROM news_readers ORDER BY updated_at DESC; """ @@ -97,7 +99,7 @@ public extension DBRepository { SELECT id, reader_id, title, source_url, source_label, summary, published_at, fetched_at FROM news_items WHERE reader_id = \(quoted(readerID)) - ORDER BY fetched_at DESC; + ORDER BY COALESCE(published_at, fetched_at) DESC; """ var statement: OpaquePointer? guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else { @@ -126,13 +128,14 @@ public extension DBRepository { name: text(1), topics: topics, sources: sources, - mode: NewsReaderMode(rawValue: text(4)) ?? .list, + mode: NewsReaderMode(rawValue: text(4)) ?? .rss, maxCount: Int(sqlite3_column_int(statement, 5)), schedule: NewsReaderSchedule(rawValue: text(6)) ?? .off, - lastError: optionalText(7), - lastFetchedAt: optionalText(8).flatMap { Self.iso8601Formatter().date(from: $0) }, - createdAt: Self.iso8601Formatter().date(from: text(9)) ?? .now, - updatedAt: Self.iso8601Formatter().date(from: text(10)) ?? .now + summaryText: optionalText(7), + lastError: optionalText(8), + lastFetchedAt: optionalText(9).flatMap { Self.iso8601Formatter().date(from: $0) }, + createdAt: Self.iso8601Formatter().date(from: text(10)) ?? .now, + updatedAt: Self.iso8601Formatter().date(from: text(11)) ?? .now ) } diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql index 6b4e532..054ea0f 100644 --- a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0005_news_readers.up.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS news_readers ( mode TEXT NOT NULL, max_count INTEGER NOT NULL, schedule TEXT NOT NULL, + summary_text TEXT, last_error TEXT, last_fetched_at TEXT, created_at TEXT NOT NULL, diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift index bbc475d..5578700 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift @@ -11,7 +11,8 @@ final class DBNewsReaderTests: XCTestCase { name: "Markets", topics: ["Financial", "rates"], sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/world/rss.xml")], - mode: .summaries, + mode: .summary, + summaryText: "Markets moved higher.", maxCount: 10, schedule: .daily ) @@ -20,7 +21,8 @@ final class DBNewsReaderTests: XCTestCase { XCTAssertEqual(listed.count, 1) XCTAssertEqual(listed[0].name, "Markets") XCTAssertEqual(listed[0].topics, ["Financial", "rates"]) - XCTAssertEqual(listed[0].mode, .summaries) + XCTAssertEqual(listed[0].mode, .summary) + XCTAssertEqual(listed[0].summaryText, "Markets moved higher.") let item = NewsItem( readerID: spec.id, diff --git a/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift b/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift index ce7af8e..34a8199 100644 --- a/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift +++ b/packages/DerrickBackend/Sources/DerrickBackend/PluginFactoryCreateWorkflow.swift @@ -36,56 +36,70 @@ enum PluginFactoryCreateWorkflow { ) async throws -> MCPToolCallResultDTO ) async throws { let input = try PluginFactoryCreateInput.decodeJSON(request.inputJSON) - guard input.pluginType == .connector else { + guard input.pluginType == .connector || input.pluginType == .custom else { try await fail( workflowID: workflowID, stage: "type", - message: "Only connector plugins can be built in the factory. News lists are created from the News reader wizard.", + message: "This plugin type cannot be built in the factory.", repositoryProvider: repositoryProvider ) return } - guard let vendor = input.vendor else { + guard let pluginID = input.pluginID, !pluginID.isEmpty else { try await fail( workflowID: workflowID, - stage: "vendor", - message: "Choose a messaging vendor for this connector.", + stage: "name", + message: "Name this plugin before creating it.", repositoryProvider: repositoryProvider ) return } - guard let pluginID = input.pluginID, !pluginID.isEmpty else { + guard !input.description.isEmpty else { try await fail( workflowID: workflowID, - stage: "name", - message: "Name this connector before creating it.", + stage: "description", + message: "Describe what the plugin should do, then try again.", repositoryProvider: repositoryProvider ) return } - guard let auth = input.auth else { + + if input.pluginType == .custom { + try await runCustomBuild( + workflowID: workflowID, + request: request, + input: input, + pluginID: pluginID, + baseContext: baseContext, + repositoryProvider: repositoryProvider, + executeTool: executeTool + ) + return + } + + guard let vendor = input.vendor else { try await fail( workflowID: workflowID, - stage: "auth", - message: "Save the connector credentials before creating it.", + stage: "vendor", + message: "Could not determine which messaging service this plugin targets.", repositoryProvider: repositoryProvider ) return } - guard auth.authScheme.isSupportedInWizard else { + guard let auth = input.auth else { try await fail( workflowID: workflowID, stage: "auth", - message: "OAuth connectors are not available yet. Use a bot token or API key.", + message: "Save the plugin credentials before creating it.", repositoryProvider: repositoryProvider ) return } - guard !input.description.isEmpty else { + guard auth.authScheme.isSupportedInWizard else { try await fail( workflowID: workflowID, - stage: "description", - message: "Choose a vendor and create the connector again.", + stage: "auth", + message: "OAuth connectors are not available yet. Use a bot token or API key.", repositoryProvider: repositoryProvider ) return @@ -194,6 +208,87 @@ enum PluginFactoryCreateWorkflow { ) } + private static func runCustomBuild( + workflowID: String, + request: WorkflowStartRequest, + input: PluginFactoryCreateInput, + pluginID: String, + baseContext: ExecutionContextWire, + repositoryProvider: @escaping @Sendable () async throws -> DBRepository, + executeTool: @escaping ( + String, + String, + ExecutionContextWire, + ServicePrincipal, + String?, + String?, + String, + String + ) async throws -> MCPToolCallResultDTO + ) async throws { + try await log( + workflowID: workflowID, + stage: "factory", + message: "Writing SKILL.md, building the guest program, and running trial tests…", + repositoryProvider: repositoryProvider + ) + let goal = input.customBuildGoal() + let buildArgs = try buildArguments(goal: goal, hostManifest: nil) + let buildResult = try await executeTool( + AllowedMCPTool.pluginFactoryBuild.rawValue, + buildArgs, + baseContext, + request.principal, + request.helperAPIKey, + request.helperReviewerModelJSON, + workflowID, + "factory" + ) + if buildResult.isError { + try await fail( + workflowID: workflowID, + stage: "factory", + message: userFacingToolError(buildResult, fallback: "Plugin factory could not finish building the plugin."), + repositoryProvider: repositoryProvider + ) + return + } + + guard let summary = decodeBuildResult(buildResult.text), + summary.ok != false, + let savedID = summary.pluginID?.trimmingCharacters(in: .whitespacesAndNewlines), + !savedID.isEmpty + else { + let decoded = decodeBuildResult(buildResult.text) + let raw = decoded?.error + ?? decoded?.reviewSummary + ?? "Plugin factory did not return a saved plugin." + try await fail( + workflowID: workflowID, + stage: "factory", + message: PluginFactoryCreateFailureMessage.userFacing(raw), + repositoryProvider: repositoryProvider + ) + return + } + + let resultJSON = try JSONEncoder.service.encode( + PluginFactoryCreateResult( + pluginID: savedID, + version: summary.version ?? "1.0.0", + vendor: "custom", + reviewSummary: summary.reviewSummary ?? "" + ) + ) + let resultText = String(decoding: resultJSON, as: UTF8.self) + try await complete( + workflowID: workflowID, + message: "Plugin saved as /\(savedID).", + resultJSON: resultText, + repositoryProvider: repositoryProvider + ) + } + private static func crawlArguments( startURL: String, vendor: PluginFactoryCreateInput.ConnectorVendor, diff --git a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift index ec7cd57..391ce1f 100644 --- a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift +++ b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift @@ -213,6 +213,7 @@ public enum DockerRunRequestValidator: Sendable { } case DockerWorkerRuntime.crawlerBinary, DockerWorkerRuntime.extractorBinary, + DockerWorkerRuntime.newsReaderBinary, DockerWorkerRuntime.guestBinaryPath: guard args == [command] else { return .disallowedDockerFlag("exec \(command)") diff --git a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift index 7daa176..cc7b002 100644 --- a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift +++ b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift @@ -240,6 +240,7 @@ struct DockerRunnerXPCTests { ["rm", "-f", "c"], ["inspect", "-f", "{{.State.Running}}", "c"], ["exec", "-i", "c", "/usr/local/bin/derrick-file-extractor"], + ["exec", "-i", "c", DockerWorkerRuntime.newsReaderBinary], [ "create", "--label", @@ -261,7 +262,7 @@ struct DockerRunnerXPCTests { @Test func rejectsInvalidGoGuestExecCommands() { for args in [ - ["exec", "-i", "c", "python3", "/tmp/guest.py"], + ["exec", "-i", "c", "python3", "/tmp/guest.go"], ["exec", "-i", "c", DockerWorkerRuntime.guestBinaryPath, "extra"], ["exec", "-i", "c", "sh", "-c", "cat > /tmp/other"], ["exec", "-i", "c", "sh", "-c", "rm -rf /"], diff --git a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift index 63835ad..5787393 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift @@ -33,4 +33,25 @@ public enum DockerImageInspector: Sendable { throw DockerImageDigestError.digestMismatch(tag: tag, expected: expected, actual: actual) } } + + /// Returns false when the image exists but predates required worker binaries (e.g. news reader). + public static func workerImageHasCurrentBinaries( + tag: String = DockerWorkerRuntime.image, + executor: @escaping DockerCLIExecutor + ) async -> Bool { + let format = "{{index .Config.Labels \"\(DockerWorkerRuntime.binariesLabelKey)\"}}" + do { + let response = try await executor( + ["image", "inspect", "--format", format, tag], + Data(), + 30 + ) + guard response.exitCode == 0 else { return false } + let label = String(decoding: response.stdout, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + return label == DockerWorkerRuntime.binariesLabelValue + } catch { + return false + } + } } diff --git a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift index 8eca7de..82f9d7b 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift @@ -36,12 +36,19 @@ public enum DockerProductImagePrewarmer: Sendable { ) async throws { let inspect = try await executor(["image", "inspect", tag], Data(), 30) if inspect.exitCode == 0 { - try await DockerImageInspector.verifyPinned( + let binariesCurrent = await DockerImageInspector.workerImageHasCurrentBinaries( tag: tag, - expected: pinnedDigest, executor: executor ) - return + if binariesCurrent { + try await DockerImageInspector.verifyPinned( + tag: tag, + expected: pinnedDigest, + executor: executor + ) + return + } + // Stale worker image (missing news reader, etc.). Rebuild overwrites the tag. } guard let repoRoot = DerrickRepositoryRoot.locate() else { diff --git a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift new file mode 100644 index 0000000..11c637f --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift @@ -0,0 +1,115 @@ +import Foundation +import Structure + +/// Runs the prebuilt news reader in a oneshot container: create, exec, rm. +public struct NewsReaderDockerExecutor: Sendable { + public static let image = DockerWorkerRuntime.image + public static let containerPrefix = "derrick-news-reader" + public static let binaryPath = DockerWorkerRuntime.newsReaderBinary + public static let maximumTimeoutSeconds = 300 + public static let dockerNetwork = "bridge" + + private let executor: DockerCLIExecutor + private let queue: DerrickDockerRunQueue + + public init( + executor: @escaping DockerCLIExecutor, + queue: DerrickDockerRunQueue = .crawler + ) { + self.executor = executor + self.queue = queue + } + + public func run( + input: Data, + timeoutSeconds: Int + ) async throws -> DockerCLIResult { + let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) + try await WorkerImageGate.shared.ensureReady(executor: executor) + let prepared = try await NewsReaderDockerInputPreparer.enrich(input) + let executor = self.executor + do { + return try await queue.withPermit { + let proxyLease = try await WebCrawlerEgressProxy.shared.lease(forHosts: prepared.leaseHosts) + do { + let result = try await OneshotDockerContainer.run( + executor: executor, + prefix: Self.containerPrefix, + createArguments: { name in + Self.createArguments( + name: name, + proxyHost: proxyLease.host, + proxyPort: proxyLease.port, + proxyToken: proxyLease.clientToken + ) + }, + createStep: "create news reader container", + startStep: "start news reader container", + body: { name in + try await executor( + ["exec", "-i", name, Self.binaryPath], + prepared.data, + timeout + ) + } + ) + await WebCrawlerEgressProxy.shared.release(forHosts: prepared.leaseHosts) + return result + } catch { + await WebCrawlerEgressProxy.shared.release(forHosts: prepared.leaseHosts) + throw error + } + } + } catch let error as OneshotDockerContainerError { + throw mappedNewsReaderError(error) + } + } + + static func createArguments( + name: String, + proxyHost: String, + proxyPort: Int, + proxyToken: String + ) -> [String] { + [ + "create", + ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ + "--network", dockerNetwork, + "--read-only", + "--tmpfs", "/tmp:rw,exec,nosuid,size=64m", + "--pids-limit", "128", + "--cpus", "1.0", + "--memory", "512m", + "--name", name, + "--env", "DERRICK_EGRESS_PROXY_HOST=\(proxyHost)", + "--env", "DERRICK_EGRESS_PROXY_PORT=\(proxyPort)", + "--env", "DERRICK_EGRESS_PROXY_TOKEN=\(proxyToken)", + "--entrypoint", "/bin/sleep", + image, + "infinity", + ] + } + + private func mappedNewsReaderError(_ error: OneshotDockerContainerError) -> Error { + switch error { + case .commandFailed(let step, let detail): + return NewsReaderDockerExecutorError.commandFailed(step, detail) + case .imageUnavailable(let detail): + return NewsReaderDockerExecutorError.imageUnavailable(detail) + } + } +} + +public enum NewsReaderDockerExecutorError: Error, LocalizedError, Sendable, Equatable { + case commandFailed(String, String) + case imageUnavailable(String) + + public var errorDescription: String? { + switch self { + case .commandFailed(let step, let detail): + return "\(step) failed: \(detail)" + case .imageUnavailable(let image): + return "News reader image is not installed: \(image)." + } + } +} diff --git a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift new file mode 100644 index 0000000..a894dbe --- /dev/null +++ b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift @@ -0,0 +1,43 @@ +import Foundation +import Structure +import WebCrawler + +/// Host-side helpers for preparing news reader Docker input and egress policy. +enum NewsReaderDockerInputPreparer { + static func enrich(_ input: Data) async throws -> (data: Data, leaseHosts: [String]) { + let request: NewsReaderWorkerRequest + do { + request = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: input) + } catch { + throw NewsReaderDockerExecutorError.commandFailed( + "prepare news reader container", + "News reader input was not valid JSON." + ) + } + guard !request.sources.isEmpty else { + throw NewsReaderDockerExecutorError.commandFailed( + "prepare news reader container", + "News reader input did not include any sources." + ) + } + + var hosts = Set() + for source in request.sources { + guard let url = URL(string: source.url) else { + throw NewsReaderDockerExecutorError.commandFailed( + "prepare news reader container", + "Source URL is not valid: \(source.url)" + ) + } + let chain = await WebCrawlerRedirectResolver.hostsInRedirectChain(from: url) + hosts.formUnion(chain) + } + guard !hosts.isEmpty else { + throw NewsReaderDockerExecutorError.commandFailed( + "prepare news reader container", + "Could not determine hosts to fetch from the configured sources." + ) + } + return (input, Array(hosts).sorted()) + } +} diff --git a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift index 1ce0f0a..25576ca 100644 --- a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift +++ b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift @@ -282,6 +282,60 @@ import WebCrawler #expect(allowed?.contains("docs.slack.dev") == true) } + @Test func workerImageLabelDetectsMissingNewsReaderBinary() async { + let recorder = DockerCallRecorder() + let executor: DockerCLIExecutor = { args, _, _ in + await recorder.append(args) + if args.first == "image", args.contains("inspect"), args.contains("--format") { + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + let current = await DockerImageInspector.workerImageHasCurrentBinaries(executor: executor) + #expect(!current) + } + + @Test func newsReaderContainerCreateAndExecPassDockerValidator() { + let createArgs = NewsReaderDockerExecutor.createArguments( + name: "derrick-news-reader-test", + proxyHost: "172.17.0.1", + proxyPort: 3128, + proxyToken: "token" + ) + #expect( + DockerRunRequestValidator.validate( + DockerHostLaunch.makeRequest(dockerArguments: createArgs, timeoutSeconds: 60) + ) == nil + ) + let execArgs = DockerHostLaunch.dockerCLIArguments([ + "exec", "-i", "derrick-news-reader-test", NewsReaderDockerExecutor.binaryPath, + ]) + #expect( + DockerRunRequestValidator.validate( + DockerHostLaunch.makeRequest(dockerArguments: execArgs, timeoutSeconds: 60) + ) == nil + ) + } + + @Test func newsReaderInputPreparerUsesSourceHostsNotCrawlerStartURL() async throws { + let input = try JSONEncoder.service.encode( + NewsReaderWorkerRequest( + mode: .rss, + sources: [ + NewsSource(label: "Google News", url: "https://news.google.com/rss?hl=en-US"), + ], + topics: ["Tech"], + maxCount: 20 + ) + ) + + let prepared = try await NewsReaderDockerInputPreparer.enrich(input) + #expect(prepared.leaseHosts.contains("news.google.com")) + let decoded = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: prepared.data) + #expect(decoded.sources.count == 1) + #expect(decoded.sources[0].url.contains("news.google.com")) + } + @Test func dockerProductImagePrewarmerSkipsBuildWhenImagePresent() async throws { let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in @@ -895,7 +949,7 @@ import WebCrawler @Test func oneshotEnsurePulledImageSkipsPullWhenImageExists() async throws { let recorder = DockerCallRecorder() - try await OneshotDockerContainer.ensurePulledImage("python:3.14.7") { arguments, _, _ in + try await OneshotDockerContainer.ensurePulledImage(DockerWorkerRuntime.image) { arguments, _, _ in await recorder.append(arguments) return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } @@ -906,7 +960,7 @@ import WebCrawler @Test func oneshotEnsurePulledImagePullsWhenMissing() async throws { let recorder = DockerCallRecorder() - try await OneshotDockerContainer.ensurePulledImage("python:3.14.7") { arguments, _, _ in + try await OneshotDockerContainer.ensurePulledImage(DockerWorkerRuntime.image) { arguments, _, _ in await recorder.append(arguments) if arguments.first == "image" { return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) @@ -915,7 +969,7 @@ import WebCrawler } let calls = await recorder.calls #expect(calls.contains { $0.starts(with: ["image", "inspect"]) }) - #expect(calls.contains { $0.first == "pull" && $0.contains("python:3.14.7") }) + #expect(calls.contains { $0.first == "pull" && $0.contains(DockerWorkerRuntime.image) }) } @Test func dockerRunQueueSerializesWhenMaxIsOne() async throws { @@ -963,7 +1017,7 @@ import WebCrawler return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, prefix: "derrick-guest-runtime", - createArguments: { name in ["create", "--name", name, "python:3.14.7"] }, + createArguments: { name in ["create", "--name", name, DockerWorkerRuntime.image] }, createStep: "create guest runtime container", startStep: "start guest runtime container", body: { _ in diff --git a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift index cee3ef6..046e54e 100644 --- a/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift +++ b/packages/Plugin/Sources/Plugin/Factory/PluginFactoryImplementation.swift @@ -240,10 +240,14 @@ public struct PluginFactory: Sendable { } let runtimeJSON = try runtimeJSON(for: manifest) + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: draft.manifestJSON + ) var files: [String: Data] = [ "plugin.json": Data(draft.manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.go": Data(draft.guestSource.utf8), + guestPath: Data(draft.guestSource.utf8), "app.derrick/plugin": artifact, ] for (path, body) in draft.skillFiles { diff --git a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift index a8e3b7e..5fa5ade 100644 --- a/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift +++ b/packages/Plugin/Tests/PluginTests/PluginFactoryTests.swift @@ -23,6 +23,39 @@ import Testing """ } + @Test func releaseVerifiesWithEntrypointGuestPath() throws { + let runtimeJSON = #"{"entrypoint":"./app.derrick/plugin.go","language":"go"}"# + let manifestJSON = """ + {"$schema":"\(PluginContract.agentPluginSchema)","name":"slack-connector-1","version":"1.0.0",\ + "extensions":{"app.derrick":{"entrypoint":"./app.derrick/plugin.go","role":"connector"}}} + """ + let guestSource = "package main\n" + let artifact = Data("binary".utf8) + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: manifestJSON + ) + #expect(guestPath == "app.derrick/plugin.go") + let files: [String: Data] = [ + "plugin.json": Data(manifestJSON.utf8), + "app.derrick/runtime.json": Data(runtimeJSON.utf8), + guestPath: Data(guestSource.utf8), + "app.derrick/plugin": artifact, + ] + let release = PluginFactoryRelease( + pluginID: "slack-connector-1", + version: "1.0.0", + manifestJSON: manifestJSON, + runtimeJSON: runtimeJSON, + guestSource: guestSource, + compiledArtifact: artifact, + skillFiles: [:], + contentHash: PluginContentHash.hash(files: files), + reviewSummary: "ok" + ) + #expect(release.verifyIntegrity()) + } + @Test func guestLanguageIsGoFromRuntimeJSON() { let release = PluginFactoryRelease( pluginID: "slack-connection", @@ -121,9 +154,17 @@ import Testing draft: PluginFactoryDraft( manifestJSON: manifestJSON(), guestSource: """ - import json, sys - _ = json.load(sys.stdin) - print("not a plugin envelope") + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + _ = json.NewDecoder(os.Stdin).Decode(&map[string]any{}) + os.Stdout.WriteString("not a plugin envelope") + } """, testInput: Data(#"{"kind":"manual"}"#.utf8) ), @@ -132,10 +173,7 @@ import Testing ) Issue.record("Expected invalid output") } catch let error as PluginFactoryError { - #expect( - error.localizedDescription.contains("invalid plugin output") - || error.localizedDescription.contains("Python draft test failed") - ) + #expect(error.localizedDescription.contains("invalid plugin output")) #expect(await reviewer.callCount == 0) } } @@ -457,17 +495,7 @@ import Testing ) let draft = PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: """ - import json, sys - event = json.load(sys.stdin) - for item in event.get("http_results") or []: - if item.get("request_id") == "send-1": - body = json.loads(item.get("body") or "{}") - if body.get("ok"): - json.dump([{"verb":"result.emit","sent_message":{"vendor_message_id":"1.0","created_at":"1.0"}}], sys.stdout) - sys.exit(0) - json.dump([{"verb":"result.emit","summary":"failed"}], sys.stdout) - """, + guestSource: connectorGuestGoSource(), testInput: testInput, userGoal: fullSyncGoal() ) @@ -630,20 +658,47 @@ private func connectorDraft(testInput: Data) -> PluginFactoryDraft { """ return PluginFactoryDraft( manifestJSON: manifestJSON, - guestSource: """ - import json, sys - event = json.load(sys.stdin) - def emit(v): - json.dump(v, sys.stdout, separators=(",", ":")) - if event.get("http_results"): - emit([{"verb":"result.emit","sent_message":{"vendor_message_id":"1.0","created_at":"1710000001.0"}}]) - else: - emit([{"verb":"http.request","request_id":"send-1","method":"POST","url":"https://slack.com/api/chat.postMessage"}]) - """, + guestSource: connectorGuestGoSource(), testInput: testInput ) } +private func connectorGuestGoSource() -> String { + """ + package main + + import ( + "encoding/json" + "os" + ) + + func main() { + var event map[string]any + _ = json.NewDecoder(os.Stdin).Decode(&event) + if _, ok := event["http_results"]; ok { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{ + "verb": "result.emit", + "sent_message": map[string]any{ + "vendor_message_id": "1.0", + "created_at": "1710000001.0", + }, + }}) + return + } + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + _ = enc.Encode([]map[string]any{{ + "verb": "http.request", + "request_id": "send-1", + "method": "POST", + "url": "https://slack.com/api/chat.postMessage", + }}) + } + """ +} + private actor SequenceFactoryBuilder: PluginFactoryBuilder { let drafts: [PluginFactoryDraft] private(set) var callCount = 0 diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift b/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift index 7dace2e..5e46fd3 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/DerrickBundledText.swift @@ -50,7 +50,7 @@ public enum DerrickBundledText: Sendable { public static func formatCodeForModel( _ source: String, heading: String, - language: String = "python" + language: String = "go" ) -> String { """ # \(heading) diff --git a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift index d7296f4..cb0c596 100644 --- a/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift +++ b/packages/Structure/Sources/AppLayerServices/AppServices/ServiceHealth.swift @@ -14,9 +14,6 @@ public enum DerrickGuestRuntime: Sendable { /// Unified Go worker image for offline guests (`script_exec` / `plugin.invoke`). public static let guestDockerImage = DockerWorkerRuntime.image - - /// Legacy Python image reported by older daemons. Hygiene retires a mismatch. - public static let legacyPythonGuestDockerImage = "python:3.14.7" } public struct ServiceHealthReport: Codable, Sendable, Hashable { diff --git a/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift b/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift index d8e8e74..eab2002 100644 --- a/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift +++ b/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift @@ -38,4 +38,7 @@ public enum InProcessServiceBridges: Sendable { public typealias RouteMessagingAgent = @Sendable (MessagingAgentRoute) async throws -> Void nonisolated(unsafe) public static var messagingAgentRoute: RouteMessagingAgent? + + public typealias RunNewsReader = @Sendable (Data) async throws -> NewsReaderRunResult + nonisolated(unsafe) public static var runNewsReader: RunNewsReader? } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift index 5d3808f..9d73214 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift @@ -19,6 +19,8 @@ import CryptoKit func callTool(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) /// Signed `searchTools` envelope. Reply is `MCPToolSearchResultDTO`. func searchTools(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) + /// Runs the Docker news reader worker (stdin JSON request). Reply is `NewsReaderRunResult`. + func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) } public struct MCPServiceBootstrapResult: Codable, Sendable, Hashable { @@ -348,4 +350,12 @@ public enum MCPServiceXPCCodec { public static func decodeString(_ data: Data) -> String { String(data: data, encoding: .utf8) ?? "" } + + public static func encodeNewsReaderRunResult(_ result: NewsReaderRunResult) throws -> Data { + try JSONEncoder.service.encode(result) + } + + public static func decodeNewsReaderRunResult(_ data: Data) throws -> NewsReaderRunResult { + try JSONDecoder.service.decode(NewsReaderRunResult.self, from: data) + } } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift index 94eec53..ed48b99 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift @@ -12,6 +12,9 @@ public enum MCPToolCallTimeouts { /// Must be at least as long as `MCPServiceDockerHelperRunner` call timeout. public static let pluginInvokeNanoseconds: UInt64 = 120_000_000_000 + /// News reader Docker worker (RSS fetch + optional summary prep). + public static let newsReaderNanoseconds: UInt64 = 200_000_000_000 + public static func nanoseconds(forToolName toolName: String) -> UInt64 { switch toolName { case "web.crawl", "plugin_factory_build", "script_exec": diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift index 6452b0e..923fdff 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift @@ -93,6 +93,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { public let description: String public let pluginID: String? public let auth: ConnectorAuthDiscovery? + public let skillMarkdown: String? public init( pluginType: PluginType, @@ -101,7 +102,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { scope: ConnectorScope = .fullSync, description: String, pluginID: String? = nil, - auth: ConnectorAuthDiscovery? = nil + auth: ConnectorAuthDiscovery? = nil, + skillMarkdown: String? = nil ) { self.pluginType = pluginType self.vendor = vendor @@ -109,6 +111,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { self.scope = scope self.pluginID = pluginID?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty self.auth = auth + self.skillMarkdown = skillMarkdown?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty self.description = Self.resolvedDescription( userDescription: description, vendor: vendor, @@ -117,6 +120,39 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { ) } + public static func makeFromSkillDraft( + _ draft: PluginSkillDraft, + auth: ConnectorAuthDiscovery? = nil + ) throws -> PluginFactoryCreateInput { + let pluginID = try draft.normalizedPluginID() + let description = draft.factoryDescription() + let skillMarkdown = draft.skillMarkdown() + switch draft.plannedKind { + case .messagingConnector: + guard let vendor = draft.inferredConnectorVendor else { + throw PluginSkillDraftError.missingConnectorVendor + } + return PluginFactoryCreateInput( + pluginType: .connector, + vendor: vendor, + scope: .fullSync, + description: description, + pluginID: pluginID, + auth: auth, + skillMarkdown: skillMarkdown + ) + case .customCapability: + return PluginFactoryCreateInput( + pluginType: .custom, + description: description, + pluginID: pluginID, + skillMarkdown: skillMarkdown + ) + case .newsDigest: + throw PluginSkillDraftError.newsUsesReaderPath + } + } + /// Builds connector workflow input. The factory goal uses the fixed scope sentence, not free-text extras. public static func makeConnector( vendor: ConnectorVendor, @@ -137,7 +173,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { scope: scope, description: userDescription, pluginID: resolvedID, - auth: auth ?? (try? ConnectorAuthDiscovery.slackBotTokenFallback()) + auth: auth ?? (try? ConnectorAuthDiscovery.slackBotTokenFallback()), + skillMarkdown: nil ) } @@ -182,6 +219,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { case description case pluginID case auth + case skillMarkdown } public init(from decoder: Decoder) throws { @@ -195,6 +233,8 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { pluginID = try container.decodeIfPresent(String.self, forKey: .pluginID)? .trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty auth = try container.decodeIfPresent(ConnectorAuthDiscovery.self, forKey: .auth) + skillMarkdown = try container.decodeIfPresent(String.self, forKey: .skillMarkdown)? + .trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty description = Self.resolvedDescription( userDescription: rawDescription, vendor: vendor, @@ -212,6 +252,7 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { try container.encode(description, forKey: .description) try container.encodeIfPresent(pluginID, forKey: .pluginID) try container.encodeIfPresent(auth, forKey: .auth) + try container.encodeIfPresent(skillMarkdown, forKey: .skillMarkdown) } public func encodedJSON() throws -> String { @@ -247,6 +288,9 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { extra.append("Host permission labels: \(auth.permissions.joined(separator: ", "))") } } + if let skillMarkdown, !skillMarkdown.isEmpty { + extra.append("SKILL.md draft:\n\(skillMarkdown)") + } extra.append( "The host writes plugin.json. Return go_source and test_input_json only. Do not invent a plugin_id or secrets list." ) @@ -267,31 +311,46 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { } } - /// Failure stage hint for returning the wizard to the right step. + public func customBuildGoal() -> String { + var lines = [ + "Create an Agent Plugin capability.", + description, + ] + if let skillMarkdown, !skillMarkdown.isEmpty { + lines.append("SKILL.md draft:\n\(skillMarkdown)") + } + lines.append( + "Return go_source, test_input_json, and skill_files. Include a valid plugin.json via the builder contract when no host manifest is supplied." + ) + return lines.joined(separator: "\n\n") + } + + /// Failure stage hint for returning the plugin studio to the right step. public enum FailureStep: String, Sendable { - case type - case vendor - case name - case auth + case goal + case skill + case preview + case credentials + case build case news - case description - case creating } public static func failureStep(forStage stage: String?) -> FailureStep { switch stage?.lowercased() { - case "type": - return .type - case "name": - return .name - case "auth", "discover": - return .auth + case "goal": + return .goal + case "skill", "name", "description", "type", "vendor": + return .skill + case "preview": + return .preview + case "auth", "discover", "credentials": + return .credentials case "news", "paywall": return .news - case "crawl", "docs", "vendor", "factory", "build", "review", "description": - return .vendor + case "crawl", "docs", "factory", "build", "review": + return .build default: - return .creating + return .build } } } diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift b/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift deleted file mode 100644 index ebfd4f9..0000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsFeedParser.swift +++ /dev/null @@ -1,154 +0,0 @@ -import Foundation - -public enum NewsFeedParser { - public static func parse(data: Data, sourceLabel: String, fallbackPageURL: URL) -> [ParsedNewsEntry] { - let text = String(data: data, encoding: .utf8) - ?? String(data: data, encoding: .isoLatin1) - ?? "" - if NewsPaywall.looksLikeFeed(text) { - return parseXMLFeed(text, sourceLabel: sourceLabel) - } - if let entry = htmlFallback(text: text, sourceLabel: sourceLabel, url: fallbackPageURL) { - return [entry] - } - return [] - } - - public struct ParsedNewsEntry: Sendable, Hashable { - public var title: String - public var sourceURL: String - public var summary: String? - public var publishedAt: Date? - - public init(title: String, sourceURL: String, summary: String? = nil, publishedAt: Date? = nil) { - self.title = title - self.sourceURL = sourceURL - self.summary = summary - self.publishedAt = publishedAt - } - } - - private static func parseXMLFeed(_ xml: String, sourceLabel: String) -> [ParsedNewsEntry] { - _ = sourceLabel - var entries: [ParsedNewsEntry] = [] - let itemBlocks = slices(of: xml, start: "") - + slices(of: xml, start: "") - for block in itemBlocks { - let title = firstTag(block, names: ["title"]) ?? "" - let link = firstTag(block, names: ["link"]) - ?? attribute(named: "href", in: firstRawTag(block, name: "link") ?? "") - ?? firstTag(block, names: ["guid", "id"]) - ?? "" - let summary = firstTag(block, names: ["description", "summary", "content"]) - let dateText = firstTag(block, names: ["pubDate", "published", "updated", "dc:date"]) - let cleanedTitle = stripTags(title).trimmingCharacters(in: .whitespacesAndNewlines) - let cleanedLink = stripTags(link).trimmingCharacters(in: .whitespacesAndNewlines) - guard !cleanedTitle.isEmpty, let url = URL(string: cleanedLink), url.scheme != nil else { - continue - } - entries.append( - ParsedNewsEntry( - title: cleanedTitle, - sourceURL: url.absoluteString, - summary: summary.map(stripTags).flatMap { $0.isEmpty ? nil : $0 }, - publishedAt: parseDate(dateText) - ) - ) - } - return entries - } - - private static func htmlFallback(text: String, sourceLabel: String, url: URL) -> ParsedNewsEntry? { - _ = sourceLabel - let title = firstTag(text, names: ["title"]) - .map(stripTags)? - .trimmingCharacters(in: .whitespacesAndNewlines) - guard let title, !title.isEmpty else { return nil } - return ParsedNewsEntry(title: title, sourceURL: url.absoluteString, summary: nil, publishedAt: nil) - } - - private static func slices(of text: String, start: String, end: String) -> [String] { - var result: [String] = [] - let startLower = start.lowercased() - let endLower = end.lowercased() - let lower = text.lowercased() - var idx = lower.startIndex - while let startRange = lower[idx...].range(of: startLower) { - guard let endRange = lower[startRange.upperBound...].range(of: endLower) else { break } - let sliceStart = startRange.lowerBound - let sliceEnd = endRange.upperBound - result.append(String(text[sliceStart.. String? { - for name in names { - let lower = xml.lowercased() - let open = "<\(name.lowercased())" - guard let openStart = lower.range(of: open) else { continue } - guard let tagClose = xml[openStart.upperBound...].firstIndex(of: ">") else { continue } - let innerStart = xml.index(after: tagClose) - let closeToken = "" - guard let close = lower[innerStart...].range(of: closeToken) else { continue } - return String(xml[innerStart.. String? { - let lower = xml.lowercased() - let open = "<\(name.lowercased())" - guard let openStart = lower.range(of: open) else { return nil } - guard let tagClose = xml[openStart.upperBound...].firstIndex(of: ">") else { return nil } - return String(xml[openStart.lowerBound...tagClose]) - } - - private static func attribute(named name: String, in tag: String) -> String? { - let pattern = "\(name)\\s*=\\s*\"([^\"]+)\"" - guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) else { - return nil - } - let range = NSRange(tag.startIndex.. 1, - let inner = Range(match.range(at: 1), in: tag) - else { - return nil - } - return String(tag[inner]) - } - - private static func stripTags(_ raw: String) -> String { - var value = raw.replacingOccurrences(of: "<[^>]+>", with: "", options: .regularExpression) - let entities: [(String, String)] = [ - ("&", "&"), - ("<", "<"), - (">", ">"), - (""", "\""), - ("'", "'"), - ("'", "'"), - (" ", " "), - ] - for (from, to) in entities { - value = value.replacingOccurrences(of: from, with: to) - } - return value - } - - private static func parseDate(_ raw: String?) -> Date? { - guard let raw, !raw.isEmpty else { return nil } - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - let rfc = DateFormatter() - rfc.locale = Locale(identifier: "en_US_POSIX") - rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss Z" - if let date = rfc.date(from: trimmed) { return date } - rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss zzz" - if let date = rfc.date(from: trimmed) { return date } - let iso = ISO8601DateFormatter() - iso.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - if let date = iso.date(from: trimmed) { return date } - iso.formatOptions = [.withInternetDateTime] - return iso.date(from: trimmed) - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift index f09967e..57ce652 100644 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift +++ b/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift @@ -1,13 +1,26 @@ import Foundation public enum NewsReaderMode: String, Codable, Sendable, Hashable, CaseIterable { + case rss case list - case summaries + case summary public var displayName: String { switch self { - case .list: return "List articles" - case .summaries: return "Summaries" + case .rss: return "RSS feed" + case .list: return "Crawl site" + case .summary: return "AI summary" + } + } + + public init(from decoder: Decoder) throws { + let raw = try decoder.singleValueContainer().decode(String.self) + switch raw { + case "rss": self = .rss + case "list": self = .list + case "summary", "summaries": self = .summary + default: + self = .rss } } } @@ -46,6 +59,7 @@ public struct NewsReaderSpec: Codable, Sendable, Hashable, Identifiable { public var mode: NewsReaderMode public var maxCount: Int public var schedule: NewsReaderSchedule + public var summaryText: String? public var lastError: String? public var lastFetchedAt: Date? public var createdAt: Date @@ -56,9 +70,10 @@ public struct NewsReaderSpec: Codable, Sendable, Hashable, Identifiable { name: String, topics: [String], sources: [NewsSource], - mode: NewsReaderMode = .list, + mode: NewsReaderMode = .rss, maxCount: Int = 20, schedule: NewsReaderSchedule = .off, + summaryText: String? = nil, lastError: String? = nil, lastFetchedAt: Date? = nil, createdAt: Date = .now, @@ -71,6 +86,7 @@ public struct NewsReaderSpec: Codable, Sendable, Hashable, Identifiable { self.mode = mode self.maxCount = min(50, max(1, maxCount)) self.schedule = schedule + self.summaryText = summaryText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty self.lastError = lastError self.lastFetchedAt = lastFetchedAt self.createdAt = createdAt @@ -132,30 +148,46 @@ public enum NewsPresetTopic: String, Sendable, CaseIterable, Identifiable { } public enum NewsPresetSource: String, Sendable, CaseIterable, Identifiable { - case bbcWorld - case npr - case bbcTech - case hn case googleNews + case foxNews + case newsmax + case nationalReview + case wsj public var id: String { rawValue } public var source: NewsSource { switch self { - case .bbcWorld: - return NewsSource(id: rawValue, label: "BBC World", url: "https://feeds.bbci.co.uk/news/world/rss.xml") - case .npr: - return NewsSource(id: rawValue, label: "NPR", url: "https://feeds.npr.org/1001/rss.xml") - case .bbcTech: - return NewsSource(id: rawValue, label: "BBC Technology", url: "https://feeds.bbci.co.uk/news/technology/rss.xml") - case .hn: - return NewsSource(id: rawValue, label: "Hacker News", url: "https://hnrss.org/frontpage") case .googleNews: return NewsSource( id: rawValue, label: "Google News", url: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" ) + case .foxNews: + return NewsSource( + id: rawValue, + label: "Fox News", + url: "https://moxie.foxnews.com/google-publisher/latest.xml" + ) + case .newsmax: + return NewsSource( + id: rawValue, + label: "Newsmax", + url: "https://www.newsmax.com/rss/Newsfront/" + ) + case .nationalReview: + return NewsSource( + id: rawValue, + label: "National Review", + url: "https://www.nationalreview.com/feed/" + ) + case .wsj: + return NewsSource( + id: rawValue, + label: "Wall Street Journal", + url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" + ) } } } @@ -167,6 +199,8 @@ public enum NewsReaderError: Error, Sendable, Equatable, LocalizedError { case emptyName case fetchFailed(url: String, detail: String) case notReady + case summarizerUnavailable + case workerUnavailable(String) public var errorDescription: String? { switch self { @@ -182,6 +216,16 @@ public enum NewsReaderError: Error, Sendable, Equatable, LocalizedError { return "Could not read \(url). \(detail)" case .notReady: return "News lists are not ready yet. Try again in a moment." + case .summarizerUnavailable: + return "Add an API key in Settings before creating an AI summary list." + case .workerUnavailable(let detail): + return "News reader worker is not available. \(detail)" } } } + +private extension String { + var nilIfEmpty: String? { + isEmpty ? nil : self + } +} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift index 37c362f..aba438e 100644 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift +++ b/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift @@ -1,139 +1,108 @@ import Foundation -public protocol NewsHTTPClient: Sendable { - func get(url: URL) async throws -> NewsHTTPResponse -} - -public struct NewsHTTPResponse: Sendable { - public var status: Int - public var contentType: String? - public var body: Data - - public init(status: Int, contentType: String?, body: Data) { - self.status = status - self.contentType = contentType - self.body = body - } -} - -public struct URLSessionNewsHTTPClient: NewsHTTPClient { - public init() {} - - public func get(url: URL) async throws -> NewsHTTPResponse { - var request = URLRequest(url: url) - request.httpMethod = "GET" - request.timeoutInterval = 20 - request.setValue( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15", - forHTTPHeaderField: "User-Agent" - ) - request.setValue("application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.8", forHTTPHeaderField: "Accept") - let (data, response) = try await URLSession.shared.data(for: request) - let http = response as? HTTPURLResponse - return NewsHTTPResponse( - status: http?.statusCode ?? 0, - contentType: http?.value(forHTTPHeaderField: "Content-Type"), - body: data - ) - } -} - public enum NewsReaderRefresh { public static func validateAndFetch( spec: NewsReaderSpec, - client: any NewsHTTPClient - ) async throws -> [NewsItem] { + worker: any NewsWorkerRunning, + summarizer: NewsSummaryGenerating? = nil + ) async throws -> NewsReaderFetchResult { let name = spec.name.trimmingCharacters(in: .whitespacesAndNewlines) guard !name.isEmpty else { throw NewsReaderError.emptyName } guard !spec.sources.isEmpty else { throw NewsReaderError.emptySources } - var collected: [NewsItem] = [] for source in spec.sources { - let parsed = try await fetchSource(source, readerID: spec.id, client: client) - collected.append(contentsOf: parsed) - } - - let filtered = filter(collected, topics: spec.topics) - let unique = uniqued(filtered) - let sorted = unique.sorted { lhs, rhs in - (lhs.publishedAt ?? lhs.fetchedAt) > (rhs.publishedAt ?? rhs.fetchedAt) - } - return Array(sorted.prefix(spec.maxCount)) - } - - public static func digest(from items: [NewsItem]) -> String { - let lines = items.prefix(8).map { item in - "• \(item.title) (\(item.sourceLabel))" + guard let url = URL(string: source.url), url.scheme == "http" || url.scheme == "https" else { + throw NewsReaderError.invalidURL(source.url) + } + if let reason = NewsPaywall.preflightRejection(url: url) { + throw NewsReaderError.paywalled(url: source.url, detail: reason) + } } - return lines.joined(separator: "\n") - } - private static func fetchSource( - _ source: NewsSource, - readerID: String, - client: any NewsHTTPClient - ) async throws -> [NewsItem] { - guard let url = URL(string: source.url), url.scheme == "http" || url.scheme == "https" else { - throw NewsReaderError.invalidURL(source.url) - } - let fetchURL = NewsSourceURL.canonicalFetchURL(url) - if let reason = NewsPaywall.preflightRejection(url: fetchURL) { - throw NewsReaderError.paywalled(url: source.url, detail: reason) - } - let response: NewsHTTPResponse - do { - response = try await client.get(url: fetchURL) - } catch { - throw NewsReaderError.fetchFailed(url: source.url, detail: error.localizedDescription) - } - if let reason = NewsPaywall.rejectionReason( - url: fetchURL, - status: response.status, - contentType: response.contentType, - body: response.body - ) { - throw NewsReaderError.paywalled(url: source.url, detail: reason) - } - if response.status != 0, response.status < 200 || response.status >= 400 { - throw NewsReaderError.fetchFailed(url: source.url, detail: "HTTP \(response.status)") - } - let entries = NewsFeedParser.parse(data: response.body, sourceLabel: source.label, fallbackPageURL: fetchURL) - guard !entries.isEmpty else { + let request = NewsReaderWorkerRequest( + mode: spec.mode, + sources: spec.sources, + topics: spec.topics, + maxCount: spec.maxCount, + contextHint: ([spec.name] + spec.topics).joined(separator: " ") + ) + let requestJSON = try request.encodedJSON() + let stdout = try await worker.run(requestJSON: requestJSON) + let result = try JSONDecoder.service.decode(NewsReaderWorkerResult.self, from: stdout) + guard result.ok else { + let detail = result.diagnostics.joined(separator: " ") throw NewsReaderError.fetchFailed( - url: source.url, - detail: "No articles were found. For Google News, Derrick uses the public RSS feed." + url: spec.sources.first?.url ?? name, + detail: detail.isEmpty ? "News reader returned no articles." : detail ) } - return entries.map { entry in + + let items = result.articles.map { article in NewsItem( - readerID: readerID, - title: entry.title, - sourceURL: entry.sourceURL, - sourceLabel: source.label, - summary: entry.summary, - publishedAt: entry.publishedAt + readerID: spec.id, + title: article.title, + sourceURL: article.url, + sourceLabel: sourceLabel(for: article.url, sources: spec.sources), + summary: article.detail?.nilIfEmpty, + publishedAt: parsePublishedAt(article.publishedAt) ) } - } - private static func filter(_ items: [NewsItem], topics: [String]) -> [NewsItem] { - let needles = topics.map { $0.lowercased() }.filter { !$0.isEmpty } - guard !needles.isEmpty else { return items } - let matched = items.filter { item in - let hay = "\(item.title) \(item.summary ?? "")".lowercased() - return needles.contains { hay.contains($0) } + var summaryText: String? + if spec.mode == .summary { + guard let summarizer else { + throw NewsReaderError.summarizerUnavailable + } + summaryText = try await summarizer.summarize( + listName: spec.name, + topics: spec.topics, + articles: items + ) } - return matched.isEmpty ? items : matched + + return NewsReaderFetchResult(items: items, summaryText: summaryText) } - private static func uniqued(_ items: [NewsItem]) -> [NewsItem] { - var seen = Set() - var result: [NewsItem] = [] - for item in items { - if seen.insert(item.sourceURL).inserted { - result.append(item) + private static func sourceLabel(for url: String, sources: [NewsSource]) -> String { + if let host = URL(string: url)?.host { + if let match = sources.first(where: { URL(string: $0.url)?.host == host }) { + return match.label } + return host } - return result + return sources.first?.label ?? "Source" + } + + private static func parsePublishedAt(_ raw: String?) -> Date? { + guard let raw, !raw.isEmpty else { return nil } + let rfc = DateFormatter() + rfc.locale = Locale(identifier: "en_US_POSIX") + rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss Z" + if let date = rfc.date(from: raw) { return date } + let iso = ISO8601DateFormatter() + iso.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + if let date = iso.date(from: raw) { return date } + iso.formatOptions = [.withInternetDateTime] + return iso.date(from: raw) + } +} + +public struct NewsReaderFetchResult: Sendable, Hashable { + public var items: [NewsItem] + public var summaryText: String? + + public init(items: [NewsItem], summaryText: String? = nil) { + self.items = items + self.summaryText = summaryText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty + } +} + +public protocol NewsSummaryGenerating: Sendable { + func summarize(listName: String, topics: [String], articles: [NewsItem]) async throws -> String +} + +private extension String { + var nilIfEmpty: String? { + isEmpty ? nil : self } } diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift new file mode 100644 index 0000000..42ddc0e --- /dev/null +++ b/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift @@ -0,0 +1,72 @@ +import Foundation + +public protocol NewsWorkerRunning: Sendable { + func run(requestJSON: Data) async throws -> Data +} + +public struct NewsReaderWorkerRequest: Codable, Sendable, Hashable { + public var mode: NewsReaderMode + public var sources: [NewsSource] + public var topics: [String] + public var maxCount: Int + public var contextHint: String? + + public init( + mode: NewsReaderMode, + sources: [NewsSource], + topics: [String], + maxCount: Int, + contextHint: String? = nil + ) { + self.mode = mode + self.sources = sources + self.topics = topics + self.maxCount = maxCount + self.contextHint = contextHint?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty + } + + public func encodedJSON() throws -> Data { + try JSONEncoder.service.encode(self) + } +} + +public struct NewsReaderWorkerArticle: Codable, Sendable, Hashable { + public var title: String + public var url: String + public var detail: String? + public var publishedAt: String? + + enum CodingKeys: String, CodingKey { + case title + case url + case detail + case publishedAt = "published_at" + } +} + +public struct NewsReaderWorkerResult: Codable, Sendable, Hashable { + public var ok: Bool + public var mode: NewsReaderMode + public var articles: [NewsReaderWorkerArticle] + public var diagnostics: [String] +} + +public struct NewsReaderRunResult: Codable, Sendable, Hashable { + public var ok: Bool + public var stdout: Data + public var stderr: Data + public var message: String + + public init(ok: Bool, stdout: Data = Data(), stderr: Data = Data(), message: String = "") { + self.ok = ok + self.stdout = stdout + self.stderr = stderr + self.message = message + } +} + +private extension String { + var nilIfEmpty: String? { + isEmpty ? nil : self + } +} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift b/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift new file mode 100644 index 0000000..0d082d2 --- /dev/null +++ b/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift @@ -0,0 +1,30 @@ +import Foundation + +public enum NewsWorkerBridge: Sendable { + public typealias Runner = @Sendable (Data) async throws -> Data + + private final class Storage: @unchecked Sendable { + var runner: Runner? + } + + private static let storage = Storage() + + public static func install(_ runner: @escaping Runner) { + storage.runner = runner + } + + public static func run(requestJSON: Data) async throws -> Data { + guard let runner = storage.runner else { + throw NewsReaderError.workerUnavailable("Docker news reader is not ready yet.") + } + return try await runner(requestJSON) + } +} + +public struct BridgedNewsWorker: NewsWorkerRunning { + public init() {} + + public func run(requestJSON: Data) async throws -> Data { + try await NewsWorkerBridge.run(requestJSON: requestJSON) + } +} diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift new file mode 100644 index 0000000..bcdc13f --- /dev/null +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift @@ -0,0 +1,447 @@ +import Foundation + +/// User-authored Agent Plugin intent before the factory materializes plugin.json, SKILL.md, and guest code. +public struct PluginSkillDraft: Sendable, Hashable { + public enum Trigger: String, Sendable, CaseIterable, Codable, Hashable { + case chat + case messaging + case schedule + case mention + + public var label: String { + switch self { + case .chat: return "When I ask in chat" + case .messaging: return "From Messaging" + case .schedule: return "On a schedule" + case .mention: return "When I type /plugin-name" + } + } + } + + public struct Example: Sendable, Hashable, Identifiable { + public var id: String + public var userSays: String + public var pluginDoes: String + + public init(id: String = UUID().uuidString, userSays: String, pluginDoes: String) { + self.id = id + self.userSays = userSays + self.pluginDoes = pluginDoes + } + } + + public enum PlannedKind: String, Sendable, Hashable { + case messagingConnector + case newsDigest + case customCapability + } + + public var goal: String + public var purpose: String + public var triggers: Set + public var examples: [Example] + public var pluginName: String + public var newsTopics: [String] + public var newsSourceURLs: [String] + + public init( + goal: String = "", + purpose: String = "", + triggers: Set = [.chat], + examples: [Example] = [], + pluginName: String = "", + newsTopics: [String] = [], + newsSourceURLs: [String] = [] + ) { + self.goal = goal + self.purpose = purpose + self.triggers = triggers + self.examples = examples + self.pluginName = pluginName + self.newsTopics = newsTopics + self.newsSourceURLs = newsSourceURLs + } + + public var plannedKind: PlannedKind { + PluginSkillDraftPlanner.inferKind(from: self) + } + + public var inferredConnectorVendor: PluginFactoryCreateInput.ConnectorVendor? { + PluginSkillDraftPlanner.inferConnectorVendor(from: self) + } + + public var isBuildable: Bool { + switch plannedKind { + case .messagingConnector: + return inferredConnectorVendor?.isSelectableInWizard == true + case .newsDigest, .customCapability: + return true + } + } + + public var buildBlockedReason: String? { + if plannedKind == .messagingConnector, + inferredConnectorVendor?.isSelectableInWizard != true { + let label = inferredConnectorVendor?.displayName ?? "That service" + return "\(label) messaging connectors are not available yet. Try Slack or describe a custom capability." + } + return nil + } + + public func isTriggerAvailable(_ trigger: Trigger) -> Bool { + PluginSkillDraftPlanner.availableTriggers(for: plannedKind).contains(trigger) + } + + public func skillMarkdown() -> String { + PluginSkillDraftPlanner.skillMarkdown(for: self) + } + + public func previewScenarios() -> [String] { + examples.map { example in + "When you say “\(example.userSays)”, the plugin will \(example.pluginDoes)." + } + } + + public func packageOutline() -> [String] { + switch plannedKind { + case .newsDigest: + return [ + "plugin.json — name, schedule, and reader settings", + "skills/\(normalizedPluginFolderName())/SKILL.md — when Derrick uses this list", + "News fetcher — loads articles with source links", + ] + case .messagingConnector, .customCapability: + return [ + "plugin.json — name, permissions, and secrets", + "skills/\(normalizedPluginFolderName())/SKILL.md — purpose and examples", + "app.derrick/plugin.go — guest program (compiled in Docker)", + "app.derrick/plugin — compiled binary", + ] + } + } + + public func factoryDescription() -> String { + PluginSkillDraftPlanner.factoryDescription(for: self) + } + + public func normalizedPluginID() throws -> String { + try PluginID.normalized(pluginName).rawValue + } + + private func normalizedPluginFolderName() -> String { + let trimmed = pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return "plugin" } + return trimmed + .lowercased() + .replacingOccurrences(of: #"[^a-z0-9]+"#, with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + } +} + +public enum PluginSkillDraftPlanner { + public static func availableTriggers( + for kind: PluginSkillDraft.PlannedKind + ) -> Set { + switch kind { + case .newsDigest: + return [.chat, .schedule] + case .messagingConnector: + return [.chat, .messaging, .mention] + case .customCapability: + return [.chat, .mention, .schedule] + } + } + + public static func sanitizeTriggers(in draft: inout PluginSkillDraft) { + let allowed = availableTriggers(for: draft.plannedKind) + draft.triggers = draft.triggers.intersection(allowed) + if draft.triggers.isEmpty { + draft.triggers = defaultTriggers(for: draft) + } + } + + public static func inferNewsMode(from draft: PluginSkillDraft) -> NewsReaderMode { + let text = combinedText(draft) + if ["summary", "summarize", "summaries", "digest", "brief", "overview"] + .contains(where: { text.contains($0) }) { + return .summary + } + if ["crawl", "website", "homepage", "web page", "webpage", "site"] + .contains(where: { text.contains($0) }) { + return .list + } + return .rss + } + + public static func inferKind(from draft: PluginSkillDraft) -> PluginSkillDraft.PlannedKind { + let text = combinedText(draft) + if looksLikeNews(text) { return .newsDigest } + if looksLikeMessaging(text) { return .messagingConnector } + return .customCapability + } + + public static func inferConnectorVendor( + from draft: PluginSkillDraft + ) -> PluginFactoryCreateInput.ConnectorVendor? { + let text = combinedText(draft) + if text.contains("slack") { return .slack } + if text.contains("telegram") { return .telegram } + if text.contains("whatsapp") { return .whatsapp } + if text.contains("discord") { return .discord } + if inferKind(from: draft) == .messagingConnector { return .slack } + return nil + } + + public static func applyGoal(_ goal: String, to draft: inout PluginSkillDraft, existingPluginIDs: [String]) { + let trimmed = goal.trimmingCharacters(in: .whitespacesAndNewlines) + draft.goal = trimmed + if draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + draft.purpose = trimmed + } + if draft.pluginName.isEmpty { + draft.pluginName = suggestPluginName(for: draft, existingIDs: existingPluginIDs) + } + if draft.examples.isEmpty { + draft.examples = defaultExamples(for: draft) + } + if draft.triggers.isEmpty { + draft.triggers = defaultTriggers(for: draft) + } + sanitizeTriggers(in: &draft) + if inferKind(from: draft) == .newsDigest, draft.newsTopics.isEmpty { + draft.newsTopics = defaultNewsTopics(from: trimmed) + } + if inferKind(from: draft) == .newsDigest, draft.newsSourceURLs.isEmpty { + draft.newsSourceURLs = [NewsPresetSource.googleNews.source.url] + } + } + + public static func skillMarkdown(for draft: PluginSkillDraft) -> String { + let name = draft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + let triggerLines = draft.triggers.sorted { $0.rawValue < $1.rawValue }.map(\.label) + let exampleBlock = draft.examples.map { example in + """ + ### User + \(example.userSays) + + ### Plugin + \(example.pluginDoes) + """ + }.joined(separator: "\n\n") + + return """ + # \(name.isEmpty ? "Plugin" : name) + + ## Purpose + \(draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines)) + + ## When to use + \(triggerLines.map { "- \($0)" }.joined(separator: "\n")) + + ## Examples + \(exampleBlock.isEmpty ? "_Add at least one example before building._" : exampleBlock) + """ + } + + public static func factoryDescription(for draft: PluginSkillDraft) -> String { + let purpose = draft.purpose.trimmingCharacters(in: .whitespacesAndNewlines) + let examples = draft.examples + .map { "User: \($0.userSays) → Plugin: \($0.pluginDoes)" } + .joined(separator: "\n") + switch draft.plannedKind { + case .messagingConnector: + let vendor = inferConnectorVendor(from: draft)?.displayName ?? "messaging" + return """ + \(purpose) + + Messaging connector for \(vendor). List conversations as tabs, load messages including reply threads, and send messages. + + Confirmed behavior: + \(examples) + """ + case .newsDigest: + return """ + \(purpose) + + News reader that fetches articles from configured sources and includes source links. + + Topics: \(draft.newsTopics.joined(separator: ", ")) + Sources: \(draft.newsSourceURLs.joined(separator: ", ")) + """ + case .customCapability: + return """ + \(purpose) + + Confirmed behavior: + \(examples) + """ + } + } + + public static func factoryGoal( + for draft: PluginSkillDraft, + crawlSummary: String?, + hostNotes: [String] + ) throws -> String { + let description = factoryDescription(for: draft) + switch draft.plannedKind { + case .messagingConnector: + guard let vendor = inferConnectorVendor(from: draft) else { + throw PluginSkillDraftError.missingConnectorVendor + } + var extra = hostNotes + extra.append("SKILL.md draft:\n\(skillMarkdown(for: draft))") + return try ConnectorContractPrompts.factoryGoal( + vendorLabel: vendor.displayName, + scope: .fullSync, + vendor: vendor, + crawlSummary: crawlSummary, + reference: extra.joined(separator: "\n"), + includeVendorBindings: true + ) + case .customCapability: + return """ + Create an Agent Plugin for this user goal. + + \(description) + + Host notes: + \(hostNotes.joined(separator: "\n")) + + SKILL.md draft (write this into skills/): + \(skillMarkdown(for: draft)) + + Return go_source, test_input_json, and skill_files. The host writes plugin.json when a host manifest is supplied; otherwise include a valid manifest in your output path via the builder contract. + """ + case .newsDigest: + throw PluginSkillDraftError.newsUsesReaderPath + } + } + + private static func combinedText(_ draft: PluginSkillDraft) -> String { + [draft.goal, draft.purpose, draft.pluginName] + .joined(separator: " ") + .lowercased() + } + + private static func looksLikeNews(_ text: String) -> Bool { + ["news", "rss", "headline", "digest", "articles", "reader"].contains { text.contains($0) } + } + + private static func looksLikeMessaging(_ text: String) -> Bool { + ["slack", "telegram", "whatsapp", "discord", "messaging", "channel", "inbox", "dm", "chat app"] + .contains { text.contains($0) } + } + + private static func suggestPluginName(for draft: PluginSkillDraft, existingIDs: [String]) -> String { + switch inferKind(from: draft) { + case .messagingConnector: + if let vendor = inferConnectorVendor(from: draft) { + return ConnectorPluginNaming.defaultPluginID(vendor: vendor, existingIDs: existingIDs) + } + return ConnectorPluginNaming.defaultPluginID(vendor: .slack, existingIDs: existingIDs) + case .newsDigest: + let base = "news-list" + if !existingIDs.contains(base) { return base } + var index = 2 + while existingIDs.contains("\(base)-\(index)") { index += 1 } + return "\(base)-\(index)" + case .customCapability: + let words = draft.goal + .lowercased() + .split { !$0.isLetter && !$0.isNumber } + .filter { $0.count > 2 } + .prefix(3) + let stem = words.isEmpty ? "custom-plugin" : String(words.joined(separator: "-")) + if !existingIDs.contains(stem) { return stem } + return "\(stem)-2" + } + } + + private static func defaultTriggers(for draft: PluginSkillDraft) -> Set { + switch inferKind(from: draft) { + case .messagingConnector: + return [.messaging, .chat] + case .newsDigest: + return [.schedule, .chat] + case .customCapability: + return [.chat] + } + } + + private static func defaultExamples(for draft: PluginSkillDraft) -> [PluginSkillDraft.Example] { + switch inferKind(from: draft) { + case .messagingConnector: + let vendor = inferConnectorVendor(from: draft)?.displayName ?? "Slack" + return [ + PluginSkillDraft.Example( + userSays: "Show my \(vendor) channels", + pluginDoes: "list conversations you can access as tabs in Messaging" + ), + PluginSkillDraft.Example( + userSays: "Send “hello” to #general", + pluginDoes: "post the message in that channel" + ), + ] + case .newsDigest: + return [ + PluginSkillDraft.Example( + userSays: "What's in my news list?", + pluginDoes: "fetch the latest articles with links to the original sources" + ), + ] + case .customCapability: + let snippet = draft.goal.trimmingCharacters(in: .whitespacesAndNewlines) + return [ + PluginSkillDraft.Example( + userSays: snippet.isEmpty ? "Do the thing I described" : snippet, + pluginDoes: "run the guest program and return a clear result" + ), + ] + } + } + + private static func defaultNewsTopics(from goal: String) -> [String] { + let lower = goal.lowercased() + var topics: [String] = [] + if lower.contains("financ") || lower.contains("market") { + topics.append(NewsPresetTopic.financial.displayName) + } + if lower.contains("tech") { + topics.append(NewsPresetTopic.tech.displayName) + } + if lower.contains("world") || lower.contains("international") { + topics.append(NewsPresetTopic.international.displayName) + } + if lower.contains("politic") { + topics.append(NewsPresetTopic.politics.displayName) + } + if lower.contains("science") { + topics.append(NewsPresetTopic.science.displayName) + } + if lower.contains("sport") { + topics.append(NewsPresetTopic.sports.displayName) + } + if topics.isEmpty { + topics.append(NewsPresetTopic.tech.displayName) + } + return topics + } +} + +public enum PluginSkillDraftError: Error, LocalizedError { + case missingConnectorVendor + case newsUsesReaderPath + case invalidPluginName + + public var errorDescription: String? { + switch self { + case .missingConnectorVendor: + return "Could not determine which messaging service this plugin targets." + case .newsUsesReaderPath: + return "News lists use the reader path, not the plugin factory." + case .invalidPluginName: + return "Choose a valid plugin name using letters, numbers, and hyphens." + } + } +} diff --git a/packages/Structure/Sources/Contract/GuestContract.swift b/packages/Structure/Sources/Contract/GuestContract.swift index fdf034a..4385f61 100644 --- a/packages/Structure/Sources/Contract/GuestContract.swift +++ b/packages/Structure/Sources/Contract/GuestContract.swift @@ -15,6 +15,7 @@ public enum GuestContract: Sendable { case workerProduct = "worker-product.schema.json" case webCrawlerResult = "web-crawler-result.schema.json" case fileExtractorResult = "file-extractor-result.schema.json" + case newsReaderResult = "news-reader-result.schema.json" case scriptExecContract = "script-exec-contract.schema.json" } diff --git a/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json new file mode 100644 index 0000000..6632851 --- /dev/null +++ b/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/news-reader-result.json", + "title": "News reader worker stdout", + "description": "JSON object written to stdout by derrick-news-reader.", + "$ref": "worker-product.schema.json#/$defs/news_reader_result" +} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json index 874f6bf..611ef5f 100644 --- a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json +++ b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://derrick.local/schemas/worker-product.json", "title": "Derrick trusted worker product contracts", - "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler, file extractor, and news reader). Swift host and Go workers must match these schemas.", "$defs": { "string_list": { "type": "array", @@ -92,6 +92,35 @@ }, "diagnostics": { "$ref": "#/$defs/string_list" } } + }, + "news_reader_mode": { + "type": "string", + "enum": ["rss", "list", "summary"] + }, + "news_reader_article": { + "type": "object", + "required": ["title", "url"], + "additionalProperties": false, + "properties": { + "title": { "type": "string" }, + "url": { "type": "string" }, + "detail": { "type": "string" }, + "published_at": { "type": "string" } + } + }, + "news_reader_result": { + "type": "object", + "required": ["ok", "mode", "articles", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "mode": { "$ref": "#/$defs/news_reader_mode" }, + "articles": { + "type": "array", + "items": { "$ref": "#/$defs/news_reader_article" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } } } } diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift index 489d77a..040be5a 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift @@ -14,6 +14,7 @@ public enum DerrickDockerRuntimeIdentity: Sendable { /// Name prefixes for current and unlabeled leftover containers. public static let namePrefixes = [ "derrick-web-crawler", + "derrick-news-reader", "derrick-guest-runtime", "derrick-swift-runtime", "derrick-file-extractor", diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift index cd1299f..666ff6b 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerProductImageDigests.generated.swift @@ -2,5 +2,5 @@ import Foundation /// Generated by scripts/record-docker-image-digests.sh — do not edit. public enum DockerProductImageDigests: Sendable { - public static let worker = DockerImageDigest(rawValue: "sha256:d686d16d5fb8fe0a54a1eb9bb9c8a27763b5d80145ad330c5205536d73afdda6") + public static let worker = DockerImageDigest(rawValue: "sha256:686c54118b75056b7ca9af17697516fc242e65b7a8da0e43bb6c0a35f31e8052") } diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift index 7a8f1f3..9a5c9d5 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift @@ -8,6 +8,14 @@ public enum DockerWorkerRuntime: Sendable { public static let crawlerBinary = "/usr/local/bin/derrick-web-crawler" public static let extractorBinary = "/usr/local/bin/derrick-file-extractor" + public static let newsReaderBinary = "/usr/local/bin/derrick-news-reader" + /// Binaries that must exist in the unified worker image. + public static let requiredBinaries: [String] = [ + crawlerBinary, + extractorBinary, + newsReaderBinary, + ] + public static let guestBinaryPath = "/tmp/guest" public static let guestSourcePath = "/tmp/plugin.go" public static let goBinaryPath = "/usr/local/go/bin/go" @@ -19,4 +27,8 @@ public enum DockerWorkerRuntime: Sendable { public static let guestReadBinaryShell = "cat /tmp/guest" public static let pinnedDigest = DockerProductImageDigests.worker + + /// OCI label written by `docker/worker/Dockerfile`; used to detect stale local images. + public static let binariesLabelKey = "derrick.worker.binaries" + public static let binariesLabelValue = "crawler,extractor,news-reader" } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift index 4a812e6..2058607 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryRuntimeTypes.swift @@ -26,4 +26,32 @@ public struct PluginFactoryRuntime: Sendable, Equatable { let language = PluginGuestLanguage(rawValue: languageRaw) ?? .go return PluginFactoryRuntime(language: language, entrypoint: entrypoint) } + + /// Package-relative guest source path used when hashing and verifying releases. + public static func guestSourcePackagePath( + runtimeJSON: String, + manifestJSON: String, + defaultPath: String = "app.derrick/plugin.go" + ) -> String { + if let runtime = decode(from: runtimeJSON) { + return normalizePackageRelativePath(runtime.entrypoint) + } + if let data = manifestJSON.data(using: .utf8), + let manifest = try? AgentPluginManifest.decode(data), + let entrypoint = manifest.derrick?.entrypoint { + return normalizePackageRelativePath(entrypoint) + } + return defaultPath + } + + private static func normalizePackageRelativePath(_ entrypoint: String) -> String { + var path = entrypoint.trimmingCharacters(in: .whitespacesAndNewlines) + if path.hasPrefix("./") { + path = String(path.dropFirst(2)) + } + while path.hasPrefix("/") { + path = String(path.dropFirst()) + } + return path + } } diff --git a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift index 2b37ff7..fdecbf5 100644 --- a/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift +++ b/packages/Structure/Sources/Plugin/Factory/PluginFactoryTypes.swift @@ -289,7 +289,6 @@ public struct PluginFactoryBuilderResponse: Codable, Sendable, Hashable { case pluginID = "plugin_id" case version, description case guestSource = "go_source" - case legacyPythonSource = "python_source" case legacySwiftSource = "swift_source" case testInputJSON = "test_input_json" case skillFiles = "skill_files" @@ -304,7 +303,6 @@ public struct PluginFactoryBuilderResponse: Codable, Sendable, Hashable { version = try container.decode(String.self, forKey: .version) description = try container.decode(String.self, forKey: .description) guestSource = try container.decodeIfPresent(String.self, forKey: .guestSource) - ?? container.decodeIfPresent(String.self, forKey: .legacyPythonSource) ?? container.decode(String.self, forKey: .legacySwiftSource) testInputJSON = try container.decode(String.self, forKey: .testInputJSON) skillFiles = try container.decodeIfPresent([PluginFactorySkillFile].self, forKey: .skillFiles) ?? [] @@ -533,10 +531,14 @@ public struct PluginFactoryRelease: Sendable, Hashable { } public func packageFiles() -> [String: Data] { + let guestPath = PluginFactoryRuntime.guestSourcePackagePath( + runtimeJSON: runtimeJSON, + manifestJSON: manifestJSON + ) var files: [String: Data] = [ "plugin.json": Data(manifestJSON.utf8), "app.derrick/runtime.json": Data(runtimeJSON.utf8), - "app.derrick/plugin.go": Data(guestSource.utf8), + guestPath: Data(guestSource.utf8), "app.derrick/plugin": compiledArtifact, ] for (path, body) in skillFiles { diff --git a/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift b/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift index bcfe23e..9b95c90 100644 --- a/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift +++ b/packages/Structure/Sources/Plugin/Manifest/DerrickRuntime.swift @@ -1,6 +1,6 @@ import Foundation -/// Derrick runtime metadata (`app.derrick/runtime.json`) for a standalone Python entrypoint. +/// Derrick runtime metadata (`app.derrick/runtime.json`) for a standalone Go entrypoint. public struct DerrickRuntime: Codable, Sendable, Hashable { public var entrypoint: String public var dependencies: [String: String] @@ -66,7 +66,7 @@ public struct DerrickRuntime: Codable, Sendable, Hashable { if trimmed.hasPrefix("./") { return try PluginPath.validateRuntimeEntrypoint(trimmed) } - guard trimmed.hasSuffix(".py"), + guard trimmed.hasSuffix(".go"), !trimmed.contains("/"), !trimmed.contains("\\") else { throw PluginManifestError.invalidEntrypoint(raw) diff --git a/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift b/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift index 8a08c89..d575c57 100644 --- a/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift +++ b/packages/Structure/Sources/Plugin/Manifest/PluginManifestError.swift @@ -47,7 +47,7 @@ public enum PluginManifestError: Error, Equatable, LocalizedError { case .invalidFieldType(let f): return "plugin.json field has the wrong type: \(f)" case .invalidEntrypoint(let p): - return "Entrypoint must be a plugin-relative .py path: \(p)" + return "Entrypoint must be a plugin-relative .go path: \(p)" case .pathNotRelative(let p): return "Path must be plugin-relative and start with ./: \(p)" case .pathEscapesRoot(let p): diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index f79aa32..b399687 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -1205,6 +1205,7 @@ import Testing #expect(DerrickDockerRuntimeIdentity.createLabelArguments == ["--label", "app.derrick=runtime"]) #expect(DerrickDockerRuntimeIdentity.namePrefixes == [ "derrick-web-crawler", + "derrick-news-reader", "derrick-guest-runtime", "derrick-swift-runtime", "derrick-file-extractor", @@ -1214,10 +1215,10 @@ import Testing #expect(!DerrickDockerRuntimeIdentity.isAllowedPsFilter("name=nginx")) #expect( DerrickDockerRuntimeIdentity.createHasRuntimeLabel( - ["create"] + DerrickDockerRuntimeIdentity.createLabelArguments + ["python:3.14.7"] + ["create"] + DerrickDockerRuntimeIdentity.createLabelArguments + [DockerWorkerRuntime.image] ) ) - #expect(!DerrickDockerRuntimeIdentity.createHasRuntimeLabel(["create", "--name", "x", "python:3.14.7"])) + #expect(!DerrickDockerRuntimeIdentity.createHasRuntimeLabel(["create", "--name", "x", DockerWorkerRuntime.image])) } @Test func webCrawlerProductImageBuildUsesPackagesContext() { @@ -1476,17 +1477,18 @@ import Testing #expect(!PluginFactoryValidationExpectations.isSendOnlyConnector(manifestJSON: sendAndReceive)) } - @Test func connectorFactoryFailureReturnsToVendorStep() { - #expect(PluginFactoryCreateInput.failureStep(forStage: "docs") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "factory") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "review") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "description") == .vendor) - #expect(PluginFactoryCreateInput.failureStep(forStage: "type") == .type) - #expect(PluginFactoryCreateInput.failureStep(forStage: "name") == .name) - #expect(PluginFactoryCreateInput.failureStep(forStage: "auth") == .auth) - #expect(PluginFactoryCreateInput.failureStep(forStage: "discover") == .auth) + @Test func pluginStudioFailureMapsToSkillFirstSteps() { + #expect(PluginFactoryCreateInput.failureStep(forStage: "docs") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "factory") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "review") == .build) + #expect(PluginFactoryCreateInput.failureStep(forStage: "description") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "type") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "name") == .skill) + #expect(PluginFactoryCreateInput.failureStep(forStage: "auth") == .credentials) + #expect(PluginFactoryCreateInput.failureStep(forStage: "discover") == .credentials) #expect(PluginFactoryCreateInput.failureStep(forStage: "paywall") == .news) #expect(PluginFactoryCreateInput.failureStep(forStage: "news") == .news) + #expect(PluginFactoryCreateInput.failureStep(forStage: "goal") == .goal) } @Test func connectorWizardOffersFullSyncOnly() { diff --git a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift index 56235d1..1af3324 100644 --- a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift +++ b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift @@ -19,179 +19,50 @@ import Structure url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")! ) == nil ) - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://feeds.bbci.co.uk/news/rss.xml")! - ) == nil - ) } - @Test func knownPaywallHostIsDetected() { - #expect(NewsPaywall.hostLooksPaywalled(URL(string: "https://www.nytimes.com/2024/01/01/world.html")!)) - #expect(!NewsPaywall.hostLooksPaywalled(URL(string: "https://feeds.bbci.co.uk/news/rss.xml")!)) - } + @Test func inferNewsModeMapsSummaryCrawlAndRSS() { + let summary = PluginSkillDraft(goal: "Give me a summary of tech news") + #expect(PluginSkillDraftPlanner.inferNewsMode(from: summary) == .summary) - @Test func htmlPaywallIsRejected() { - let html = "Subscribe to continue reading this article" - let reason = NewsPaywall.rejectionReason( - url: URL(string: "https://www.nytimes.com/story")!, - status: 200, - contentType: "text/html", - body: Data(html.utf8) - ) - #expect(reason != nil) - #expect(reason?.localizedCaseInsensitiveContains("paywall") == true) - } + let crawl = PluginSkillDraft(goal: "Crawl the BBC homepage for headlines") + #expect(PluginSkillDraftPlanner.inferNewsMode(from: crawl) == .list) - @Test func rssFromPaywalledPublisherIsAllowed() { - let rss = """ - Feed - """ - let reason = NewsPaywall.rejectionReason( - url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")!, - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - ) - #expect(reason == nil) + let rss = PluginSkillDraft(goal: "Fetch tech headlines from Google News RSS") + #expect(PluginSkillDraftPlanner.inferNewsMode(from: rss) == .rss) } - @Test func parserReadsRssItemsWithLinks() { - let rss = """ - - - Hello worldhttps://example.com/helloBody - - """ - let entries = NewsFeedParser.parse( - data: Data(rss.utf8), - sourceLabel: "Example", - fallbackPageURL: URL(string: "https://example.com/feed")! + @Test func legacySummariesModeDecodesToSummary() throws { + struct Wrapper: Decodable { let mode: NewsReaderMode } + let wrapper = try JSONDecoder.service.decode( + Wrapper.self, + from: Data(#"{"mode":"summaries"}"#.utf8) ) - #expect(entries.count == 1) - #expect(entries[0].title == "Hello world") - #expect(entries[0].sourceURL == "https://example.com/hello") + #expect(wrapper.mode == .summary) } - @Test func refreshFailsPaywalledURL() async throws { - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "text/html", - body: Data("This article is for subscribers only".utf8) - )) - let spec = NewsReaderSpec( - name: "Test", - topics: [], - sources: [NewsSource(label: "NYT", url: "https://www.nytimes.com/story")] - ) - do { - _ = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - Issue.record("expected paywall failure") - } catch let error as NewsReaderError { - guard case .paywalled = error else { - Issue.record("expected paywalled, got \(error)") - return - } + @Test func presetSourcesUsePublicFeedsNotPaywalledArticlePages() { + for preset in NewsPresetSource.allCases { + let url = URL(string: preset.source.url)! + #expect( + NewsPaywall.preflightRejection(url: url) == nil, + "Expected \(preset.source.label) feed to pass paywall preflight" + ) } } - @Test func refreshReturnsLinkedItemsCapped() async throws { - let rss = """ - - Onehttps://example.com/1 - Twohttps://example.com/2 - Threehttps://example.com/3 - - """ - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - )) - let spec = NewsReaderSpec( - name: "Cap", - topics: [], - sources: [NewsSource(label: "Ex", url: "https://example.com/rss.xml")], - maxCount: 2 - ) - let items = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - #expect(items.count == 2) - #expect(items.allSatisfy { !$0.sourceURL.isEmpty }) - } - - @Test func googleNewsHomepageMapsToRSS() { - let mapped = NewsSourceURL.canonicalFetchURL(URL(string: "https://news.google.com/")!) - #expect(mapped.host == "news.google.com") - #expect(mapped.path == "/rss") - let already = NewsSourceURL.canonicalFetchURL( - URL(string: "https://news.google.com/rss?hl=en-US")! - ) - #expect(already.path.contains("rss")) - } - - @Test func googleNewsTopicPageMapsToSectionRSSWhenHintMentionsTech() { - let topic = URL( - string: "https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB" - )! - let mapped = NewsSourceURL.canonicalFetchURL( - topic, - contextHint: "today's technology headlines" - ) - #expect(mapped.path == "/rss/headlines/section/topic/TECHNOLOGY") - } - - @Test func googleNewsTopicPageFallsBackToGeneralRSSWithoutHint() { - let topic = URL( - string: "https://news.google.com/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB" - )! - let mapped = NewsSourceURL.canonicalFetchURL(topic) - #expect(mapped.path == "/rss") - } - - @Test func refreshParsesGoogleNewsHomepageViaRSS() async throws { - let rss = """ - - World headlinehttps://news.google.com/articles/abcSummary bit - - """ - let client = StubNewsClient(response: NewsHTTPResponse( - status: 200, - contentType: "application/rss+xml", - body: Data(rss.utf8) - )) - let spec = NewsReaderSpec( - name: "Google", - topics: [], - sources: [NewsSource(label: "Google News", url: "https://news.google.com")], - mode: .summaries, - maxCount: 10 + @Test func workerRequestEncodesModeAndSources() throws { + let request = NewsReaderWorkerRequest( + mode: .rss, + sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/rss.xml")], + topics: ["Tech"], + maxCount: 10, + contextHint: "tech-news" ) - let items = try await NewsReaderRefresh.validateAndFetch(spec: spec, client: client) - #expect(items.count == 1) - #expect(items[0].title == "World headline") - #expect(items[0].sourceURL.contains("news.google.com")) - #expect(NewsReaderRefresh.digest(from: items).contains("World headline")) - } - - @Test func liveGoogleNewsRSSHasLinkedSummaries() async throws { - let spec = NewsReaderSpec( - name: "Google live", - topics: [], - sources: [NewsPresetSource.googleNews.source], - mode: .summaries, - maxCount: 8 - ) - let items = try await NewsReaderRefresh.validateAndFetch( - spec: spec, - client: URLSessionNewsHTTPClient() - ) - #expect(!items.isEmpty) - #expect(items.allSatisfy { !$0.sourceURL.isEmpty && !$0.title.isEmpty }) - #expect(!NewsReaderRefresh.digest(from: items).isEmpty) - } - - private struct StubNewsClient: NewsHTTPClient { - let response: NewsHTTPResponse - func get(url: URL) async throws -> NewsHTTPResponse { response } + let json = try request.encodedJSON() + let decoded = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: json) + #expect(decoded.mode == .rss) + #expect(decoded.sources.count == 1) + #expect(decoded.topics == ["Tech"]) } } diff --git a/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift new file mode 100644 index 0000000..561a0a5 --- /dev/null +++ b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift @@ -0,0 +1,118 @@ +import Foundation +import Testing +@testable import Structure + +@Suite struct PluginSkillDraftTests { + @Test func infersSummariesModeFromSummaryGoal() { + var draft = PluginSkillDraft(goal: "Give me a summary of tech headlines") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(PluginSkillDraftPlanner.inferNewsMode(from: draft) == .summary) + } + + @Test func infersNewsFromGoal() { + var draft = PluginSkillDraft(goal: "Fetch tech news headlines daily") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .newsDigest) + #expect(!draft.newsTopics.isEmpty) + #expect(!draft.newsSourceURLs.isEmpty) + } + + @Test func infersSlackConnectorFromGoal() { + var draft = PluginSkillDraft(goal: "Send messages in Slack from Messaging") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .messagingConnector) + #expect(draft.inferredConnectorVendor == .slack) + #expect(draft.examples.count >= 1) + } + + @Test func infersCustomCapabilityFromGenericGoal() { + var draft = PluginSkillDraft(goal: "Summarize my clipboard when I ask") + PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) + #expect(draft.plannedKind == .customCapability) + } + + @Test func skillMarkdownIncludesPurposeAndExamples() { + var draft = PluginSkillDraft( + goal: "Do something", + purpose: "Help with tasks", + triggers: [.chat], + examples: [ + PluginSkillDraft.Example(userSays: "run it", pluginDoes: "returns a result"), + ], + pluginName: "my-plugin" + ) + let markdown = draft.skillMarkdown() + #expect(markdown.contains("my-plugin")) + #expect(markdown.contains("Help with tasks")) + #expect(markdown.contains("run it")) + } + + @Test func makeFromSkillDraftBuildsConnectorInput() throws { + var draft = PluginSkillDraft( + goal: "Slack connector", + purpose: "Messaging", + examples: [ + PluginSkillDraft.Example(userSays: "hi", pluginDoes: "send"), + ], + pluginName: "slack-connector-1" + ) + let input = try PluginFactoryCreateInput.makeFromSkillDraft(draft) + #expect(input.pluginType == .connector) + #expect(input.vendor == .slack) + #expect(input.pluginID == "slack-connector-1") + #expect(input.skillMarkdown != nil) + } + + @Test func makeFromSkillDraftBuildsCustomInput() throws { + var draft = PluginSkillDraft( + goal: "Summarize text", + purpose: "Summarize", + examples: [ + PluginSkillDraft.Example(userSays: "summarize", pluginDoes: "returns summary"), + ], + pluginName: "summarizer" + ) + let input = try PluginFactoryCreateInput.makeFromSkillDraft(draft) + #expect(input.pluginType == .custom) + #expect(input.vendor == nil) + #expect(input.pluginID == "summarizer") + } + + @Test func newsDigestOnlyAllowsChatAndScheduleTriggers() { + let allowed = PluginSkillDraftPlanner.availableTriggers(for: .newsDigest) + #expect(allowed == Set([.chat, .schedule])) + var draft = PluginSkillDraft( + goal: "tech news", + triggers: [.chat, .schedule, .mention, .messaging], + pluginName: "tech-news" + ) + PluginSkillDraftPlanner.sanitizeTriggers(in: &draft) + #expect(draft.triggers == Set([.chat, .schedule])) + } + + @Test func messagingConnectorDisallowsScheduleTrigger() { + let allowed = PluginSkillDraftPlanner.availableTriggers(for: .messagingConnector) + #expect(allowed == Set([.chat, .messaging, .mention])) + var draft = PluginSkillDraft( + goal: "Slack connector", + triggers: [.schedule, .chat, .messaging], + pluginName: "slack-1" + ) + PluginSkillDraftPlanner.sanitizeTriggers(in: &draft) + #expect(draft.triggers == Set([.chat, .messaging])) + } + + @Test func newsDraftRejectsFactoryInput() { + let draft = PluginSkillDraft( + goal: "news digest", + purpose: "News", + examples: [PluginSkillDraft.Example(userSays: "news", pluginDoes: "fetch")], + pluginName: "news-list", + newsTopics: ["Tech"], + newsSourceURLs: ["https://news.google.com/rss"] + ) + #expect(throws: PluginSkillDraftError.newsUsesReaderPath) { + try PluginFactoryCreateInput.makeFromSkillDraft(draft) + } + } +} diff --git a/scripts/reset-local-state.sh b/scripts/reset-local-state.sh new file mode 100755 index 0000000..fb845c6 --- /dev/null +++ b/scripts/reset-local-state.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +APP_GROUP="$HOME/Library/Group Containers/VUSK4B2YKQ.derrick.shared" + +echo "==> Quit Derrick before resetting local state." +echo "==> This removes all local SQLite data (chats, plugins, messaging, credentials in DB)." +echo "==> Keychain plugin secrets are not removed." + +removed_dbs=0 +if [[ -d "$APP_GROUP" ]]; then + while IFS= read -r db; do + rm -f "$db" "${db}-wal" "${db}-shm" + echo "removed $(basename "$db") at ${db%/*}" + removed_dbs=$((removed_dbs + 1)) + done < <(find "$APP_GROUP" -name 'derrick.sqlite3' 2>/dev/null) +fi + +if [[ "$removed_dbs" -eq 0 ]]; then + echo "no derrick.sqlite3 files found under $APP_GROUP" +fi + +if command -v docker >/dev/null 2>&1; then + echo "==> Removing Derrick runtime containers" + for filter in 'name=derrick-guest-runtime' 'name=derrick-swift-runtime' 'label=app.derrick=runtime'; do + ids="$(docker ps -aq --filter "$filter" || true)" + if [[ -n "$ids" ]]; then + docker rm -f $ids >/dev/null 2>&1 || true + fi + done + + echo "==> Removing obsolete guest images (Python / legacy guest-runtime)" + for image in \ + 'derrick-guest-runtime:python-v1' \ + 'python:3.14.7'; do + if docker image inspect "$image" >/dev/null 2>&1; then + docker rmi -f "$image" >/dev/null + echo "removed image $image" + fi + done +else + echo "docker not available — skipped container/image cleanup" +fi + +echo +echo "Done. Reopen Derrick from $ROOT (go-workers) to recreate an empty database." +echo "Policy rules seed automatically on first UI launch." diff --git a/ui/JobKeepAlive/DaemonModuleBootstrap.swift b/ui/JobKeepAlive/DaemonModuleBootstrap.swift index 638ec76..df0c8d8 100644 --- a/ui/JobKeepAlive/DaemonModuleBootstrap.swift +++ b/ui/JobKeepAlive/DaemonModuleBootstrap.swift @@ -37,6 +37,9 @@ enum DaemonModuleBootstrap { let tools = try await MCPServiceToolHost.shared.searchTools(query: query, principal: principal) return MCPToolSearchResultDTO(ok: true, tools: tools, message: "ok") } + InProcessServiceBridges.runNewsReader = { requestJSON in + try await MCPServiceToolHost.shared.runNewsReader(requestJSON: requestJSON) + } InProcessServiceBridges.jobLocalProxy = JobServiceExportedObject() InProcessServiceBridges.jobNetworkPreflight = { toolName, argumentsJSON, jobID in let repo = try await JobServiceStore.shared.sharedRepository() @@ -99,7 +102,7 @@ enum DaemonModuleBootstrap { } /// Remove leftover runtime containers before the job scheduler starts. - /// Guest image pull stays in the background so a cold Python pull does not block jobs. + /// Guest image pull stays in the background so a cold worker image pull does not block jobs. private static func sweepEmbeddedDockerLeftovers() async { guard DerrickProcessRole.isDaemon else { return } do { diff --git a/ui/JobKeepAlive/DaemonUnifiedXPC.swift b/ui/JobKeepAlive/DaemonUnifiedXPC.swift index 4806931..1bd52e6 100644 --- a/ui/JobKeepAlive/DaemonUnifiedXPC.swift +++ b/ui/JobKeepAlive/DaemonUnifiedXPC.swift @@ -234,6 +234,10 @@ final class DaemonUnifiedExportedObject: NSObject, DerrickDaemonServiceXPC, @unc mcp.searchTools(requestJSON: requestJSON, withReply: reply) } + func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { + mcp.runNewsReader(requestJSON: requestJSON, withReply: reply) + } + // MARK: - Workflow runtime func startWorkflow(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { diff --git a/ui/MCPService/MCPServiceExportedObject.swift b/ui/MCPService/MCPServiceExportedObject.swift index e1a90b7..d7dc754 100644 --- a/ui/MCPService/MCPServiceExportedObject.swift +++ b/ui/MCPService/MCPServiceExportedObject.swift @@ -177,4 +177,21 @@ final class MCPServiceExportedObject: NSObject, MCPServiceXPC { } } } + + func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { + let data = requestJSON as Data + Task { + do { + let result = try await MCPServiceToolHost.shared.runNewsReader(requestJSON: data) + reply((try MCPServiceXPCCodec.encodeNewsReaderRunResult(result)) as NSData) + } catch { + fputs("[MCPService] runNewsReader failed: \(error.localizedDescription)\n", stderr) + let result = NewsReaderRunResult(ok: false, message: error.localizedDescription) + reply( + (try? MCPServiceXPCCodec.encodeNewsReaderRunResult(result)) as NSData? + ?? Data("{}".utf8) as NSData + ) + } + } + } } diff --git a/ui/MCPService/MCPServiceToolHost.swift b/ui/MCPService/MCPServiceToolHost.swift index d3aa0af..7a7e533 100644 --- a/ui/MCPService/MCPServiceToolHost.swift +++ b/ui/MCPService/MCPServiceToolHost.swift @@ -195,6 +195,36 @@ actor MCPServiceToolHost { return made } + func runNewsReader(requestJSON: Data, timeoutSeconds: Int = 180) async throws -> NewsReaderRunResult { + _ = try await ensureReady() + let executor = NewsReaderDockerExecutor( + executor: MCPServiceDockerHelperRunner.shared.makeStdinCLIExecutor() + ) + let result = try await executor.run(input: requestJSON, timeoutSeconds: timeoutSeconds) + let stderrText = String(decoding: result.stderr, as: UTF8.self) + if result.exitCode != 0 { + let detail = newsReaderFailureDetail(exitCode: result.exitCode, stderr: stderrText) + await MCPServiceStore.shared.log( + level: .error, + message: "news reader failed: \(detail)", + code: "news_reader_failed" + ) + return NewsReaderRunResult( + ok: false, + stdout: result.stdout, + stderr: result.stderr, + message: detail + ) + } + guard !result.stdout.isEmpty else { + return NewsReaderRunResult( + ok: false, + message: "News reader returned no output." + ) + } + return NewsReaderRunResult(ok: true, stdout: result.stdout, stderr: result.stderr) + } + func searchTools(query: String, principal: ServicePrincipal) async throws -> [MCPToolDescriptorDTO] { let client = try await ensureReady().client await MCPServiceStore.shared.log( @@ -358,3 +388,15 @@ private func pluginFactoryFailureDetail(for error: Error) -> String { } return error.localizedDescription } + +private func newsReaderFailureDetail(exitCode: Int32, stderr: String) -> String { + if exitCode == 126 { + return """ + The worker image on this Mac is missing the news reader binary. Quit Derrick completely and reopen it so the worker image can rebuild, then try again. + """ + } + if !stderr.isEmpty { + return stderr + } + return "exit \(exitCode)" +} diff --git a/ui/SharedAgentRuntime/Services/MCPServiceClient.swift b/ui/SharedAgentRuntime/Services/MCPServiceClient.swift index 048f7ce..4f4dbd1 100644 --- a/ui/SharedAgentRuntime/Services/MCPServiceClient.swift +++ b/ui/SharedAgentRuntime/Services/MCPServiceClient.swift @@ -176,6 +176,25 @@ public final class MCPServiceClient: @unchecked Sendable { } } + public func runNewsReader(requestJSON: Data, timeoutSeconds: Int = 180) async throws -> NewsReaderRunResult { + if DerrickProcessRole.isDaemon, let run = InProcessServiceBridges.runNewsReader { + return try await run(requestJSON) + } + nonisolated(unsafe) let proxy = try remoteProxy() + let payload = requestJSON as NSData + return try await invoke(timeout: MCPToolCallTimeouts.newsReaderNanoseconds) { + try await withCheckedThrowingContinuation { cont in + proxy.runNewsReader(requestJSON: payload) { data in + do { + cont.resume(returning: try MCPServiceXPCCodec.decodeNewsReaderRunResult(data as Data)) + } catch { + cont.resume(throwing: error) + } + } + } + } + } + public func searchTools(principal: ServicePrincipal, query: String = "") async throws -> MCPToolSearchResultDTO { if DerrickProcessRole.isDaemon, let search = InProcessServiceBridges.mcpSearchTools { return try await search(principal, query) diff --git a/ui/ui/News/MCPServiceNewsWorker.swift b/ui/ui/News/MCPServiceNewsWorker.swift new file mode 100644 index 0000000..54c8277 --- /dev/null +++ b/ui/ui/News/MCPServiceNewsWorker.swift @@ -0,0 +1,23 @@ +import Foundation +import Structure + +/// Runs the Docker news reader in MCPService where the egress proxy can bind. +struct MCPServiceNewsWorker: NewsWorkerRunning { + func run(requestJSON: Data) async throws -> Data { + _ = try await MCPServiceClient.shared.ensureUpAndHealth(retries: 2) + let result = try await MCPServiceClient.shared.runNewsReader(requestJSON: requestJSON) + guard result.ok else { + let detail = result.message.nilIfEmpty + ?? String(decoding: result.stderr, as: UTF8.self).nilIfEmpty + ?? "News reader failed." + throw NewsReaderError.workerUnavailable(detail) + } + return result.stdout + } +} + +private extension String { + var nilIfEmpty: String? { + isEmpty ? nil : self + } +} diff --git a/ui/ui/News/NewsReaderStore.swift b/ui/ui/News/NewsReaderStore.swift index e5ea5dd..dde776d 100644 --- a/ui/ui/News/NewsReaderStore.swift +++ b/ui/ui/News/NewsReaderStore.swift @@ -14,18 +14,30 @@ final class NewsReaderStore: ObservableObject { @Published private(set) var lastError: String? private var repository: DBRepository? - private let client: any NewsHTTPClient + private let worker: any NewsWorkerRunning + private var summarizer: NewsReaderSummarizer? - init(client: any NewsHTTPClient = URLSessionNewsHTTPClient()) { - self.client = client + init( + worker: any NewsWorkerRunning = MCPServiceNewsWorker(), + summarizer: NewsReaderSummarizer? = nil + ) { + self.worker = worker + self.summarizer = summarizer + } + + func attachSummarizer(_ settings: LLMModelSettings) { + self.summarizer = NewsReaderSummarizer(settings: settings) } var selectedReader: NewsReaderSpec? { readers.first { $0.id == selectedReaderID } } - func configure(repository: DBRepository) async { + func configure(repository: DBRepository, summarizerSettings: LLMModelSettings? = nil) async { self.repository = repository + if let summarizerSettings { + attachSummarizer(summarizerSettings) + } await reload() } @@ -62,14 +74,19 @@ final class NewsReaderStore: ObservableObject { } var next = spec next.updatedAt = .now - let items = try await NewsReaderRefresh.validateAndFetch(spec: next, client: client) + let fetched = try await NewsReaderRefresh.validateAndFetch( + spec: next, + worker: worker, + summarizer: summarizer + ) next.lastFetchedAt = .now next.lastError = nil + next.summaryText = fetched.summaryText try await repository.upsertNewsReader(next) - try await repository.replaceNewsItems(readerID: next.id, items: items) - await reload() + try await repository.replaceNewsItems(readerID: next.id, items: fetched.items) selectedReaderID = next.id - self.items = items + self.items = fetched.items + await reload() return next } @@ -78,13 +95,18 @@ final class NewsReaderStore: ObservableObject { isRefreshing = true defer { isRefreshing = false } do { - let fetched = try await NewsReaderRefresh.validateAndFetch(spec: reader, client: client) + let fetched = try await NewsReaderRefresh.validateAndFetch( + spec: reader, + worker: worker, + summarizer: summarizer + ) reader.lastFetchedAt = .now reader.lastError = nil reader.updatedAt = .now + reader.summaryText = fetched.summaryText try await repository.upsertNewsReader(reader) - try await repository.replaceNewsItems(readerID: reader.id, items: fetched) - items = fetched + try await repository.replaceNewsItems(readerID: reader.id, items: fetched.items) + items = fetched.items lastError = nil await reload() } catch { diff --git a/ui/ui/News/NewsReaderSummarizer.swift b/ui/ui/News/NewsReaderSummarizer.swift new file mode 100644 index 0000000..74b441c --- /dev/null +++ b/ui/ui/News/NewsReaderSummarizer.swift @@ -0,0 +1,65 @@ +import Foundation +import LLMAgentClient +import Structure + +struct NewsReaderSummarizer: NewsSummaryGenerating { + let settings: LLMModelSettings + + func summarize(listName: String, topics: [String], articles: [NewsItem]) async throws -> String { + let model = await MainActor.run { settings.summarizerModel } + guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) else { + throw NewsReaderError.summarizerUnavailable + } + + let prompt = Self.prompt(listName: listName, topics: topics, articles: articles) + let request = AgentRequest.prompt( + prompt, + system: """ + You summarize news for the user. Write clear prose in plain English. + Include markdown links to the original articles using the URLs provided. + Do not invent stories or URLs. + """, + temperature: 0.2 + ) + let text: String + switch model { + case .gemini(let geminiModel): + let client = GeminiAgentClient(provider: GeminiProvider(apiKey: apiKey)) + text = try await collect(client.stream(request, model: geminiModel)) + case .openai(let openAIModel): + let client = OpenAIAgentClient(provider: OpenAIProvider(apiKey: apiKey)) + text = try await collect(client.stream(request, model: openAIModel)) + } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { + throw NewsReaderError.fetchFailed(url: listName, detail: "Summarizer returned an empty response.") + } + return trimmed + } + + private func collect(_ stream: AsyncThrowingStream) async throws -> String { + let (text, usage) = try await collectAgentStream(stream) + if let usage { + _ = await UsageLimitsService.shared.recordAPIUsage(usage) + } + return text + } + + private static func prompt(listName: String, topics: [String], articles: [NewsItem]) -> String { + var lines = [ + "Summarize these articles for the list \"\(listName)\".", + ] + if !topics.isEmpty { + lines.append("Topics: \(topics.joined(separator: ", "))") + } + lines.append("Articles:") + for article in articles.prefix(12) { + var entry = "- \(article.title) (\(article.sourceURL))" + if let detail = article.summary, !detail.isEmpty { + entry += "\n \(detail)" + } + lines.append(entry) + } + return lines.joined(separator: "\n") + } +} diff --git a/ui/ui/News/NewsWorkspaceView.swift b/ui/ui/News/NewsWorkspaceView.swift index ff8a4ab..43a54e1 100644 --- a/ui/ui/News/NewsWorkspaceView.swift +++ b/ui/ui/News/NewsWorkspaceView.swift @@ -56,7 +56,7 @@ struct NewsWorkspaceView: View { .foregroundStyle(.secondary) Text("No news lists yet") .font(.title3.weight(.semibold)) - Text("Create a News reader from Plugins. You can pick several topics and several sources.") + Text("Create a news list from Plugins. Derrick fetches articles in Docker, then shows them here.") .font(.subheadline) .foregroundStyle(.secondary) .multilineTextAlignment(.center) @@ -77,11 +77,14 @@ struct NewsWorkspaceView: View { .frame(maxWidth: .infinity, alignment: .leading) .background(Color.red.opacity(0.08), in: RoundedRectangle(cornerRadius: 10)) } - if store.selectedReader?.mode == .summaries, !store.items.isEmpty { + + if store.selectedReader?.mode == .summary, + let summary = store.selectedReader?.summaryText, + !summary.isEmpty { VStack(alignment: .leading, spacing: 8) { Text("Summary") .font(.subheadline.weight(.semibold)) - Text(NewsReaderRefresh.digest(from: store.items)) + Text(LocalizedStringKey(summary)) .font(.body) .textSelection(.enabled) } @@ -89,31 +92,34 @@ struct NewsWorkspaceView: View { .frame(maxWidth: .infinity, alignment: .leading) .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) } - ForEach(store.items) { item in - VStack(alignment: .leading, spacing: 6) { - Link(destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) { - Text(item.title) - .font(.body.weight(.semibold)) - .multilineTextAlignment(.leading) - } - HStack(spacing: 8) { - Text(item.sourceLabel) - .font(.caption) - .foregroundStyle(.secondary) - Link("Source", destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) - .font(.caption.weight(.semibold)) - } - if let summary = item.summary, !summary.isEmpty, store.selectedReader?.mode == .list { - Text(summary) - .font(.caption) - .foregroundStyle(.secondary) - .lineLimit(4) + + if store.selectedReader?.mode != .summary { + ForEach(store.items) { item in + articleCard(item) + } + } else { + VStack(alignment: .leading, spacing: 8) { + Text("Sources") + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + ForEach(store.items) { item in + HStack(alignment: .top, spacing: 8) { + Text("•") + VStack(alignment: .leading, spacing: 2) { + Link(item.title, destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) + .font(.caption.weight(.semibold)) + Text(item.sourceLabel) + .font(.caption2) + .foregroundStyle(.secondary) + } + } } } .padding(14) .frame(maxWidth: .infinity, alignment: .leading) .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) } + if store.items.isEmpty, store.lastError == nil, store.selectedReader?.lastError == nil { Text("No articles yet. Refresh this list.") .font(.subheadline) @@ -124,6 +130,36 @@ struct NewsWorkspaceView: View { } } + @ViewBuilder + private func articleCard(_ item: NewsItem) -> some View { + VStack(alignment: .leading, spacing: 6) { + Link(destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) { + Text(item.title) + .font(.body.weight(.semibold)) + .multilineTextAlignment(.leading) + } + HStack(spacing: 8) { + Text(item.sourceLabel) + .font(.caption) + .foregroundStyle(.secondary) + if let host = URL(string: item.sourceURL)?.host { + Link(host, destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) + .font(.caption.weight(.semibold)) + .lineLimit(1) + } + } + if let summary = item.summary, !summary.isEmpty { + Text(summary) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(4) + } + } + .padding(14) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) + } + private func headerSubtitle(_ reader: NewsReaderSpec) -> String { let topics = reader.topics.isEmpty ? "All topics" : reader.topics.joined(separator: ", ") let sources = "\(reader.sources.count) source\(reader.sources.count == 1 ? "" : "s")" diff --git a/ui/ui/Plugins/PluginCreationController.swift b/ui/ui/Plugins/PluginCreationController.swift index 8888d39..fb65491 100644 --- a/ui/ui/Plugins/PluginCreationController.swift +++ b/ui/ui/Plugins/PluginCreationController.swift @@ -2,21 +2,25 @@ import Combine import DBRepository import Foundation import Structure +import SwiftUI @MainActor final class PluginCreationController: ObservableObject { enum Phase: Equatable { case intro - case chooseType - case chooseVendor - case chooseName - case chooseNews + case goal + case skill + case preview case discoveringAuth case creating case collectCredentials(pluginID: String) case failed(step: PluginFactoryCreateInput.FailureStep, message: String, technicalDetail: String? = nil) - case succeeded(pluginID: String) - case succeededNews(readerID: String) + case succeeded(pluginID: String, outcome: SuccessOutcome) + } + + enum SuccessOutcome: Equatable { + case plugin + case newsList } struct ProgressStepState: Identifiable, Equatable { @@ -35,22 +39,7 @@ final class PluginCreationController: ObservableObject { @Published private(set) var phase: Phase = .intro @Published private(set) var statusMessage = "" @Published private(set) var progressSteps: [ProgressStepState] = [] - @Published var selectedType: PluginFactoryCreateInput.PluginType = .connector - @Published var selectedVendor: PluginFactoryCreateInput.ConnectorVendor = .slack - @Published var selectedScope: PluginFactoryCreateInput.ConnectorScope = .fullSync - @Published var customVendorName = "" - @Published var connectorName = "" - @Published private(set) var defaultNameReady = false - @Published var newsName = "" - @Published var selectedNewsTopics: Set = [] - @Published var extraNewsTopics: [String] = [] - @Published var newsTopicDraft = "" - @Published var selectedNewsSources: Set = [] - @Published var extraNewsURLs: [String] = [] - @Published var newsURLDraft = "" - @Published var newsMode: NewsReaderMode = .list - @Published var newsMaxCount = 20 - @Published var newsSchedule: NewsReaderSchedule = .off + @Published var skillDraft = PluginSkillDraft() @Published private(set) var credentialFields: [PluginCredentialFieldPresentation] = [] @Published var credentialDrafts: [String: String] = [:] @@ -59,8 +48,6 @@ final class PluginCreationController: ObservableObject { private var pollAfterSeq = 0 private var pollTask: Task? private var discoverTask: Task? - private var namePrepareTask: Task? - private var generatedConnectorName = "" private var pendingAuth: ConnectorAuthDiscovery? private var creationAPIKey: String? private var creationReviewerModelJSON: String? @@ -69,27 +56,33 @@ final class PluginCreationController: ObservableObject { deinit { pollTask?.cancel() discoverTask?.cancel() - namePrepareTask?.cancel() } - var canConfirmName: Bool { - guard defaultNameReady else { return false } - let trimmed = connectorName.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return false } - return (try? PluginID.normalized(trimmed)) != nil + var canContinueFromGoal: Bool { + !skillDraft.goal.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } - var canConfirmVendor: Bool { - guard selectedVendor.isSelectableInWizard else { return false } - if selectedVendor == .custom { - return !customVendorName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + var canContinueFromSkill: Bool { + guard canConfirmPluginName else { return false } + guard !skillDraft.purpose.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return false } + guard !skillDraft.examples.isEmpty else { return false } + guard skillDraft.examples.allSatisfy({ + !$0.userSays.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + && !$0.pluginDoes.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + }) else { return false } + if skillDraft.plannedKind == .newsDigest { + return !skillDraft.newsTopics.isEmpty && !skillDraft.newsSourceURLs.isEmpty } - return true + return skillDraft.buildBlockedReason == nil } - var canConfirmNews: Bool { - let nameOK = !newsName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - return nameOK && !builtNewsSources().isEmpty + var canConfirmPluginName: Bool { + let trimmed = skillDraft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return false } + if skillDraft.plannedKind == .newsDigest { + return true + } + return (try? PluginID.normalized(trimmed)) != nil } var canSaveCredentials: Bool { @@ -104,281 +97,168 @@ final class PluginCreationController: ObservableObject { self.repository = repository } + func skillDraftBinding(_ keyPath: WritableKeyPath) -> Binding { + Binding( + get: { self.skillDraft[keyPath: keyPath] }, + set: { newValue in + var draft = self.skillDraft + draft[keyPath: keyPath] = newValue + self.skillDraft = draft + } + ) + } + func showIntro() { cancelPolling() discoverTask?.cancel() - namePrepareTask?.cancel() pendingAuth = nil phase = .intro statusMessage = "" progressSteps = [] credentialFields = [] credentialDrafts = [:] - selectedScope = .fullSync - connectorName = "" - generatedConnectorName = "" - defaultNameReady = false + skillDraft = PluginSkillDraft() } func beginCreate() { cancelPolling() - phase = .chooseType + phase = .goal statusMessage = "" progressSteps = [] } - func selectType(_ type: PluginFactoryCreateInput.PluginType) { - selectedType = type - } - - func confirmTypeSelection() { - if selectedType == .newsReader { - resetNewsDraft() - phase = .chooseNews - return - } - guard selectedType == .connector else { - phase = .failed( - step: .type, - message: "Custom plugins are not available yet." - ) - return - } - selectedVendor = .slack - refreshDefaultConnectorName() - phase = .chooseVendor - } - - func confirmVendor( - sessionID: String, - helperAPIKey: String?, - helperReviewerModelJSON: String? - ) { - selectedScope = .fullSync - creationSessionID = sessionID - creationAPIKey = helperAPIKey - creationReviewerModelJSON = helperReviewerModelJSON - defaultNameReady = false - phase = .chooseName - namePrepareTask?.cancel() - namePrepareTask = Task { @MainActor in + func continueFromGoal() { + guard canContinueFromGoal else { return } + Task { @MainActor in await PluginFactoryListStore.shared.reload() - guard !Task.isCancelled else { return } - refreshDefaultConnectorName() - defaultNameReady = true - startAuthDiscovery() + var draft = skillDraft + PluginSkillDraftPlanner.applyGoal( + draft.goal, + to: &draft, + existingPluginIDs: PluginFactoryListStore.shared.pluginIDs + ) + skillDraft = draft + phase = .skill } } - func confirmConnectorName() { - guard canConfirmName else { return } - if let pendingAuth { - presentCredentialsOrFail(auth: pendingAuth) - } else { - phase = .discoveringAuth - statusMessage = "Reading how this service authenticates…" - } + func continueToPreview() { + guard canContinueFromSkill else { return } + phase = .preview } - func goBackToTypeSelection() { - namePrepareTask?.cancel() + func goBackToGoal() { discoverTask?.cancel() pendingAuth = nil - defaultNameReady = false - phase = .chooseType + phase = .goal } - func goBackToVendor() { - namePrepareTask?.cancel() + func goBackToSkill() { discoverTask?.cancel() pendingAuth = nil - defaultNameReady = false - phase = .chooseVendor + phase = .skill } - func addNewsTopic() { - let topic = newsTopicDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !topic.isEmpty else { return } - if !extraNewsTopics.contains(where: { $0.compare(topic, options: .caseInsensitive) == .orderedSame }) { - extraNewsTopics.append(topic) + func addExample() { + mutateSkillDraft { + $0.examples.append(PluginSkillDraft.Example(userSays: "", pluginDoes: "")) } - newsTopicDraft = "" } - func removeNewsTopic(_ topic: String) { - extraNewsTopics.removeAll { $0 == topic } + func removeExample(id: String) { + mutateSkillDraft { $0.examples.removeAll { $0.id == id } } } - func addNewsURL() { - let url = newsURLDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !url.isEmpty else { return } - extraNewsURLs.append(url) - newsURLDraft = "" + func updateExample(id: String, userSays: String? = nil, pluginDoes: String? = nil) { + mutateSkillDraft { draft in + guard let index = draft.examples.firstIndex(where: { $0.id == id }) else { return } + if let userSays { draft.examples[index].userSays = userSays } + if let pluginDoes { draft.examples[index].pluginDoes = pluginDoes } + } } - func removeNewsURL(_ url: String) { - extraNewsURLs.removeAll { $0 == url } + func toggleTrigger(_ trigger: PluginSkillDraft.Trigger) { + mutateSkillDraft { draft in + guard draft.isTriggerAvailable(trigger) else { return } + if draft.triggers.contains(trigger) { + draft.triggers.remove(trigger) + } else { + draft.triggers.insert(trigger) + } + } } - func startNewsCreation() { - let spec = NewsReaderSpec( - name: newsName, - topics: builtNewsTopics(), - sources: builtNewsSources(), - mode: newsMode, - maxCount: newsMaxCount, - schedule: newsSchedule - ) - if let blocked = spec.sources.compactMap({ source -> (NewsSource, String)? in - guard let url = URL(string: source.url), - let reason = NewsPaywall.preflightRejection(url: url) else { return nil } - return (source, reason) - }).first { - phase = .failed( - step: .news, - message: NewsReaderError.paywalled(url: blocked.0.url, detail: blocked.1).errorDescription - ?? "This source is behind a paywall, which is not supported yet." - ) - return - } - phase = .creating - statusMessage = "Checking sources…" - progressSteps = [ - ProgressStepState(id: "sources", title: "Check sources for paywalls", status: .active), - ProgressStepState(id: "fetch", title: "Fetch articles with source links", status: .pending), - ] - pollTask?.cancel() - pollTask = Task { @MainActor in - do { - let saved = try await NewsReaderStore.shared.create(spec) - setProgressStep("sources", status: .completed) - setProgressStep("fetch", status: .completed) - phase = .succeededNews(readerID: saved.id) - } catch let error as NewsReaderError { - setProgressStep("sources", status: .failed) - phase = .failed( - step: .news, - message: error.localizedDescription, - technicalDetail: String(describing: error) - ) - } catch { - setProgressStep("sources", status: .failed) - phase = .failed( - step: .news, - message: error.localizedDescription - ) + func addNewsTopic() { + let topic = skillDraft.goal.trimmingCharacters(in: .whitespacesAndNewlines) + guard !topic.isEmpty else { return } + mutateSkillDraft { draft in + if !draft.newsTopics.contains(where: { $0.compare(topic, options: .caseInsensitive) == .orderedSame }) { + draft.newsTopics.append(topic) } } } - func builtNewsTopics() -> [String] { - selectedNewsTopics.map(\.displayName) + extraNewsTopics + func removeNewsTopic(_ topic: String) { + mutateSkillDraft { $0.newsTopics.removeAll { $0 == topic } } } - func builtNewsSources() -> [NewsSource] { - var sources = selectedNewsSources.map(\.source) - for raw in extraNewsURLs + [newsURLDraft] { - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { continue } - let normalized = NewsSourceURL.canonicalFetchURL( - URL(string: trimmed.contains("://") ? trimmed : "https://\(trimmed)") - ?? URL(string: "https://news.google.com/rss")! - ).absoluteString - sources.append(NewsSource(label: hostLabel(normalized), url: normalized)) + func addNewsTopicFromPreset(_ topic: String) { + let trimmed = topic.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + mutateSkillDraft { draft in + if !draft.newsTopics.contains(trimmed) { + draft.newsTopics.append(trimmed) + } } - var seen = Set() - return sources.filter { seen.insert($0.url).inserted } } - private func resetNewsDraft() { - newsName = "" - selectedNewsTopics = [] - extraNewsTopics = [] - newsTopicDraft = "" - selectedNewsSources = [] - extraNewsURLs = [] - newsURLDraft = "" - newsMode = .list - newsMaxCount = 20 - newsSchedule = .off + func addNewsURL(_ raw: String) { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + let normalized = NewsSourceURL.canonicalFetchURL( + URL(string: trimmed.contains("://") ? trimmed : "https://\(trimmed)") + ?? URL(string: "https://news.google.com/rss")! + ).absoluteString + mutateSkillDraft { draft in + if !draft.newsSourceURLs.contains(normalized) { + draft.newsSourceURLs.append(normalized) + } + } + } + + func removeNewsURL(_ url: String) { + mutateSkillDraft { $0.newsSourceURLs.removeAll { $0 == url } } } - private func hostLabel(_ urlString: String) -> String { - URL(string: urlString)?.host ?? urlString + private func mutateSkillDraft(_ transform: (inout PluginSkillDraft) -> Void) { + var draft = skillDraft + transform(&draft) + skillDraft = draft } - func startCreation( + func confirmPreview( sessionID: String, helperAPIKey: String?, helperReviewerModelJSON: String? ) { - guard selectedVendor.isSelectableInWizard else { - phase = .failed( - step: .vendor, - message: "Only Slack connectors can be created right now." - ) - return - } - guard let helperAPIKey, !helperAPIKey.isEmpty else { - phase = .failed( - step: .vendor, - message: "Add an API key in Settings before creating a plugin." - ) - return - } - guard let pluginID = normalizedConnectorName(), - let auth = pendingAuth - else { - phase = .failed( - step: .name, - message: "Name this connector and save its credentials before creating it." - ) + creationSessionID = sessionID + creationAPIKey = helperAPIKey + creationReviewerModelJSON = helperReviewerModelJSON + + if skillDraft.plannedKind == .newsDigest { + startNewsCreation() return } - guard auth.authScheme.isSupportedInWizard else { - phase = .failed( - step: .auth, - message: "OAuth connectors are not available yet. Use a bot token or API key." - ) + + if skillDraft.plannedKind == .messagingConnector { + phase = .discoveringAuth + statusMessage = "Reading how this service authenticates…" + resetProgressSteps() + startAuthDiscovery() return } - selectedScope = .fullSync - cancelPolling() - phase = .creating - statusMessage = "Starting connector creation…" - resetProgressSteps() - pollAfterSeq = 0 - - let input = PluginFactoryCreateInput.makeConnector( - vendor: selectedVendor, - pluginID: pluginID, - auth: auth, - customVendorName: selectedVendor == .custom ? customVendorName : nil, - scope: selectedScope, - userDescription: "" - ) - - pollTask = Task { @MainActor in - do { - let inputJSON = try input.encodedJSON() - let handle = try await WorkflowRuntimeClient.shared.startWorkflow( - WorkflowStartRequest( - kind: .pluginFactoryCreate, - sessionID: sessionID, - agentID: "ui", - inputJSON: inputJSON, - principal: .agent(sessionID: sessionID, agentID: "ui"), - helperAPIKey: helperAPIKey, - helperReviewerModelJSON: helperReviewerModelJSON - ) - ) - workflowID = handle.workflowID - await pollUntilTerminal() - } catch { - phase = .failed(step: .creating, message: error.localizedDescription) - } - } + startFactoryCreation() } func saveCredentialsAndFinish() { @@ -390,14 +270,10 @@ final class PluginCreationController: ObservableObject { drafts: credentialDrafts ) markProgressCompleted("credentials") - startCreation( - sessionID: creationSessionID, - helperAPIKey: creationAPIKey, - helperReviewerModelJSON: creationReviewerModelJSON - ) + startFactoryCreation() } catch { phase = .failed( - step: .auth, + step: .credentials, message: "Could not save credentials: \(error.localizedDescription)" ) } @@ -407,13 +283,11 @@ final class PluginCreationController: ObservableObject { switch phase { case .failed(let step, _, _): switch step { - case .type: phase = .chooseType - case .news: phase = .chooseNews - case .name: phase = .chooseName - case .auth: phase = .chooseName - case .vendor, .description, .creating: - selectedVendor = .slack - phase = .chooseVendor + case .goal: phase = .goal + case .skill, .news: phase = .skill + case .preview: phase = .preview + case .credentials: phase = .preview + case .build: phase = .preview } default: phase = .intro @@ -424,13 +298,119 @@ final class PluginCreationController: ObservableObject { showIntro() } + private func startNewsCreation() { + let sources = skillDraft.newsSourceURLs.map { url in + NewsSource(label: URL(string: url)?.host ?? url, url: url) + } + let spec = NewsReaderSpec( + name: skillDraft.pluginName, + topics: skillDraft.newsTopics, + sources: sources, + mode: PluginSkillDraftPlanner.inferNewsMode(from: skillDraft), + maxCount: 20, + schedule: .off + ) + if let blocked = spec.sources.compactMap({ source -> (NewsSource, String)? in + guard let url = URL(string: source.url), + let reason = NewsPaywall.preflightRejection(url: url) else { return nil } + return (source, reason) + }).first { + phase = .failed( + step: .news, + message: NewsReaderError.paywalled(url: blocked.0.url, detail: blocked.1).errorDescription + ?? "This source is behind a paywall, which is not supported yet." + ) + return + } + phase = .creating + statusMessage = "Starting news reader in Docker…" + progressSteps = [ + ProgressStepState(id: "skill", title: "Write SKILL.md", status: .completed), + ProgressStepState(id: "sources", title: "Check sources", status: .active), + ProgressStepState(id: "fetch", title: "Run news reader", status: .pending), + ] + if spec.mode == .summary { + progressSteps.append( + ProgressStepState(id: "summary", title: "Summarize with AI", status: .pending) + ) + } + pollTask?.cancel() + pollTask = Task { @MainActor in + do { + let saved = try await NewsReaderStore.shared.create(spec) + setProgressStep("sources", status: .completed) + setProgressStep("fetch", status: .completed) + if spec.mode == .summary { + setProgressStep("summary", status: .completed) + } + phase = .succeeded(pluginID: saved.id, outcome: .newsList) + } catch let error as NewsReaderError { + setProgressStep("sources", status: .failed) + phase = .failed( + step: .news, + message: error.localizedDescription, + technicalDetail: String(describing: error) + ) + } catch { + setProgressStep("sources", status: .failed) + phase = .failed(step: .news, message: error.localizedDescription) + } + } + } + + private func startFactoryCreation() { + guard let creationAPIKey, !creationAPIKey.isEmpty else { + phase = .failed( + step: .build, + message: "Add an API key in Settings before creating a plugin." + ) + return + } + do { + let input = try PluginFactoryCreateInput.makeFromSkillDraft(skillDraft, auth: pendingAuth) + cancelPolling() + phase = .creating + statusMessage = "Building your plugin…" + resetProgressSteps() + pollAfterSeq = 0 + + pollTask = Task { @MainActor in + do { + let inputJSON = try input.encodedJSON() + let handle = try await WorkflowRuntimeClient.shared.startWorkflow( + WorkflowStartRequest( + kind: .pluginFactoryCreate, + sessionID: creationSessionID, + agentID: "ui", + inputJSON: inputJSON, + principal: .agent(sessionID: creationSessionID, agentID: "ui"), + helperAPIKey: creationAPIKey, + helperReviewerModelJSON: creationReviewerModelJSON + ) + ) + workflowID = handle.workflowID + await pollUntilTerminal() + } catch { + phase = .failed(step: .build, message: error.localizedDescription) + } + } + } catch { + phase = .failed(step: .skill, message: error.localizedDescription) + } + } + private func resetProgressSteps() { progressSteps = [ - ProgressStepState(id: "docs", title: "Read vendor API docs", status: .pending), - ProgressStepState(id: "factory", title: "Build and test plugin", status: .pending), + ProgressStepState(id: "skill", title: "Write SKILL.md", status: .completed), + ProgressStepState(id: "docs", title: "Read API docs", status: .pending), + ProgressStepState(id: "factory", title: "Build guest program", status: .pending), ProgressStepState(id: "review", title: "Safety review", status: .pending), - ProgressStepState(id: "credentials", title: "Save credentials to Keychain", status: .pending), + ProgressStepState(id: "trial", title: "Trial run", status: .pending), + ProgressStepState(id: "credentials", title: "Save credentials", status: .pending), ] + if skillDraft.plannedKind == .customCapability { + setProgressStep("docs", status: .completed) + } } private func setProgressStep(_ id: String, status: ProgressStepState.Status) { @@ -450,16 +430,15 @@ final class PluginCreationController: ObservableObject { switch stage?.lowercased() { case "crawl", "docs": setProgressStep("docs", status: .failed) - case "factory", "build", "review": + case "factory", "build": markProgressCompleted("docs") setProgressStep("factory", status: .failed) + case "review": + markProgressCompleted("docs") + markProgressCompleted("factory") setProgressStep("review", status: .failed) default: - if progressSteps.first(where: { $0.id == "docs" })?.status == .completed { - setProgressStep("factory", status: .failed) - } else { - setProgressStep("docs", status: .failed) - } + setProgressStep("factory", status: .failed) } } @@ -476,6 +455,7 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("factory") markProgressCompleted("review") + markProgressCompleted("trial") default: break } @@ -488,10 +468,11 @@ final class PluginCreationController: ObservableObject { if message.contains("review decision=approved") { markProgressCompleted("factory") markProgressCompleted("review") + markProgressActive("trial") } if message.contains("review decision=rejected") || message.contains("review rejected=") { markProgressCompleted("factory") - markProgressActive("review") + setProgressStep("review", status: .failed) } default: break @@ -520,30 +501,27 @@ final class PluginCreationController: ObservableObject { markProgressCompleted("docs") markProgressCompleted("factory") markProgressCompleted("review") + markProgressCompleted("trial") await PluginFactoryListStore.shared.reload() if let pluginID = parseSuccessPluginID(result.resultJSON) { markProgressCompleted("credentials") - phase = .succeeded(pluginID: pluginID) + phase = .succeeded(pluginID: pluginID, outcome: .plugin) + } else if let saved = PluginFactoryListStore.shared.releases.first { + markProgressCompleted("credentials") + phase = .succeeded(pluginID: saved.pluginID, outcome: .plugin) } else { - if let saved = PluginFactoryListStore.shared.releases.first { - markProgressCompleted("credentials") - phase = .succeeded(pluginID: saved.pluginID) - } else { - phase = .failed( - step: .creating, - message: """ - The connector was not saved. Creation reported success but no plugin release was found. - """, - technicalDetail: result.resultJSON - ) - } + phase = .failed( + step: .build, + message: "The plugin was not saved. Creation reported success but no release was found.", + technicalDetail: result.resultJSON + ) } cancelPolling() return case .failed: let stage = result.events.last(where: { $0.kind == "log" })?.stage markProgressFailed(fromStage: stage) - let raw = result.errorMessage ?? "Connector creation failed." + let raw = result.errorMessage ?? "Plugin creation failed." let presentation = PluginFactoryCreateFailureMessage.presentation(raw) phase = .failed( step: PluginFactoryCreateInput.failureStep(forStage: stage), @@ -555,8 +533,8 @@ final class PluginCreationController: ObservableObject { case .cancelled: markProgressFailed(fromStage: "factory") phase = .failed( - step: .creating, - message: "The connector was not saved. Creation was cancelled before it finished." + step: .build, + message: "Plugin creation was cancelled before it finished." ) cancelPolling() return @@ -564,7 +542,7 @@ final class PluginCreationController: ObservableObject { break } } catch { - phase = .failed(step: .creating, message: error.localizedDescription) + phase = .failed(step: .build, message: error.localizedDescription) cancelPolling() return } @@ -581,29 +559,13 @@ final class PluginCreationController: ObservableObject { return result.pluginID } - private func refreshDefaultConnectorName() { - let existing = PluginFactoryListStore.shared.pluginIDs - let generated = ConnectorPluginNaming.defaultPluginID( - vendor: selectedVendor, - existingIDs: existing - ) - if connectorName.isEmpty - || ConnectorPluginNaming.isGeneratedDefault(pluginID: connectorName, vendor: selectedVendor) - || connectorName == generatedConnectorName { - connectorName = generated - } - generatedConnectorName = generated - } - - private func normalizedConnectorName() -> String? { - let trimmed = connectorName.trimmingCharacters(in: .whitespacesAndNewlines) - return try? PluginID.normalized(trimmed).rawValue - } - private func startAuthDiscovery() { discoverTask?.cancel() pendingAuth = nil - let vendor = selectedVendor + guard let vendor = skillDraft.inferredConnectorVendor else { + phase = .failed(step: .skill, message: "Could not determine which messaging service this plugin targets.") + return + } let sessionID = creationSessionID let apiKey = creationAPIKey let reviewerJSON = creationReviewerModelJSON @@ -674,20 +636,20 @@ final class PluginCreationController: ObservableObject { private func presentCredentialsOrFail(auth: ConnectorAuthDiscovery) { guard auth.authScheme.isSupportedInWizard else { phase = .failed( - step: .auth, + step: .credentials, message: "OAuth connectors are not available yet. Use a bot token or API key." ) return } - guard let pluginID = normalizedConnectorName() else { - phase = .chooseName + guard let pluginID = try? skillDraft.normalizedPluginID() else { + phase = .skill return } let descriptors = auth.secrets.map(\.descriptor) guard !descriptors.isEmpty else { phase = .failed( - step: .auth, - message: "Could not determine which credentials this connector needs." + step: .credentials, + message: "Could not determine which credentials this plugin needs." ) return } @@ -704,7 +666,7 @@ final class PluginCreationController: ObservableObject { let env = PluginSecretDevelopmentSource.resolve(pluginID: pluginID, fieldID: field.id) ?? "" return (field.id, env) }) - statusMessage = auth.setupHint ?? "Enter the credentials this connector needs. They are stored in Keychain on your Mac." + statusMessage = auth.setupHint ?? "Enter the credentials this plugin needs. They are stored in Keychain on your Mac." phase = .collectCredentials(pluginID: pluginID) } diff --git a/ui/ui/Plugins/PluginsWorkspaceView.swift b/ui/ui/Plugins/PluginsWorkspaceView.swift index 5034315..11467e6 100644 --- a/ui/ui/Plugins/PluginsWorkspaceView.swift +++ b/ui/ui/Plugins/PluginsWorkspaceView.swift @@ -10,6 +10,9 @@ struct PluginsWorkspaceView: View { let onOpenMessagingConnector: (String) -> Void var onOpenNewsReader: (String) -> Void = { _ in } + @State private var newsTopicDraft = "" + @State private var newsURLDraft = "" + var body: some View { ZStack { Color(red: 252.0 / 255.0, green: 252.0 / 255.0, blue: 250.0 / 255.0) @@ -33,8 +36,8 @@ struct PluginsWorkspaceView: View { isPresented: true, minWidth: 400, minHeight: 0, - maxWidth: 520, - maxHeight: controller.phase == .chooseNews ? 720 : 560, + maxWidth: 560, + maxHeight: modalMaxHeight, onBackdropDismiss: canDismiss ? { controller.showIntro() } : nil, onEscape: canDismiss ? { controller.showIntro() } : nil, header: { @@ -57,6 +60,13 @@ struct PluginsWorkspaceView: View { ) } + private var modalMaxHeight: CGFloat { + switch controller.phase { + case .skill, .preview: return 720 + default: return 560 + } + } + private var canDismiss: Bool { switch controller.phase { case .creating, .discoveringAuth: return false @@ -67,17 +77,31 @@ struct PluginsWorkspaceView: View { private var modalTitle: String { switch controller.phase { case .intro: return "Create a plugin" - case .chooseType: return "Create a plugin" - case .chooseVendor: return "Choose a vendor" - case .chooseName: return "Name this connector" - case .chooseNews: return "News list" + case .goal: return "What should it do?" + case .skill: return "Define the skill" + case .preview: return "Preview" case .discoveringAuth: return "Reading authentication docs" - case .creating: return controller.selectedType == .newsReader ? "Creating news list" : "Creating connector" - case .collectCredentials: return "Connector credentials" - case .failed: return controller.selectedType == .newsReader ? "Could not create news list" : "Could not create connector" - case .succeeded: return "Connector ready" - case .succeededNews: return "News list ready" + case .creating: return creatingTitle + case .collectCredentials: return "Plugin credentials" + case .failed: return failureTitle + case .succeeded(_, let outcome): + return outcome == .newsList ? "News list ready" : "Plugin ready" + } + } + + private var creatingTitle: String { + switch controller.skillDraft.plannedKind { + case .newsDigest: return "Creating news list" + case .messagingConnector: return "Creating connector" + case .customCapability: return "Building plugin" + } + } + + private var failureTitle: String { + if case .failed(let step, _, _) = controller.phase, step == .news { + return "Could not create news list" } + return "Could not create plugin" } @ViewBuilder @@ -85,101 +109,31 @@ struct PluginsWorkspaceView: View { switch controller.phase { case .intro: Text(""" - A plugin is a small program that extends the capabilities of Derrick. This form will guide you through the process of building your own unique and secure plugins. + Describe what you want Derrick to do. Derrick will draft a skill, show you a preview, and build a secure plugin package. """) .font(.body) .fixedSize(horizontal: false, vertical: true) - case .chooseType: + case .goal: VStack(alignment: .leading, spacing: 10) { - Text("What kind of plugin do you want?") + Text("What do you want Derrick to do?") .font(.subheadline) .foregroundStyle(.secondary) - typeButton( - title: "Connector", - subtitle: "Messaging integration (Slack)", - type: .connector, - enabled: true - ) - typeButton( - title: "News reader", - subtitle: "Saved lists from topics and sources", - type: .newsReader, - enabled: true - ) - typeButton( - title: "Custom", - subtitle: "Coming soon", - type: .custom, - enabled: false + TextField( + "e.g. Send Slack messages from Messaging, or fetch tech headlines", + text: controller.skillDraftBinding(\.goal), + axis: .vertical ) + .textFieldStyle(.roundedBorder) + .lineLimit(3...6) + .accessibilityIdentifier("plugin-goal-field") } - case .chooseVendor: - VStack(alignment: .leading, spacing: 10) { - Text("Which service should this connector use?") - .font(.subheadline) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 120), spacing: 8)], spacing: 8) { - ForEach(PluginFactoryCreateInput.ConnectorVendor.allCases, id: \.self) { vendor in - let enabled = vendor.isSelectableInWizard - Button { - guard enabled else { return } - controller.selectedVendor = vendor - } label: { - VStack(spacing: 4) { - Text(vendor.displayName) - .font(.subheadline.weight(.medium)) - if !enabled { - Text("Soon") - .font(.caption2) - .foregroundStyle(.secondary) - } - } - .frame(maxWidth: .infinity) - .padding(.vertical, 10) - .background( - enabled && controller.selectedVendor == vendor - ? Color.accentColor.opacity(0.15) - : Color.primary.opacity(enabled ? 0.05 : 0.03) - ) - .foregroundStyle(enabled ? .primary : .secondary) - .clipShape(RoundedRectangle(cornerRadius: 10)) - } - .buttonStyle(.plain) - .disabled(!enabled) - .accessibilityLabel(enabled ? vendor.displayName : "\(vendor.displayName), coming soon") - } - } - if controller.selectedVendor == .custom { - TextField("Vendor name", text: $controller.customVendorName) - .textFieldStyle(.roundedBorder) - } - Text("This connector lists conversations as tabs, including reply threads, then sends and receives new messages.") - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - .padding(.top, 4) - if controller.selectedVendor == .slack { - Text(ConnectorReplyThreadAccessMessage.slackSetupHint) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - } - - case .chooseName: - VStack(alignment: .leading, spacing: 10) { - Text("Give this connector a name. You can change the default.") - .font(.subheadline) - .foregroundStyle(.secondary) - TextField("Connector name", text: $controller.connectorName) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("connector-plugin-name") - } + case .skill: + skillBuilderForm - case .chooseNews: - newsReaderForm + case .preview: + previewForm case .discoveringAuth, .creating: VStack(alignment: .leading, spacing: 14) { @@ -198,7 +152,7 @@ struct PluginsWorkspaceView: View { .font(.subheadline) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) - if controller.selectedVendor == .slack { + if controller.skillDraft.inferredConnectorVendor == .slack { Text(ConnectorReplyThreadAccessMessage.slackSetupHint) .font(.caption) .foregroundStyle(.secondary) @@ -210,58 +164,10 @@ struct PluginsWorkspaceView: View { } case .failed(_, let message, let technicalDetail): - VStack(alignment: .leading, spacing: 10) { - if controller.selectedType == .newsReader { - Label( - message.localizedCaseInsensitiveContains("paywall") - ? "Blocked because of a paywall" - : "News list was not created", - systemImage: "exclamationmark.triangle.fill" - ) - .font(.subheadline.weight(.semibold)) - .foregroundStyle(message.localizedCaseInsensitiveContains("paywall") ? Color.orange : Color.secondary) - .accessibilityIdentifier( - message.localizedCaseInsensitiveContains("paywall") - ? "news-paywall-blocked" - : "news-create-failed" - ) - if message.localizedCaseInsensitiveContains("paywall") { - Text(NewsPaywall.userWarning) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - } else { - Label("Nothing was installed", systemImage: "minus.circle") - .font(.subheadline.weight(.semibold)) - .foregroundStyle(.secondary) - Text("Your sidebar and Messaging are unchanged.") - .font(.caption) - .foregroundStyle(.secondary) - } - Text(message) - .font(.body) - .fixedSize(horizontal: false, vertical: true) - .padding(.top, 4) - if let technicalDetail, !technicalDetail.isEmpty { - DisclosureGroup("Technical details") { - Text(technicalDetail) - .font(.caption) - .foregroundStyle(.secondary) - .fixedSize(horizontal: false, vertical: true) - .textSelection(.enabled) - } - } - } + failureBody(message: message, technicalDetail: technicalDetail) - case .succeeded(let pluginID): - Text("Your connector /\(pluginID) is ready. Open it to start talking in Messaging.") - .font(.body) - .fixedSize(horizontal: false, vertical: true) - case .succeededNews: - Text("Your news list is ready. Every article includes a source link.") - .font(.body) - .fixedSize(horizontal: false, vertical: true) + case .succeeded(let pluginID, let outcome): + successBody(pluginID: pluginID, outcome: outcome) } } @@ -271,62 +177,48 @@ struct PluginsWorkspaceView: View { case .intro: HStack { Spacer() - Button("Create plugin") { controller.beginCreate() } + Button("Begin") { controller.beginCreate() } .buttonStyle(ModalPrimaryButtonStyle()) .keyboardShortcut(.defaultAction) } - case .chooseType: + case .goal: HStack { Button("Back") { controller.showIntro() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Continue") { controller.confirmTypeSelection() } + Button("Continue") { controller.continueFromGoal() } .buttonStyle(ModalPrimaryButtonStyle()) - .disabled( - controller.selectedType != .connector - && controller.selectedType != .newsReader - ) + .disabled(!controller.canContinueFromGoal) .keyboardShortcut(.defaultAction) } - case .chooseVendor: - HStack { - Button("Back") { controller.goBackToTypeSelection() } - .buttonStyle(ModalSecondaryButtonStyle()) - Spacer() - Button("Continue") { - controller.confirmVendor( - sessionID: sessionID, - helperAPIKey: helperAPIKey, - helperReviewerModelJSON: helperReviewerModelJSON - ) - } - .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!sessionReady || !controller.canConfirmVendor) - .keyboardShortcut(.defaultAction) - } - - case .chooseName: + case .skill: HStack { - Button("Back") { controller.goBackToVendor() } + Button("Back") { controller.goBackToGoal() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Continue") { controller.confirmConnectorName() } + Button("Preview") { controller.continueToPreview() } .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!controller.canConfirmName) + .disabled(!controller.canContinueFromSkill) .keyboardShortcut(.defaultAction) } - case .chooseNews: + case .preview: HStack { - Button("Back") { controller.goBackToTypeSelection() } + Button("Back") { controller.goBackToSkill() } .buttonStyle(ModalSecondaryButtonStyle()) Spacer() - Button("Create") { controller.startNewsCreation() } - .buttonStyle(ModalPrimaryButtonStyle()) - .disabled(!controller.canConfirmNews) - .keyboardShortcut(.defaultAction) + Button(buildButtonTitle) { + controller.confirmPreview( + sessionID: sessionID, + helperAPIKey: helperAPIKey, + helperReviewerModelJSON: helperReviewerModelJSON + ) + } + .buttonStyle(ModalPrimaryButtonStyle()) + .disabled(!sessionReady || !controller.canContinueFromSkill) + .keyboardShortcut(.defaultAction) } case .discoveringAuth, .creating: @@ -352,28 +244,448 @@ struct PluginsWorkspaceView: View { .buttonStyle(ModalSecondaryButtonStyle()) } - case .succeeded(let pluginID): + case .succeeded(let pluginID, let outcome): HStack { - Button("Done") { controller.dismissSuccess() } - .buttonStyle(ModalSecondaryButtonStyle()) + if outcome != .plugin || controller.skillDraft.plannedKind == .messagingConnector { + Button("Done") { controller.dismissSuccess() } + .buttonStyle(ModalSecondaryButtonStyle()) + } Spacer() - Button("Open connector") { - onOpenMessagingConnector(pluginID) + switch outcome { + case .newsList: + Button("Open news list") { + onOpenNewsReader(pluginID) + } + .buttonStyle(ModalPrimaryButtonStyle()) + .keyboardShortcut(.defaultAction) + case .plugin where controller.skillDraft.plannedKind == .messagingConnector: + Button("Open connector") { + onOpenMessagingConnector(pluginID) + } + .buttonStyle(ModalPrimaryButtonStyle()) + .keyboardShortcut(.defaultAction) + case .plugin: + Button("Done") { controller.dismissSuccess() } + .buttonStyle(ModalPrimaryButtonStyle()) + .keyboardShortcut(.defaultAction) } - .buttonStyle(ModalPrimaryButtonStyle()) - .keyboardShortcut(.defaultAction) } - case .succeededNews(let readerID): + } + } + + private var buildButtonTitle: String { + switch controller.skillDraft.plannedKind { + case .newsDigest: return "Create" + default: return "Build plugin" + } + } + + private var skillBuilderForm: some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + plannedKindBadge + + VStack(alignment: .leading, spacing: 6) { + Text("Purpose") + .font(.caption) + .foregroundStyle(.secondary) + TextField("What this plugin does", text: controller.skillDraftBinding(\.purpose), axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(2...4) + } + + VStack(alignment: .leading, spacing: 6) { + Text("When to use") + .font(.caption) + .foregroundStyle(.secondary) + FlowLayout(spacing: 8) { + ForEach(PluginSkillDraft.Trigger.allCases, id: \.self) { trigger in + triggerChip(trigger) + } + } + } + + examplesSection + + if controller.skillDraft.plannedKind == .newsDigest { + newsFieldsSection + } + + nameFieldSection + + if let blocked = controller.skillDraft.buildBlockedReason { + Text(blocked) + .font(.caption) + .foregroundStyle(.orange) + .fixedSize(horizontal: false, vertical: true) + } + } + } + } + + private var plannedKindBadge: some View { + let (label, icon) = plannedKindPresentation + return Label(label, systemImage: icon) + .font(.caption.weight(.semibold)) + .foregroundStyle(.secondary) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(Color.primary.opacity(0.05), in: Capsule()) + } + + private var plannedKindPresentation: (String, String) { + switch controller.skillDraft.plannedKind { + case .messagingConnector: + let vendor = controller.skillDraft.inferredConnectorVendor?.displayName ?? "Messaging" + return ("\(vendor) connector", "bubble.left.and.bubble.right") + case .newsDigest: + return ("News list", "newspaper") + case .customCapability: + return ("Custom capability", "wand.and.stars") + } + } + + private var examplesSection: some View { + VStack(alignment: .leading, spacing: 8) { HStack { - Button("Done") { controller.dismissSuccess() } - .buttonStyle(ModalSecondaryButtonStyle()) + Text("Examples") + .font(.caption) + .foregroundStyle(.secondary) Spacer() - Button("Open news list") { - onOpenNewsReader(readerID) + Button("Add example") { controller.addExample() } + .font(.caption) + } + ForEach(controller.skillDraft.examples) { example in + VStack(alignment: .leading, spacing: 6) { + TextField("You say…", text: exampleBinding(example.id, field: .userSays)) + .textFieldStyle(.roundedBorder) + TextField("Plugin does…", text: exampleBinding(example.id, field: .pluginDoes)) + .textFieldStyle(.roundedBorder) + if controller.skillDraft.examples.count > 1 { + Button("Remove") { controller.removeExample(id: example.id) } + .font(.caption) + } + } + .padding(10) + .background(Color.primary.opacity(0.04), in: RoundedRectangle(cornerRadius: 8)) + } + } + } + + private enum ExampleField { case userSays, pluginDoes } + + private func exampleBinding(_ id: String, field: ExampleField) -> Binding { + Binding( + get: { + guard let index = controller.skillDraft.examples.firstIndex(where: { $0.id == id }) else { + return "" + } + switch field { + case .userSays: return controller.skillDraft.examples[index].userSays + case .pluginDoes: return controller.skillDraft.examples[index].pluginDoes + } + }, + set: { newValue in + switch field { + case .userSays: controller.updateExample(id: id, userSays: newValue) + case .pluginDoes: controller.updateExample(id: id, pluginDoes: newValue) + } + } + ) + } + + private var newsFieldsSection: some View { + VStack(alignment: .leading, spacing: 12) { + Text("Topics") + .font(.caption) + .foregroundStyle(.secondary) + LazyVGrid(columns: [GridItem(.adaptive(minimum: 110), spacing: 8)], spacing: 8) { + ForEach(NewsPresetTopic.allCases) { topic in + let on = controller.skillDraft.newsTopics.contains(topic.displayName) + Button { + if on { + controller.removeNewsTopic(topic.displayName) + } else if !controller.skillDraft.newsTopics.contains(topic.displayName) { + controller.addNewsTopicFromPreset(topic.displayName) + } + } label: { + Text(topic.displayName) + .font(.caption.weight(.medium)) + .frame(maxWidth: .infinity) + .padding(.vertical, 8) + .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } + .buttonStyle(.plain) + } + } + HStack { + TextField("Add a custom topic", text: $newsTopicDraft) + .textFieldStyle(.roundedBorder) + .onSubmit { addNewsTopicFromDraft() } + Button("Add") { addNewsTopicFromDraft() } + } + Text("Sources") + .font(.caption) + .foregroundStyle(.secondary) + LazyVGrid(columns: [GridItem(.adaptive(minimum: 130), spacing: 8)], spacing: 8) { + ForEach(NewsPresetSource.allCases) { source in + let url = source.source.url + let on = controller.skillDraft.newsSourceURLs.contains(url) + Button { + if on { + controller.removeNewsURL(url) + } else { + controller.addNewsURL(url) + } + } label: { + Text(source.source.label) + .font(.caption.weight(.medium)) + .frame(maxWidth: .infinity) + .padding(.vertical, 8) + .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } + .buttonStyle(.plain) + } + } + HStack { + TextField("https://…", text: $newsURLDraft) + .textFieldStyle(.roundedBorder) + .accessibilityIdentifier("news-url-field") + .onSubmit { addNewsURLFromDraft() } + Button("Add URL") { addNewsURLFromDraft() } + } + ForEach(controller.skillDraft.newsSourceURLs, id: \.self) { url in + HStack { + Text(url) + .font(.caption) + .lineLimit(1) + Spacer() + Button("Remove") { controller.removeNewsURL(url) } + .font(.caption) + } + } + HStack(alignment: .top, spacing: 8) { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.orange) + Text(NewsPaywall.userWarning) + .fixedSize(horizontal: false, vertical: true) + } + .font(.caption) + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.orange.opacity(0.12), in: RoundedRectangle(cornerRadius: 8)) + } + } + + private func addNewsTopicFromDraft() { + let topic = newsTopicDraft.trimmingCharacters(in: .whitespacesAndNewlines) + guard !topic.isEmpty else { return } + controller.addNewsTopicFromPreset(topic) + newsTopicDraft = "" + } + + private func triggerChip(_ trigger: PluginSkillDraft.Trigger) -> some View { + let available = controller.skillDraft.isTriggerAvailable(trigger) + let on = available && controller.skillDraft.triggers.contains(trigger) + return Button { + controller.toggleTrigger(trigger) + } label: { + Text(trigger.label) + .font(.caption.weight(.medium)) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .background(chipBackground(on: on, available: available)) + .foregroundStyle(available ? .primary : .tertiary) + .clipShape(Capsule()) + } + .buttonStyle(.plain) + .disabled(!available) + .help(triggerHelp(trigger, available: available)) + } + + private func chipBackground(on: Bool, available: Bool) -> Color { + guard available else { return Color.primary.opacity(0.03) } + return on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05) + } + + private func triggerHelp( + _ trigger: PluginSkillDraft.Trigger, + available: Bool + ) -> String { + guard !available else { return "" } + switch controller.skillDraft.plannedKind { + case .newsDigest: + switch trigger { + case .mention: + return "News lists live in the sidebar, not as /slash commands." + case .messaging: + return "News lists are not messaging connectors." + default: + return "Not available for news lists." + } + case .messagingConnector: + switch trigger { + case .schedule: + return "Connectors respond in Messaging, not on a timer." + default: + return "Not available for messaging connectors." + } + case .customCapability: + switch trigger { + case .messaging: + return "Only messaging connectors use the Messaging tab." + default: + return "Not available for this plugin type." + } + } + } + + private var nameFieldSection: some View { + let isNews = controller.skillDraft.plannedKind == .newsDigest + return VStack(alignment: .leading, spacing: 6) { + Text(isNews ? "List name" : "Plugin name") + .font(.caption) + .foregroundStyle(.secondary) + TextField(isNews ? "List name" : "Plugin name", text: controller.skillDraftBinding(\.pluginName)) + .textFieldStyle(.roundedBorder) + .accessibilityIdentifier("connector-plugin-name") + if isNews { + Text("This name appears in the sidebar. News lists are not slash commands.") + .font(.caption2) + .foregroundStyle(.tertiary) + } else if controller.canConfirmPluginName { + Text("Invoke this plugin in chat as /\(normalizedPluginID()).") + .font(.caption2) + .foregroundStyle(.tertiary) + } else { + Text("Use letters, numbers, and hyphens (for example tech-news).") + .font(.caption2) + .foregroundStyle(.orange) + } + } + } + + private func normalizedPluginID() -> String { + let trimmed = controller.skillDraft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) + if let normalized = try? PluginID.normalized(trimmed) { + return normalized.rawValue + } + return trimmed.isEmpty ? "plugin" : trimmed + } + + private func addNewsURLFromDraft() { + controller.addNewsURL(newsURLDraft) + newsURLDraft = "" + } + + private var previewForm: some View { + ScrollView { + VStack(alignment: .leading, spacing: 16) { + plannedKindBadge + + nameFieldSection + + if controller.skillDraft.plannedKind == .newsDigest { + Text( + "Display mode: \(PluginSkillDraftPlanner.inferNewsMode(from: controller.skillDraft).displayName)" + ) + .font(.caption) + .foregroundStyle(.secondary) + } + + VStack(alignment: .leading, spacing: 6) { + Text("Scenarios") + .font(.caption) + .foregroundStyle(.secondary) + ForEach(controller.skillDraft.previewScenarios(), id: \.self) { scenario in + Text(scenario) + .font(.subheadline) + .fixedSize(horizontal: false, vertical: true) + } + } + + VStack(alignment: .leading, spacing: 6) { + Text("Package") + .font(.caption) + .foregroundStyle(.secondary) + ForEach(controller.skillDraft.packageOutline(), id: \.self) { line in + Label(line, systemImage: "doc") + .font(.caption) + .foregroundStyle(.secondary) + } + } + + VStack(alignment: .leading, spacing: 6) { + Text("SKILL.md") + .font(.caption) + .foregroundStyle(.secondary) + Text(controller.skillDraft.skillMarkdown()) + .font(.caption.monospaced()) + .foregroundStyle(.secondary) + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.primary.opacity(0.04), in: RoundedRectangle(cornerRadius: 8)) + .textSelection(.enabled) + } + } + } + } + + @ViewBuilder + private func failureBody(message: String, technicalDetail: String?) -> some View { + VStack(alignment: .leading, spacing: 10) { + if case .failed(let step, _, _) = controller.phase, step == .news { + Label( + message.localizedCaseInsensitiveContains("paywall") + ? "Blocked because of a paywall" + : "News list was not created", + systemImage: "exclamationmark.triangle.fill" + ) + .font(.subheadline.weight(.semibold)) + .foregroundStyle(message.localizedCaseInsensitiveContains("paywall") ? Color.orange : Color.secondary) + } else { + Label("Nothing was installed", systemImage: "minus.circle") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(.secondary) + Text("Your sidebar and Messaging are unchanged.") + .font(.caption) + .foregroundStyle(.secondary) + } + Text(message) + .font(.body) + .fixedSize(horizontal: false, vertical: true) + .padding(.top, 4) + if let technicalDetail, !technicalDetail.isEmpty { + DisclosureGroup("Technical details") { + Text(technicalDetail) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + .textSelection(.enabled) } - .buttonStyle(ModalPrimaryButtonStyle()) - .keyboardShortcut(.defaultAction) + } + } + } + + @ViewBuilder + private func successBody(pluginID: String, outcome: PluginCreationController.SuccessOutcome) -> some View { + switch outcome { + case .newsList: + Text("Your news list is ready. Every article includes a source link.") + .font(.body) + .fixedSize(horizontal: false, vertical: true) + case .plugin: + if controller.skillDraft.plannedKind == .messagingConnector { + Text("Your connector /\(pluginID) is ready. Open it to start talking in Messaging.") + .font(.body) + .fixedSize(horizontal: false, vertical: true) + } else { + Text("Your plugin /\(pluginID) is ready.") + .font(.body) + .fixedSize(horizontal: false, vertical: true) } } } @@ -443,169 +755,47 @@ struct PluginsWorkspaceView: View { set: { controller.credentialDrafts[id] = $0 } ) } +} - private func typeButton( - title: String, - subtitle: String, - type: PluginFactoryCreateInput.PluginType, - enabled: Bool - ) -> some View { - Button { - guard enabled else { return } - controller.selectType(type) - } label: { - HStack(alignment: .top, spacing: 10) { - VStack(alignment: .leading, spacing: 4) { - Text(title) - .font(.body.weight(.semibold)) - .foregroundStyle(enabled ? .primary : .secondary) - Text(subtitle) - .font(.caption) - .foregroundStyle(.secondary) - } - Spacer() - if !enabled { - Text("Soon") - .font(.caption2) - .foregroundStyle(.secondary) - } else if controller.selectedType == type { - Image(systemName: "checkmark.circle.fill") - .foregroundStyle(Color.accentColor) - } - } - .padding(12) - .frame(maxWidth: .infinity, alignment: .leading) - .background( - controller.selectedType == type && enabled - ? Color.accentColor.opacity(0.12) - : Color.primary.opacity(enabled ? 0.05 : 0.03) +/// Simple horizontal flow for trigger chips when `Layout` is unavailable. +private struct FlowLayout: Layout { + var spacing: CGFloat = 8 + + func sizeThatFits(proposal: ProposedViewSize, subviews: Subviews, cache: inout ()) -> CGSize { + let result = arrange(proposal: proposal, subviews: subviews) + return result.size + } + + func placeSubviews(in bounds: CGRect, proposal: ProposedViewSize, subviews: Subviews, cache: inout ()) { + let result = arrange(proposal: proposal, subviews: subviews) + for (index, frame) in result.frames.enumerated() { + subviews[index].place( + at: CGPoint(x: bounds.minX + frame.minX, y: bounds.minY + frame.minY), + proposal: ProposedViewSize(frame.size) ) - .clipShape(RoundedRectangle(cornerRadius: 10)) } - .buttonStyle(.plain) - .disabled(!enabled) } - private var newsReaderForm: some View { - ScrollView { - VStack(alignment: .leading, spacing: 14) { - TextField("Name this list", text: $controller.newsName) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("news-list-name") - Text("Topics") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 110), spacing: 8)], spacing: 8) { - ForEach(NewsPresetTopic.allCases) { topic in - let on = controller.selectedNewsTopics.contains(topic) - Button { - if on { - controller.selectedNewsTopics.remove(topic) - } else { - controller.selectedNewsTopics.insert(topic) - } - } label: { - Text(topic.displayName) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("Add a custom topic", text: $controller.newsTopicDraft) - .textFieldStyle(.roundedBorder) - .onSubmit { controller.addNewsTopic() } - Button("Add") { controller.addNewsTopic() } - } - if !controller.extraNewsTopics.isEmpty { - Text(controller.extraNewsTopics.joined(separator: ", ")) - .font(.caption) - .foregroundStyle(.secondary) - } - Text("Sources") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 130), spacing: 8)], spacing: 8) { - ForEach(NewsPresetSource.allCases) { source in - let on = controller.selectedNewsSources.contains(source) - Button { - if on { - controller.selectedNewsSources.remove(source) - } else { - controller.selectedNewsSources.insert(source) - } - } label: { - Text(source.source.label) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("https://…", text: $controller.newsURLDraft) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("news-url-field") - .onSubmit { controller.addNewsURL() } - Button("Add URL") { controller.addNewsURL() } - } - ForEach(controller.extraNewsURLs, id: \.self) { url in - VStack(alignment: .leading, spacing: 2) { - HStack { - Text(url) - .font(.caption) - .lineLimit(1) - Spacer() - Button("Remove") { controller.removeNewsURL(url) } - .font(.caption) - } - if let reason = newsURLPaywallReason(url) { - Text(reason) - .font(.caption2) - .foregroundStyle(.orange) - } - } - } - HStack(alignment: .top, spacing: 8) { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundStyle(.orange) - Text(NewsPaywall.userWarning) - .fixedSize(horizontal: false, vertical: true) - } - .font(.caption) - .padding(10) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.orange.opacity(0.12), in: RoundedRectangle(cornerRadius: 8)) - .accessibilityElement(children: .combine) - .accessibilityIdentifier("news-paywall-warning") - Picker("Mode", selection: $controller.newsMode) { - ForEach(NewsReaderMode.allCases, id: \.self) { mode in - Text(mode.displayName).tag(mode) - } - } - Stepper("Up to \(controller.newsMaxCount) items", value: $controller.newsMaxCount, in: 5...50, step: 5) - Picker("Schedule", selection: $controller.newsSchedule) { - ForEach(NewsReaderSchedule.allCases, id: \.self) { schedule in - Text(schedule.displayName).tag(schedule) - } - } + private func arrange(proposal: ProposedViewSize, subviews: Subviews) -> (size: CGSize, frames: [CGRect]) { + let maxWidth = proposal.width ?? .infinity + var x: CGFloat = 0 + var y: CGFloat = 0 + var rowHeight: CGFloat = 0 + var frames: [CGRect] = [] + + for subview in subviews { + let size = subview.sizeThatFits(.unspecified) + if x + size.width > maxWidth, x > 0 { + x = 0 + y += rowHeight + spacing + rowHeight = 0 } + frames.append(CGRect(x: x, y: y, width: size.width, height: size.height)) + rowHeight = max(rowHeight, size.height) + x += size.width + spacing } - } - private func newsURLPaywallReason(_ raw: String) -> String? { - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return nil } - let normalized = trimmed.contains("://") ? trimmed : "https://\(trimmed)" - guard let url = URL(string: normalized) else { return nil } - return NewsPaywall.preflightRejection(url: url) + return (CGSize(width: maxWidth, height: y + rowHeight), frames) } } diff --git a/ui/ui/Views/ContentView.swift b/ui/ui/Views/ContentView.swift index 1380c2d..9254ea3 100644 --- a/ui/ui/Views/ContentView.swift +++ b/ui/ui/Views/ContentView.swift @@ -846,7 +846,10 @@ struct ContentView: View { await ContainerLifecycleSettingsService.shared.configure(repository: repository) await OrchestrationLimitsSettingsService.shared.configure(repository: repository) await PluginFactoryListStore.shared.configure(repository: repository) - await NewsReaderStore.shared.configure(repository: repository) + await NewsReaderStore.shared.configure( + repository: repository, + summarizerSettings: helperModelSettings + ) await AgentProfileStore.shared.configure(repository: repository) pluginCreationController.configure(repository: repository) } diff --git a/ui/uiTests/NewsReaderFlowTests.swift b/ui/uiTests/NewsReaderFlowTests.swift new file mode 100644 index 0000000..6ee5364 --- /dev/null +++ b/ui/uiTests/NewsReaderFlowTests.swift @@ -0,0 +1,58 @@ +import Foundation +import Testing +import Structure + +@Suite struct NewsReaderFlowTests { + @Test func skillDraftForTechNewsSummaryRequest() { + var draft = PluginSkillDraft() + PluginSkillDraftPlanner.applyGoal( + "Give me summaries of today's tech news", + to: &draft, + existingPluginIDs: [] + ) + draft.pluginName = "tech-news" + draft.newsSourceURLs = [ + NewsPresetSource.googleNews.source.url, + NewsPresetSource.wsj.source.url, + ] + draft.newsTopics = ["Tech"] + draft.examples = [ + PluginSkillDraft.Example( + userSays: "Give me summaries of tech news", + pluginDoes: "fetch headlines and summarize them with source links" + ), + ] + + #expect(draft.plannedKind == .newsDigest) + #expect(PluginSkillDraftPlanner.inferNewsMode(from: draft) == .summary) + #expect(draft.pluginName == "tech-news") + #expect(draft.newsSourceURLs.contains(NewsPresetSource.googleNews.source.url)) + #expect(draft.newsSourceURLs.contains(NewsPresetSource.wsj.source.url)) + } + + @Test func newsReaderWorkerRequestMatchesWizardSpec() throws { + let sources = [ + NewsPresetSource.googleNews.source, + NewsPresetSource.wsj.source, + ] + let spec = NewsReaderSpec( + name: "tech-news", + topics: ["Tech"], + sources: sources, + mode: .summary, + maxCount: 20, + schedule: .off + ) + let request = NewsReaderWorkerRequest( + mode: spec.mode, + sources: spec.sources, + topics: spec.topics, + maxCount: spec.maxCount, + contextHint: spec.name + ) + let json = try request.encodedJSON() + let object = try JSONSerialization.jsonObject(with: json) as? [String: Any] + #expect(object?["mode"] as? String == "summary") + #expect((object?["sources"] as? [[String: Any]])?.count == 2) + } +} diff --git a/workers/go/cmd/derrick-news-reader/main.go b/workers/go/cmd/derrick-news-reader/main.go new file mode 100644 index 0000000..a078c1d --- /dev/null +++ b/workers/go/cmd/derrick-news-reader/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "strconv" + + "github.com/jsoneaday/derrick/workers/internal/contract" + "github.com/jsoneaday/derrick/workers/internal/newsreader" +) + +func main() { + input, err := io.ReadAll(os.Stdin) + if err != nil { + writeResult(blockedResult(newsreader.ModeRSS, "News reader input must be a valid JSON object.")) + return + } + + var req newsreader.Request + if err := json.Unmarshal(input, &req); err != nil { + writeResult(blockedResult(newsreader.ModeRSS, "News reader input must be a valid JSON object.")) + return + } + + validated, err := newsreader.Validate(req) + if err != nil { + writeResult(blockedResult(req.Mode, err.Error())) + return + } + + result := newsreader.Run(context.Background(), validated, readProxy()) + writeResult(result) +} + +func readProxy() *newsreader.ProxyConfig { + host := os.Getenv("DERRICK_EGRESS_PROXY_HOST") + portStr := os.Getenv("DERRICK_EGRESS_PROXY_PORT") + token := os.Getenv("DERRICK_EGRESS_PROXY_TOKEN") + if host == "" && portStr == "" && token == "" { + return nil + } + port, _ := strconv.Atoi(portStr) + return &newsreader.ProxyConfig{Host: host, Port: port, Token: token} +} + +func blockedResult(mode newsreader.Mode, message string) newsreader.Result { + if mode == "" { + mode = newsreader.ModeRSS + } + return newsreader.Result{ + OK: false, + Mode: mode, + Articles: []newsreader.Article{}, + Diagnostics: []string{message}, + } +} + +func writeResult(result newsreader.Result) { + payload, err := encodedResult(result) + if err != nil { + writeEncodedResult(blockedResult(result.Mode, "News reader output violated worker contract.")) + return + } + _, _ = os.Stdout.Write(payload) +} + +func encodedResult(result newsreader.Result) ([]byte, error) { + if result.Articles == nil { + result.Articles = []newsreader.Article{} + } + if result.Diagnostics == nil { + result.Diagnostics = []string{} + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(result); err != nil { + return nil, err + } + data := bytes.TrimSpace(buf.Bytes()) + if err := contract.ValidateNewsReaderResultJSON(data); err != nil { + return nil, err + } + return append(data, '\n'), nil +} + +func writeEncodedResult(result newsreader.Result) { + payload, err := encodedResult(result) + if err != nil { + fallback := blockedResult(newsreader.ModeRSS, "News reader failed to produce schema-compliant output.") + payload, _ = encodedResult(fallback) + } + _, _ = os.Stdout.Write(payload) +} diff --git a/workers/go/internal/contract/contract.go b/workers/go/internal/contract/contract.go index ba341c0..e36ecc3 100644 --- a/workers/go/internal/contract/contract.go +++ b/workers/go/internal/contract/contract.go @@ -34,6 +34,11 @@ func ValidateFileExtractorResultJSON(data []byte) error { return validate("file-extractor-result.schema.json", data) } +// ValidateNewsReaderResultJSON checks news reader stdout against news-reader-result.schema.json. +func ValidateNewsReaderResultJSON(data []byte) error { + return validate("news-reader-result.schema.json", data) +} + func validate(schemaName string, data []byte) error { compiler := jsonschema.NewCompiler() if err := loadSchemas(compiler); err != nil { diff --git a/workers/go/internal/contract/schemas/news-reader-result.schema.json b/workers/go/internal/contract/schemas/news-reader-result.schema.json new file mode 100644 index 0000000..6632851 --- /dev/null +++ b/workers/go/internal/contract/schemas/news-reader-result.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://derrick.local/schemas/news-reader-result.json", + "title": "News reader worker stdout", + "description": "JSON object written to stdout by derrick-news-reader.", + "$ref": "worker-product.schema.json#/$defs/news_reader_result" +} diff --git a/workers/go/internal/contract/schemas/worker-product.schema.json b/workers/go/internal/contract/schemas/worker-product.schema.json index 874f6bf..611ef5f 100644 --- a/workers/go/internal/contract/schemas/worker-product.schema.json +++ b/workers/go/internal/contract/schemas/worker-product.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://derrick.local/schemas/worker-product.json", "title": "Derrick trusted worker product contracts", - "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler, file extractor, and news reader). Swift host and Go workers must match these schemas.", "$defs": { "string_list": { "type": "array", @@ -92,6 +92,35 @@ }, "diagnostics": { "$ref": "#/$defs/string_list" } } + }, + "news_reader_mode": { + "type": "string", + "enum": ["rss", "list", "summary"] + }, + "news_reader_article": { + "type": "object", + "required": ["title", "url"], + "additionalProperties": false, + "properties": { + "title": { "type": "string" }, + "url": { "type": "string" }, + "detail": { "type": "string" }, + "published_at": { "type": "string" } + } + }, + "news_reader_result": { + "type": "object", + "required": ["ok", "mode", "articles", "diagnostics"], + "additionalProperties": false, + "properties": { + "ok": { "type": "boolean" }, + "mode": { "$ref": "#/$defs/news_reader_mode" }, + "articles": { + "type": "array", + "items": { "$ref": "#/$defs/news_reader_article" } + }, + "diagnostics": { "$ref": "#/$defs/string_list" } + } } } } diff --git a/workers/go/internal/newsreader/engine.go b/workers/go/internal/newsreader/engine.go new file mode 100644 index 0000000..74e6081 --- /dev/null +++ b/workers/go/internal/newsreader/engine.go @@ -0,0 +1,34 @@ +package newsreader + +import ( + "context" +) + +func Run(ctx context.Context, req Request, proxy *ProxyConfig) Result { + client := newHTTPClient(proxy) + collected := make([]Article, 0, req.MaxCount) + diagnostics := make([]string, 0) + + for _, source := range req.Sources { + articles, notes := fetchSource(ctx, client, source, req.Mode, req.ContextHint) + diagnostics = append(diagnostics, notes...) + collected = append(collected, articles...) + } + + filtered := filterTopics(collected, req.Topics) + unique := uniqueArticles(filtered) + if len(unique) > req.MaxCount { + unique = unique[:req.MaxCount] + } + + ok := len(unique) > 0 + if !ok && len(diagnostics) == 0 { + diagnostics = append(diagnostics, "No articles were found for the configured sources.") + } + return Result{ + OK: ok, + Mode: req.Mode, + Articles: unique, + Diagnostics: diagnostics, + } +} diff --git a/workers/go/internal/newsreader/fetch.go b/workers/go/internal/newsreader/fetch.go new file mode 100644 index 0000000..904a82f --- /dev/null +++ b/workers/go/internal/newsreader/fetch.go @@ -0,0 +1,110 @@ +package newsreader + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +func fetchSource( + ctx context.Context, + client *http.Client, + source Source, + mode Mode, + contextHint string, +) ([]Article, []string) { + fetchURL := CanonicalFetchURL(source.URL, contextHint) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil) + if err != nil { + return nil, []string{fmt.Sprintf("%s: %v", source.Label, err)} + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; DerrickNewsReader/1.0)") + req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.8") + + resp, err := client.Do(req) + if err != nil { + return nil, []string{fmt.Sprintf("Could not read %s: %v", source.URL, err)} + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return nil, []string{fmt.Sprintf("Could not read %s: HTTP %d", source.URL, resp.StatusCode)} + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return nil, []string{fmt.Sprintf("Could not read %s: %v", source.URL, err)} + } + text := string(body) + if looksLikeFeed(text) { + articles := parseFeed(text, source.Label) + if len(articles) > 0 { + return articles, nil + } + } + if mode == ModeRSS { + articles := parseFeed(text, source.Label) + if len(articles) > 0 { + return articles, nil + } + return nil, []string{fmt.Sprintf("No articles were found in %s", source.URL)} + } + if mode == ModeList || mode == ModeSummary { + articles := parseHTMLArticles(text, fetchURL, source.Label) + if len(articles) > 0 { + return articles, nil + } + return nil, []string{fmt.Sprintf("No articles were found on %s", source.URL)} + } + return nil, []string{fmt.Sprintf("No articles were found in %s", source.URL)} +} + +func newHTTPClient(proxy *ProxyConfig) *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + return &http.Client{ + Timeout: time.Duration(DefaultTimeoutSeconds) * time.Second, + Transport: transport, + } +} + +func filterTopics(articles []Article, topics []string) []Article { + needles := make([]string, 0, len(topics)) + for _, topic := range topics { + trimmed := strings.ToLower(strings.TrimSpace(topic)) + if trimmed != "" { + needles = append(needles, trimmed) + } + } + if len(needles) == 0 { + return articles + } + matched := make([]Article, 0, len(articles)) + for _, article := range articles { + hay := strings.ToLower(article.Title + " " + article.Detail) + for _, needle := range needles { + if strings.Contains(hay, needle) { + matched = append(matched, article) + break + } + } + } + if len(matched) == 0 { + return articles + } + return matched +} + +func uniqueArticles(articles []Article) []Article { + seen := map[string]bool{} + out := make([]Article, 0, len(articles)) + for _, article := range articles { + if seen[article.URL] { + continue + } + seen[article.URL] = true + out = append(out, article) + } + return out +} diff --git a/workers/go/internal/newsreader/list.go b/workers/go/internal/newsreader/list.go new file mode 100644 index 0000000..9d3c373 --- /dev/null +++ b/workers/go/internal/newsreader/list.go @@ -0,0 +1,88 @@ +package newsreader + +import ( + "net/url" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +func parseHTMLArticles(body string, pageURL string, sourceLabel string) []Article { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + return nil + } + articles := make([]Article, 0, 32) + seen := map[string]bool{} + + add := func(title string, href string, detail string) { + title = SanitizeDetail(title, 240) + href = strings.TrimSpace(href) + if title == "" || href == "" || seen[href] { + return + } + seen[href] = true + articles = append(articles, Article{ + Title: title, + URL: href, + Detail: SanitizeDetail(detail, 280), + }) + } + + doc.Find("article a[href]").Each(func(_ int, sel *goquery.Selection) { + href := resolveHref(pageURL, sel.AttrOr("href", "")) + if href == "" { + return + } + title := strings.TrimSpace(sel.Text()) + if title == "" { + title = strings.TrimSpace(sel.Closest("article").Find("h1,h2,h3").First().Text()) + } + add(title, href, sel.Closest("article").Text()) + }) + + doc.Find("h1 a[href], h2 a[href], h3 a[href]").Each(func(_ int, sel *goquery.Selection) { + href := resolveHref(pageURL, sel.AttrOr("href", "")) + title := strings.TrimSpace(sel.Text()) + add(title, href, sel.Parent().Text()) + }) + + if len(articles) == 0 { + doc.Find("a[href]").Each(func(_ int, sel *goquery.Selection) { + href := resolveHref(pageURL, sel.AttrOr("href", "")) + title := strings.TrimSpace(sel.Text()) + if len(title) < 12 || len(title) > 200 { + return + } + add(title, href, "") + }) + } + + if len(articles) == 0 { + title := strings.TrimSpace(doc.Find("title").First().Text()) + if title != "" { + add(title, pageURL, doc.Find("meta[name=description]").AttrOr("content", "")) + } + } + return articles +} + +func resolveHref(pageURL string, href string) string { + href = strings.TrimSpace(href) + if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") { + return "" + } + base, err := url.Parse(pageURL) + if err != nil { + return href + } + ref, err := url.Parse(href) + if err != nil { + return "" + } + resolved := base.ResolveReference(ref) + if resolved.Scheme != "http" && resolved.Scheme != "https" { + return "" + } + return resolved.String() +} diff --git a/workers/go/internal/newsreader/newsreader_test.go b/workers/go/internal/newsreader/newsreader_test.go new file mode 100644 index 0000000..1e5e024 --- /dev/null +++ b/workers/go/internal/newsreader/newsreader_test.go @@ -0,0 +1,30 @@ +package newsreader + +import "testing" + +func TestValidateRequiresSources(t *testing.T) { + _, err := Validate(Request{Mode: ModeRSS, Sources: []Source{}}) + if err == nil { + t.Fatal("expected validation error") + } +} + +func TestParseFeedReadsRSSItem(t *testing.T) { + rss := ` + Hellohttps://example.com/aBody + ` + articles := parseFeed(rss, "Example") + if len(articles) != 1 { + t.Fatalf("expected 1 article, got %d", len(articles)) + } + if articles[0].Title != "Hello" { + t.Fatalf("unexpected title %q", articles[0].Title) + } +} + +func TestCanonicalFetchURLRewritesGoogleNewsHomepage(t *testing.T) { + got := CanonicalFetchURL("https://news.google.com/", "tech news") + if got == "https://news.google.com/" { + t.Fatalf("expected RSS rewrite, got %q", got) + } +} diff --git a/workers/go/internal/newsreader/rss.go b/workers/go/internal/newsreader/rss.go new file mode 100644 index 0000000..3d8372e --- /dev/null +++ b/workers/go/internal/newsreader/rss.go @@ -0,0 +1,207 @@ +package newsreader + +import ( + "encoding/xml" + "regexp" + "strings" +) + +type rssFeed struct { + Channel rssChannel `xml:"channel"` +} + +type rssChannel struct { + Items []rssItem `xml:"item"` +} + +type rssItem struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + PubDate string `xml:"pubDate"` +} + +type atomFeed struct { + Entries []atomEntry `xml:"entry"` +} + +type atomEntry struct { + Title string `xml:"title"` + Link atomLink `xml:"link"` + Summary string `xml:"summary"` + Updated string `xml:"updated"` + Content atomContent `xml:"content"` +} + +type atomLink struct { + Href string `xml:"href,attr"` +} + +type atomContent struct { + Value string `xml:",chardata"` +} + +func looksLikeFeed(body string) bool { + lower := strings.ToLower(body) + return strings.Contains(lower, "") + blocks = append(blocks, splitBlocks(body, "")...) + articles := make([]Article, 0, len(blocks)) + for _, block := range blocks { + title := firstTag(block, "title") + link := firstTag(block, "link") + if link == "" { + link = firstAttr(block, "link", "href") + } + if link == "" { + link = firstTag(block, "guid") + } + description := firstTag(block, "description") + if description == "" { + description = firstTag(block, "summary") + } + if description == "" { + description = firstTag(block, "content") + } + cleanTitle := SanitizeDetail(title, 240) + cleanLink := strings.TrimSpace(StripTags(link)) + if cleanTitle == "" || cleanLink == "" { + continue + } + resolved := PreferredArticleURL(cleanLink, description) + detail := SanitizeDetail(description, 280) + articles = append(articles, Article{ + Title: cleanTitle, + URL: resolved, + Detail: detail, + PublishedAt: strings.TrimSpace(firstTag(block, "pubDate")), + }) + } + return articles +} + +func splitBlocks(body string, startToken string, endToken string) []string { + lower := strings.ToLower(body) + startLower := strings.ToLower(startToken) + endLower := strings.ToLower(endToken) + var blocks []string + idx := 0 + for { + start := strings.Index(lower[idx:], startLower) + if start < 0 { + break + } + start += idx + end := strings.Index(lower[start:], endLower) + if end < 0 { + break + } + end += start + len(endToken) + blocks = append(blocks, body[start:end]) + idx = end + } + return blocks +} + +func firstTag(block string, name string) string { + open := "<" + strings.ToLower(name) + lower := strings.ToLower(block) + start := strings.Index(lower, open) + if start < 0 { + return "" + } + closeTag := strings.Index(block[start:], ">") + if closeTag < 0 { + return "" + } + contentStart := start + closeTag + 1 + endToken := "" + end := strings.Index(strings.ToLower(block[contentStart:]), endToken) + if end < 0 { + return "" + } + return block[contentStart : contentStart+end] +} + +func firstAttr(block string, tagName string, attr string) string { + open := "<" + strings.ToLower(tagName) + lower := strings.ToLower(block) + start := strings.Index(lower, open) + if start < 0 { + return "" + } + closeTag := strings.Index(block[start:], ">") + if closeTag < 0 { + return "" + } + tag := block[start : start+closeTag+1] + pattern := regexp.MustCompile(`(?i)` + attr + `\s*=\s*"([^"]+)"`) + match := pattern.FindStringSubmatch(tag) + if len(match) < 2 { + return "" + } + return match[1] +} diff --git a/workers/go/internal/newsreader/text.go b/workers/go/internal/newsreader/text.go new file mode 100644 index 0000000..9f8e952 --- /dev/null +++ b/workers/go/internal/newsreader/text.go @@ -0,0 +1,76 @@ +package newsreader + +import ( + "net/url" + "regexp" + "strings" +) + +var tagPattern = regexp.MustCompile(`<[^>]+>`) +var hrefPattern = regexp.MustCompile(`(?i)href\s*=\s*"([^"]+)"`) +var whitespacePattern = regexp.MustCompile(`\s+`) + +func StripTags(raw string) string { + value := tagPattern.ReplaceAllString(raw, " ") + value = strings.ReplaceAll(value, "", "") + return DecodeEntities(value) +} + +func DecodeEntities(raw string) string { + replacements := []struct { + from string + to string + }{ + {"&", "&"}, + {"<", "<"}, + {">", ">"}, + {""", "\""}, + {"'", "'"}, + {"'", "'"}, + {" ", " "}, + } + value := raw + for _, pair := range replacements { + value = strings.ReplaceAll(value, pair.from, pair.to) + } + return value +} + +func CollapseWhitespace(raw string) string { + return strings.TrimSpace(whitespacePattern.ReplaceAllString(raw, " ")) +} + +func SanitizeDetail(raw string, maxLen int) string { + cleaned := CollapseWhitespace(StripTags(raw)) + if cleaned == "" { + return "" + } + if len(cleaned) <= maxLen { + return cleaned + } + return strings.TrimSpace(cleaned[:maxLen]) + "…" +} + +func PreferredArticleURL(link string, htmlSnippet string) string { + link = strings.TrimSpace(link) + for _, match := range hrefPattern.FindAllStringSubmatch(htmlSnippet, -1) { + if len(match) < 2 { + continue + } + href := strings.TrimSpace(match[1]) + if !strings.HasPrefix(href, "http") { + continue + } + parsed, err := url.Parse(href) + if err != nil { + continue + } + host := strings.ToLower(parsed.Hostname()) + if strings.Contains(host, "google.com") || strings.Contains(host, "googleusercontent.com") { + continue + } + return href + } + return link +} diff --git a/workers/go/internal/newsreader/types.go b/workers/go/internal/newsreader/types.go new file mode 100644 index 0000000..3738664 --- /dev/null +++ b/workers/go/internal/newsreader/types.go @@ -0,0 +1,49 @@ +package newsreader + +const ( + DefaultMaxCount = 20 + MaximumMaxCount = 50 + DefaultTimeoutSeconds = 120 + MaximumTimeoutSeconds = 300 +) + +type Mode string + +const ( + ModeRSS Mode = "rss" + ModeList Mode = "list" + ModeSummary Mode = "summary" +) + +type Source struct { + Label string `json:"label"` + URL string `json:"url"` +} + +type Request struct { + Mode Mode `json:"mode"` + Sources []Source `json:"sources"` + Topics []string `json:"topics"` + MaxCount int `json:"maxCount"` + ContextHint string `json:"contextHint,omitempty"` +} + +type Article struct { + Title string `json:"title"` + URL string `json:"url"` + Detail string `json:"detail,omitempty"` + PublishedAt string `json:"published_at,omitempty"` +} + +type Result struct { + OK bool `json:"ok"` + Mode Mode `json:"mode"` + Articles []Article `json:"articles"` + Diagnostics []string `json:"diagnostics"` +} + +type ProxyConfig struct { + Host string + Port int + Token string +} diff --git a/workers/go/internal/newsreader/urls.go b/workers/go/internal/newsreader/urls.go new file mode 100644 index 0000000..871c1b6 --- /dev/null +++ b/workers/go/internal/newsreader/urls.go @@ -0,0 +1,57 @@ +package newsreader + +import ( + "net/url" + "strings" +) + +func CanonicalFetchURL(raw string, contextHint string) string { + parsed, err := url.Parse(raw) + if err != nil { + return raw + } + host := strings.ToLower(parsed.Hostname()) + if !strings.Contains(host, "news.google.com") { + return raw + } + path := strings.ToLower(parsed.Path) + if strings.Contains(path, "/rss") || strings.HasSuffix(path, ".xml") { + return raw + } + if strings.Contains(path, "/topics/") { + if section := googleNewsSection(contextHint); section != "" { + return "https://news.google.com/rss/headlines/section/topic/" + section + "?hl=en-US&gl=US&ceid=US:en" + } + return "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" + } + if parsed.RawQuery == "" { + return "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" + } + return raw +} + +func googleNewsSection(hint string) string { + lower := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(hint, "-", " "), "_", " ")) + if strings.Contains(lower, "tech") { + return "TECHNOLOGY" + } + if strings.Contains(lower, "business") || strings.Contains(lower, "finance") || strings.Contains(lower, "market") { + return "BUSINESS" + } + if strings.Contains(lower, "science") { + return "SCIENCE" + } + if strings.Contains(lower, "sport") { + return "SPORTS" + } + if strings.Contains(lower, "health") { + return "HEALTH" + } + if strings.Contains(lower, "entertainment") { + return "ENTERTAINMENT" + } + if strings.Contains(lower, "world") { + return "WORLD" + } + return "" +} diff --git a/workers/go/internal/newsreader/validate.go b/workers/go/internal/newsreader/validate.go new file mode 100644 index 0000000..a9ec819 --- /dev/null +++ b/workers/go/internal/newsreader/validate.go @@ -0,0 +1,38 @@ +package newsreader + +import ( + "fmt" + "net/url" + "strings" +) + +func Validate(req Request) (Request, error) { + out := req + if out.Mode != ModeRSS && out.Mode != ModeList && out.Mode != ModeSummary { + return out, fmt.Errorf("mode must be rss, list, or summary") + } + if len(out.Sources) == 0 { + return out, fmt.Errorf("add at least one source URL") + } + if out.MaxCount <= 0 { + out.MaxCount = DefaultMaxCount + } + if out.MaxCount > MaximumMaxCount { + out.MaxCount = MaximumMaxCount + } + for i, source := range out.Sources { + trimmed := strings.TrimSpace(source.URL) + if trimmed == "" { + return out, fmt.Errorf("source %d is missing a URL", i+1) + } + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" { + return out, fmt.Errorf("source %d is not a usable web address", i+1) + } + out.Sources[i].URL = trimmed + if strings.TrimSpace(out.Sources[i].Label) == "" { + out.Sources[i].Label = parsed.Host + } + } + return out, nil +} From fae578116196be8f40ff8cca18e8d31866c3a805 Mon Sep 17 00:00:00 2001 From: David Choi Date: Thu, 10 Sep 2026 23:56:05 -0400 Subject: [PATCH 07/12] fix news view --- .../AppLayerServices/News/NewsSourceURL.swift | 4 +++ .../StructureTests/NewsReaderTests.swift | 6 ++++ ui/ui/News/NewsReaderSummarizer.swift | 4 ++- ui/ui/News/NewsWorkspaceView.swift | 17 +++++---- ui/ui/Views/MarkdownView.swift | 36 +++++++++++++++++++ workers/go/internal/newsreader/fetch.go | 27 +++++++++++--- .../go/internal/newsreader/newsreader_test.go | 26 ++++++++++++++ workers/go/internal/newsreader/urls.go | 5 +++ 8 files changed, 112 insertions(+), 13 deletions(-) diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift index 7317b3e..454b7d6 100644 --- a/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift +++ b/packages/Structure/Sources/AppLayerServices/News/NewsSourceURL.swift @@ -9,6 +9,10 @@ public enum NewsSourceURL { guard isGoogleNewsHost(host) else { return url } let path = url.path.lowercased() if path.contains("/rss") || path.hasSuffix(".xml") { + if !path.contains("/headlines/section/topic/"), + let section = googleNewsSection(from: contextHint) { + return googleNewsSectionRSS(section: section) + } return url } if path.contains("/topics/") { diff --git a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift index 1af3324..7077ce0 100644 --- a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift +++ b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift @@ -51,6 +51,12 @@ import Structure } } + @Test func canonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint() { + let url = URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + let canonical = NewsSourceURL.canonicalFetchURL(url, contextHint: "tech-news Tech") + #expect(canonical.path.contains("/headlines/section/topic/TECHNOLOGY")) + } + @Test func workerRequestEncodesModeAndSources() throws { let request = NewsReaderWorkerRequest( mode: .rss, diff --git a/ui/ui/News/NewsReaderSummarizer.swift b/ui/ui/News/NewsReaderSummarizer.swift index 74b441c..e0b0404 100644 --- a/ui/ui/News/NewsReaderSummarizer.swift +++ b/ui/ui/News/NewsReaderSummarizer.swift @@ -16,7 +16,9 @@ struct NewsReaderSummarizer: NewsSummaryGenerating { prompt, system: """ You summarize news for the user. Write clear prose in plain English. - Include markdown links to the original articles using the URLs provided. + Use bullet points. Include markdown links to the original articles using the URLs provided. + Do not add a title or markdown heading for the list name. + Cover every article provided when possible. Do not invent stories or URLs. """, temperature: 0.2 diff --git a/ui/ui/News/NewsWorkspaceView.swift b/ui/ui/News/NewsWorkspaceView.swift index 43a54e1..8228570 100644 --- a/ui/ui/News/NewsWorkspaceView.swift +++ b/ui/ui/News/NewsWorkspaceView.swift @@ -82,11 +82,12 @@ struct NewsWorkspaceView: View { let summary = store.selectedReader?.summaryText, !summary.isEmpty { VStack(alignment: .leading, spacing: 8) { - Text("Summary") + Text("AI summary") .font(.subheadline.weight(.semibold)) - Text(LocalizedStringKey(summary)) - .font(.body) - .textSelection(.enabled) + Text("A digest of the articles below.") + .font(.caption) + .foregroundStyle(.secondary) + MarkdownResponseView(text: summary, allowsCSVExport: false) } .padding(14) .frame(maxWidth: .infinity, alignment: .leading) @@ -97,10 +98,12 @@ struct NewsWorkspaceView: View { ForEach(store.items) { item in articleCard(item) } - } else { + } else if !store.items.isEmpty { VStack(alignment: .leading, spacing: 8) { - Text("Sources") - .font(.caption.weight(.semibold)) + Text("Articles (\(store.items.count))") + .font(.subheadline.weight(.semibold)) + Text("Stories fetched from your feeds. The header feed count is how many RSS sources are configured, not how many articles appear here.") + .font(.caption) .foregroundStyle(.secondary) ForEach(store.items) { item in HStack(alignment: .top, spacing: 8) { diff --git a/ui/ui/Views/MarkdownView.swift b/ui/ui/Views/MarkdownView.swift index d39b067..c5b9f78 100644 --- a/ui/ui/Views/MarkdownView.swift +++ b/ui/ui/Views/MarkdownView.swift @@ -2,6 +2,7 @@ import AppKit import SwiftUI enum MarkdownBlock: Identifiable { + case heading(level: Int, text: String) case paragraph(String) case bullet(String) case numbered(number: Int, text: String) @@ -10,6 +11,8 @@ enum MarkdownBlock: Identifiable { var id: String { switch self { + case .heading(let level, let text): + return "h\(level)-\(text.hashValue)" case .paragraph(let text): return "p-\(text.hashValue)" case .bullet(let text): @@ -66,6 +69,22 @@ enum MarkdownBlock: Identifiable { continue } + if trimmed.hasPrefix("#") { + var level = 0 + var index = trimmed.startIndex + while index < trimmed.endIndex, trimmed[index] == "#", level < 6 { + level += 1 + index = trimmed.index(after: index) + } + if level > 0, index < trimmed.endIndex, trimmed[index] == " " { + flushParagraph() + let headingText = String(trimmed[trimmed.index(after: index)...]) + .trimmingCharacters(in: .whitespaces) + blocks.append(.heading(level: level, text: headingText)) + continue + } + } + // Check bullet if trimmed.hasPrefix("- ") || trimmed.hasPrefix("* ") { flushParagraph() @@ -290,6 +309,13 @@ struct MarkdownResponseView: View { @ViewBuilder private func blockView(for block: MarkdownBlock) -> some View { switch block { + case .heading(let level, let text): + Text((try? AttributedString(markdown: text)) ?? AttributedString(text)) + .font(headingFont(level: level)) + .fontWeight(.semibold) + .padding(.horizontal, 2) + .frame(maxWidth: .infinity, alignment: .leading) + .textSelection(.enabled) case .paragraph(let text): Text((try? AttributedString(markdown: text)) ?? AttributedString(text)) .lineSpacing(2) @@ -352,6 +378,16 @@ struct MarkdownResponseView: View { } } + private func headingFont(level: Int) -> Font { + switch level { + case 1: return .title + case 2: return .title2 + case 3: return .title3 + case 4: return .headline + default: return .subheadline + } + } + @ViewBuilder private func csvTableView(table: CSVTable, source: String) -> some View { VStack(alignment: .leading, spacing: 10) { diff --git a/workers/go/internal/newsreader/fetch.go b/workers/go/internal/newsreader/fetch.go index 904a82f..7c2b047 100644 --- a/workers/go/internal/newsreader/fetch.go +++ b/workers/go/internal/newsreader/fetch.go @@ -81,21 +81,38 @@ func filterTopics(articles []Article, topics []string) []Article { return articles } matched := make([]Article, 0, len(articles)) + unmatched := make([]Article, 0, len(articles)) for _, article := range articles { hay := strings.ToLower(article.Title + " " + article.Detail) - for _, needle := range needles { - if strings.Contains(hay, needle) { - matched = append(matched, article) - break - } + if topicMatches(hay, needles) { + matched = append(matched, article) + } else { + unmatched = append(unmatched, article) } } if len(matched) == 0 { return articles } + // Prefer topic matches, but keep other articles when filtering would drop too many + // (for example WSJ headlines that do not literally contain "tech"). + if len(matched) < 3 && len(unmatched) > 0 { + out := make([]Article, 0, len(articles)) + out = append(out, matched...) + out = append(out, unmatched...) + return out + } return matched } +func topicMatches(hay string, needles []string) bool { + for _, needle := range needles { + if strings.Contains(hay, needle) { + return true + } + } + return false +} + func uniqueArticles(articles []Article) []Article { seen := map[string]bool{} out := make([]Article, 0, len(articles)) diff --git a/workers/go/internal/newsreader/newsreader_test.go b/workers/go/internal/newsreader/newsreader_test.go index 1e5e024..3bee2d7 100644 --- a/workers/go/internal/newsreader/newsreader_test.go +++ b/workers/go/internal/newsreader/newsreader_test.go @@ -28,3 +28,29 @@ func TestCanonicalFetchURLRewritesGoogleNewsHomepage(t *testing.T) { t.Fatalf("expected RSS rewrite, got %q", got) } } + +func TestCanonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint(t *testing.T) { + got := CanonicalFetchURL( + "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en", + "tech-news Tech", + ) + want := "https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en" + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestFilterTopicsKeepsUnmatchedArticlesWhenFewMatches(t *testing.T) { + articles := []Article{ + {Title: "TechCrunch story", URL: "https://example.com/a"}, + {Title: "Wall Street earnings", URL: "https://example.com/b"}, + {Title: "Another market report", URL: "https://example.com/c"}, + } + got := filterTopics(articles, []string{"Tech"}) + if len(got) != 3 { + t.Fatalf("expected all articles when only one matches, got %d", len(got)) + } + if got[0].Title != "TechCrunch story" { + t.Fatalf("expected matched article first, got %q", got[0].Title) + } +} diff --git a/workers/go/internal/newsreader/urls.go b/workers/go/internal/newsreader/urls.go index 871c1b6..599fe7d 100644 --- a/workers/go/internal/newsreader/urls.go +++ b/workers/go/internal/newsreader/urls.go @@ -16,6 +16,11 @@ func CanonicalFetchURL(raw string, contextHint string) string { } path := strings.ToLower(parsed.Path) if strings.Contains(path, "/rss") || strings.HasSuffix(path, ".xml") { + if !strings.Contains(path, "/headlines/section/topic/") { + if section := googleNewsSection(contextHint); section != "" { + return "https://news.google.com/rss/headlines/section/topic/" + section + "?hl=en-US&gl=US&ceid=US:en" + } + } return raw } if strings.Contains(path, "/topics/") { From d487084bb8e54834fb2d3c335db26c54f7db8858 Mon Sep 17 00:00:00 2001 From: David Choi Date: Fri, 11 Sep 2026 21:48:05 -0400 Subject: [PATCH 08/12] Fix NewsReaderSpec argument order in DB tests so CI compiles. Co-authored-by: Cursor --- .../Tests/DBRepositoryTests/DBNewsReaderTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift index 5578700..c87e195 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift @@ -12,9 +12,9 @@ final class DBNewsReaderTests: XCTestCase { topics: ["Financial", "rates"], sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/world/rss.xml")], mode: .summary, - summaryText: "Markets moved higher.", maxCount: 10, - schedule: .daily + schedule: .daily, + summaryText: "Markets moved higher." ) try await repository.upsertNewsReader(spec) let listed = try await repository.listNewsReaders() From b34c262399baa57eb70c300a3ef10e21bf7b6858 Mon Sep 17 00:00:00 2001 From: David Choi Date: Fri, 11 Sep 2026 22:15:00 -0400 Subject: [PATCH 09/12] remove ci issue; old news reader --- docker/worker/Dockerfile | 4 +- .../DBRepository/DBRepositoryNews.swift | 160 ------------ .../Sources/DBRepository/DatabaseSchema.swift | 4 +- .../0009_drop_news_readers.down.sql | 30 +++ .../Migrations/0009_drop_news_readers.up.sql | 2 + .../DBRepositoryTests/DBNewsReaderTests.swift | 62 ----- .../DBRepositoryTests/DBRepositoryTests.swift | 2 +- .../DockerRunRequestValidator.swift | 1 - .../DockerRunnerXPCTests.swift | 1 - .../MCPServer/NewsReaderDockerExecutor.swift | 115 --------- .../NewsReaderDockerInputPreparer.swift | 43 ---- .../Tests/MCPServerTests/MCPServerTests.swift | 43 +--- .../InProcessServiceBridges.swift | 3 - .../MCPService/MCPServiceXPC.swift | 10 - .../MCPService/MCPToolCallTimeouts.swift | 3 - .../MCPService/PluginFactoryCreateInput.swift | 6 - .../News/NewsReaderModels.swift | 231 ------------------ .../News/NewsReaderRefresh.swift | 108 -------- .../News/NewsReaderWorkerTypes.swift | 72 ------ .../News/NewsWorkerBridge.swift | 30 --- .../Plugin/PluginSkillDraft.swift | 99 +------- .../Sources/Contract/GuestContract.swift | 1 - .../schemas/news-reader-result.schema.json | 7 - .../schemas/worker-product.schema.json | 31 +-- .../DerrickDockerRuntimeIdentity.swift | 5 +- .../DockerRunnerXPC/DockerWorkerRuntime.swift | 4 +- .../AppLayerServicesWireTests.swift | 3 - .../StructureTests/NewsPaywallTests.swift | 29 +++ .../StructureTests/NewsReaderTests.swift | 74 ------ .../PluginSkillDraftTests.swift | 38 +-- ui/JobKeepAlive/DaemonModuleBootstrap.swift | 3 - ui/JobKeepAlive/DaemonUnifiedXPC.swift | 4 - ui/MCPService/MCPServiceExportedObject.swift | 17 -- ui/MCPService/MCPServiceToolHost.swift | 42 ---- .../Services/MCPServiceClient.swift | 19 -- ui/ui/Messaging/AppWorkspace.swift | 1 - ui/ui/News/MCPServiceNewsWorker.swift | 23 -- ui/ui/News/NewsReaderStore.swift | 148 ----------- ui/ui/News/NewsReaderSummarizer.swift | 67 ----- ui/ui/News/NewsWorkspaceView.swift | 171 ------------- ui/ui/Plugins/PluginCreationController.swift | 116 +-------- ui/ui/Plugins/PluginsWorkspaceView.swift | 179 +------------- ui/ui/Views/ContentView.swift | 17 +- ui/ui/Views/SidebarView.swift | 55 ----- ui/uiTests/NewsReaderFlowTests.swift | 58 ----- workers/go/cmd/derrick-news-reader/main.go | 97 -------- workers/go/internal/contract/contract.go | 5 - .../schemas/news-reader-result.schema.json | 7 - .../schemas/worker-product.schema.json | 31 +-- workers/go/internal/newsreader/engine.go | 34 --- workers/go/internal/newsreader/fetch.go | 127 ---------- workers/go/internal/newsreader/list.go | 88 ------- .../go/internal/newsreader/newsreader_test.go | 56 ----- workers/go/internal/newsreader/rss.go | 207 ---------------- workers/go/internal/newsreader/text.go | 76 ------ workers/go/internal/newsreader/types.go | 49 ---- workers/go/internal/newsreader/urls.go | 62 ----- workers/go/internal/newsreader/validate.go | 38 --- 58 files changed, 89 insertions(+), 2929 deletions(-) delete mode 100644 packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift create mode 100644 packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql create mode 100644 packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql delete mode 100644 packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift delete mode 100644 packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift delete mode 100644 packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift delete mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift delete mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift delete mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift delete mode 100644 packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift delete mode 100644 packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json create mode 100644 packages/Structure/Tests/StructureTests/NewsPaywallTests.swift delete mode 100644 packages/Structure/Tests/StructureTests/NewsReaderTests.swift delete mode 100644 ui/ui/News/MCPServiceNewsWorker.swift delete mode 100644 ui/ui/News/NewsReaderStore.swift delete mode 100644 ui/ui/News/NewsReaderSummarizer.swift delete mode 100644 ui/ui/News/NewsWorkspaceView.swift delete mode 100644 ui/uiTests/NewsReaderFlowTests.swift delete mode 100644 workers/go/cmd/derrick-news-reader/main.go delete mode 100644 workers/go/internal/contract/schemas/news-reader-result.schema.json delete mode 100644 workers/go/internal/newsreader/engine.go delete mode 100644 workers/go/internal/newsreader/fetch.go delete mode 100644 workers/go/internal/newsreader/list.go delete mode 100644 workers/go/internal/newsreader/newsreader_test.go delete mode 100644 workers/go/internal/newsreader/rss.go delete mode 100644 workers/go/internal/newsreader/text.go delete mode 100644 workers/go/internal/newsreader/types.go delete mode 100644 workers/go/internal/newsreader/urls.go delete mode 100644 workers/go/internal/newsreader/validate.go diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 7098c07..0e11fc1 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -9,11 +9,10 @@ COPY workers/go ./ RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-web-crawler ./cmd/derrick-crawler RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-file-extractor ./cmd/derrick-file-extractor -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/derrick-news-reader ./cmd/derrick-news-reader FROM debian:bookworm-slim -LABEL derrick.worker.binaries="crawler,extractor,news-reader" +LABEL derrick.worker.binaries="crawler,extractor" RUN apt-get update \ && apt-get install -y --no-install-recommends poppler-utils ca-certificates \ @@ -23,7 +22,6 @@ RUN apt-get update \ COPY --from=build /usr/local/go /usr/local/go COPY --from=build /out/derrick-web-crawler /usr/local/bin/derrick-web-crawler COPY --from=build /out/derrick-file-extractor /usr/local/bin/derrick-file-extractor -COPY --from=build /out/derrick-news-reader /usr/local/bin/derrick-news-reader RUN mkdir -p /data/in /data/out && chown -R worker:worker /data ENV PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift b/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift deleted file mode 100644 index 65b5595..0000000 --- a/packages/DBRepository/Sources/DBRepository/DBRepositoryNews.swift +++ /dev/null @@ -1,160 +0,0 @@ -import Foundation -import SQLite3 -import Structure - -public extension DBRepository { - func upsertNewsReader(_ spec: NewsReaderSpec) throws { - let topicsData = try JSONEncoder().encode(spec.topics) - let sourcesData = try JSONEncoder().encode(spec.sources) - let topics = String(data: topicsData, encoding: .utf8) ?? "[]" - let sources = String(data: sourcesData, encoding: .utf8) ?? "[]" - try withDatabaseHandle { handle in - try Self.execute(""" - INSERT INTO news_readers ( - id, name, topics_json, sources_json, mode, max_count, schedule, - summary_text, last_error, last_fetched_at, created_at, updated_at - ) VALUES ( - \(quoted(spec.id)), - \(quoted(spec.name)), - \(quoted(topics)), - \(quoted(sources)), - \(quoted(spec.mode.rawValue)), - \(spec.maxCount), - \(quoted(spec.schedule.rawValue)), - \(sqlValue(spec.summaryText)), - \(sqlValue(spec.lastError)), - \(sqlValue(spec.lastFetchedAt.map { Self.iso8601Formatter().string(from: $0) })), - \(quoted(Self.iso8601Formatter().string(from: spec.createdAt))), - \(quoted(Self.iso8601Formatter().string(from: spec.updatedAt))) - ) - ON CONFLICT(id) DO UPDATE SET - name = excluded.name, - topics_json = excluded.topics_json, - sources_json = excluded.sources_json, - mode = excluded.mode, - max_count = excluded.max_count, - schedule = excluded.schedule, - summary_text = excluded.summary_text, - last_error = excluded.last_error, - last_fetched_at = excluded.last_fetched_at, - updated_at = excluded.updated_at; - """, on: handle) - } - } - - func listNewsReaders() throws -> [NewsReaderSpec] { - try withDatabaseHandle { handle in - let sql = """ - SELECT id, name, topics_json, sources_json, mode, max_count, schedule, - summary_text, last_error, last_fetched_at, created_at, updated_at - FROM news_readers - ORDER BY updated_at DESC; - """ - var statement: OpaquePointer? - guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else { - throw Self.sqliteError(handle: handle, fallback: "Failed to list news readers.") - } - defer { sqlite3_finalize(statement) } - var rows: [NewsReaderSpec] = [] - while sqlite3_step(statement) == SQLITE_ROW { - rows.append(try decodeNewsReader(statement: statement)) - } - return rows - } - } - - func deleteNewsReader(id: String) throws { - try withDatabaseHandle { handle in - try Self.execute("DELETE FROM news_readers WHERE id = \(quoted(id));", on: handle) - } - } - - func replaceNewsItems(readerID: String, items: [NewsItem]) throws { - try withDatabaseHandle { handle in - try Self.withImmediateTransaction(on: handle) { - try Self.execute("DELETE FROM news_items WHERE reader_id = \(quoted(readerID));", on: handle) - for item in items { - try Self.execute(""" - INSERT INTO news_items ( - id, reader_id, title, source_url, source_label, summary, published_at, fetched_at - ) VALUES ( - \(quoted(item.id)), - \(quoted(item.readerID)), - \(quoted(item.title)), - \(quoted(item.sourceURL)), - \(quoted(item.sourceLabel)), - \(sqlValue(item.summary)), - \(sqlValue(item.publishedAt.map { Self.iso8601Formatter().string(from: $0) })), - \(quoted(Self.iso8601Formatter().string(from: item.fetchedAt))) - ); - """, on: handle) - } - } - } - } - - func listNewsItems(readerID: String) throws -> [NewsItem] { - try withDatabaseHandle { handle in - let sql = """ - SELECT id, reader_id, title, source_url, source_label, summary, published_at, fetched_at - FROM news_items - WHERE reader_id = \(quoted(readerID)) - ORDER BY COALESCE(published_at, fetched_at) DESC; - """ - var statement: OpaquePointer? - guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK, let statement else { - throw Self.sqliteError(handle: handle, fallback: "Failed to list news items.") - } - defer { sqlite3_finalize(statement) } - var rows: [NewsItem] = [] - while sqlite3_step(statement) == SQLITE_ROW { - rows.append(try decodeNewsItem(statement: statement)) - } - return rows - } - } - - private func decodeNewsReader(statement: OpaquePointer) throws -> NewsReaderSpec { - func text(_ index: Int32) -> String { - String(cString: sqlite3_column_text(statement, index)) - } - func optionalText(_ index: Int32) -> String? { - sqlite3_column_type(statement, index) == SQLITE_NULL ? nil : text(index) - } - let topics = (try? JSONDecoder().decode([String].self, from: Data(text(2).utf8))) ?? [] - let sources = (try? JSONDecoder().decode([NewsSource].self, from: Data(text(3).utf8))) ?? [] - return NewsReaderSpec( - id: text(0), - name: text(1), - topics: topics, - sources: sources, - mode: NewsReaderMode(rawValue: text(4)) ?? .rss, - maxCount: Int(sqlite3_column_int(statement, 5)), - schedule: NewsReaderSchedule(rawValue: text(6)) ?? .off, - summaryText: optionalText(7), - lastError: optionalText(8), - lastFetchedAt: optionalText(9).flatMap { Self.iso8601Formatter().date(from: $0) }, - createdAt: Self.iso8601Formatter().date(from: text(10)) ?? .now, - updatedAt: Self.iso8601Formatter().date(from: text(11)) ?? .now - ) - } - - private func decodeNewsItem(statement: OpaquePointer) throws -> NewsItem { - func text(_ index: Int32) -> String { - String(cString: sqlite3_column_text(statement, index)) - } - func optionalText(_ index: Int32) -> String? { - sqlite3_column_type(statement, index) == SQLITE_NULL ? nil : text(index) - } - return NewsItem( - id: text(0), - readerID: text(1), - title: text(2), - sourceURL: text(3), - sourceLabel: text(4), - summary: optionalText(5), - publishedAt: optionalText(6).flatMap { Self.iso8601Formatter().date(from: $0) }, - fetchedAt: Self.iso8601Formatter().date(from: text(7)) ?? .now - ) - } -} diff --git a/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift b/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift index 6e3121c..b2aee8f 100644 --- a/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift +++ b/packages/DBRepository/Sources/DBRepository/DatabaseSchema.swift @@ -2,7 +2,7 @@ import Foundation import Structure public enum DatabaseSchema { - public static let latestVersion = 8 + public static let latestVersion = 9 public static func migrationSQL(version: Int, isUp: Bool) throws -> String { let migrationName = String(format: "%04d_%@", version, migrationFileBaseName(for: version)) @@ -39,6 +39,8 @@ public enum DatabaseSchema { return "messaging_agent_handled" case 8: return "messaging_thread_default_profile" + case 9: + return "drop_news_readers" default: return "unknown" } diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql new file mode 100644 index 0000000..054ea0f --- /dev/null +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.down.sql @@ -0,0 +1,30 @@ +CREATE TABLE IF NOT EXISTS news_readers ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + topics_json TEXT NOT NULL DEFAULT '[]', + sources_json TEXT NOT NULL DEFAULT '[]', + mode TEXT NOT NULL, + max_count INTEGER NOT NULL, + schedule TEXT NOT NULL, + summary_text TEXT, + last_error TEXT, + last_fetched_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS news_items ( + id TEXT PRIMARY KEY NOT NULL, + reader_id TEXT NOT NULL, + title TEXT NOT NULL, + source_url TEXT NOT NULL, + source_label TEXT NOT NULL, + summary TEXT, + published_at TEXT, + fetched_at TEXT NOT NULL, + UNIQUE(reader_id, source_url), + FOREIGN KEY(reader_id) REFERENCES news_readers(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_news_items_reader + ON news_items(reader_id, fetched_at DESC); diff --git a/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql new file mode 100644 index 0000000..cbb5bff --- /dev/null +++ b/packages/DBRepository/Sources/DBRepository/Resources/Migrations/0009_drop_news_readers.up.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS news_items; +DROP TABLE IF EXISTS news_readers; diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift deleted file mode 100644 index c87e195..0000000 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBNewsReaderTests.swift +++ /dev/null @@ -1,62 +0,0 @@ -import XCTest -import Structure -@testable import DBRepository - -final class DBNewsReaderTests: XCTestCase { - func testNewsReaderRoundTripAndItems() async throws { - let repository = try makeRepository() - _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - - let spec = NewsReaderSpec( - name: "Markets", - topics: ["Financial", "rates"], - sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/world/rss.xml")], - mode: .summary, - maxCount: 10, - schedule: .daily, - summaryText: "Markets moved higher." - ) - try await repository.upsertNewsReader(spec) - let listed = try await repository.listNewsReaders() - XCTAssertEqual(listed.count, 1) - XCTAssertEqual(listed[0].name, "Markets") - XCTAssertEqual(listed[0].topics, ["Financial", "rates"]) - XCTAssertEqual(listed[0].mode, .summary) - XCTAssertEqual(listed[0].summaryText, "Markets moved higher.") - - let item = NewsItem( - readerID: spec.id, - title: "Rates rise", - sourceURL: "https://example.com/rates", - sourceLabel: "BBC", - summary: "A summary" - ) - try await repository.replaceNewsItems(readerID: spec.id, items: [item]) - let items = try await repository.listNewsItems(readerID: spec.id) - XCTAssertEqual(items.count, 1) - XCTAssertEqual(items[0].sourceURL, "https://example.com/rates") - - try await repository.deleteNewsReader(id: spec.id) - let remainingReaders = try await repository.listNewsReaders() - let remainingItems = try await repository.listNewsItems(readerID: spec.id) - XCTAssertTrue(remainingReaders.isEmpty) - XCTAssertTrue(remainingItems.isEmpty) - } - - private func makeRepository() throws -> DBRepository { - let directory = FileManager.default.temporaryDirectory.appendingPathComponent( - UUID().uuidString, - isDirectory: true - ) - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - return DBRepository( - configuration: DBRepositoryConfiguration( - applicationName: "ui", - databaseName: "derrick", - databaseDirectoryURL: directory, - username: "app-user", - password: "app-secret" - ) - ) - } -} diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift index 9adf82a..4d2ed20 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift @@ -108,7 +108,7 @@ final class DBRepositoryTests: XCTestCase { _ = try await repository.migrateSessionMemory(username: "app-user", password: "app-secret") XCTAssertEqual(try schemaVersion(at: url), DatabaseSchema.latestVersion) - XCTAssertTrue(try tableExists(named: "news_readers", at: url)) + XCTAssertFalse(try tableExists(named: "news_readers", at: url)) XCTAssertTrue(try tableExists(named: "agent_profiles", at: url)) XCTAssertTrue(try tableExists(named: "messaging_agent_handled", at: url)) let loaded = try await repository.pluginFactoryRelease(pluginID: "keep-me", version: "1.0.0") diff --git a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift index 391ce1f..ec7cd57 100644 --- a/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift +++ b/packages/DockerRunnerXPC/Sources/DockerRunnerXPC/DockerRunRequestValidator.swift @@ -213,7 +213,6 @@ public enum DockerRunRequestValidator: Sendable { } case DockerWorkerRuntime.crawlerBinary, DockerWorkerRuntime.extractorBinary, - DockerWorkerRuntime.newsReaderBinary, DockerWorkerRuntime.guestBinaryPath: guard args == [command] else { return .disallowedDockerFlag("exec \(command)") diff --git a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift index cc7b002..bd38778 100644 --- a/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift +++ b/packages/DockerRunnerXPC/Tests/DockerRunnerXPCTests/DockerRunnerXPCTests.swift @@ -240,7 +240,6 @@ struct DockerRunnerXPCTests { ["rm", "-f", "c"], ["inspect", "-f", "{{.State.Running}}", "c"], ["exec", "-i", "c", "/usr/local/bin/derrick-file-extractor"], - ["exec", "-i", "c", DockerWorkerRuntime.newsReaderBinary], [ "create", "--label", diff --git a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift deleted file mode 100644 index 11c637f..0000000 --- a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerExecutor.swift +++ /dev/null @@ -1,115 +0,0 @@ -import Foundation -import Structure - -/// Runs the prebuilt news reader in a oneshot container: create, exec, rm. -public struct NewsReaderDockerExecutor: Sendable { - public static let image = DockerWorkerRuntime.image - public static let containerPrefix = "derrick-news-reader" - public static let binaryPath = DockerWorkerRuntime.newsReaderBinary - public static let maximumTimeoutSeconds = 300 - public static let dockerNetwork = "bridge" - - private let executor: DockerCLIExecutor - private let queue: DerrickDockerRunQueue - - public init( - executor: @escaping DockerCLIExecutor, - queue: DerrickDockerRunQueue = .crawler - ) { - self.executor = executor - self.queue = queue - } - - public func run( - input: Data, - timeoutSeconds: Int - ) async throws -> DockerCLIResult { - let timeout = min(max(timeoutSeconds, 1), Self.maximumTimeoutSeconds) - try await WorkerImageGate.shared.ensureReady(executor: executor) - let prepared = try await NewsReaderDockerInputPreparer.enrich(input) - let executor = self.executor - do { - return try await queue.withPermit { - let proxyLease = try await WebCrawlerEgressProxy.shared.lease(forHosts: prepared.leaseHosts) - do { - let result = try await OneshotDockerContainer.run( - executor: executor, - prefix: Self.containerPrefix, - createArguments: { name in - Self.createArguments( - name: name, - proxyHost: proxyLease.host, - proxyPort: proxyLease.port, - proxyToken: proxyLease.clientToken - ) - }, - createStep: "create news reader container", - startStep: "start news reader container", - body: { name in - try await executor( - ["exec", "-i", name, Self.binaryPath], - prepared.data, - timeout - ) - } - ) - await WebCrawlerEgressProxy.shared.release(forHosts: prepared.leaseHosts) - return result - } catch { - await WebCrawlerEgressProxy.shared.release(forHosts: prepared.leaseHosts) - throw error - } - } - } catch let error as OneshotDockerContainerError { - throw mappedNewsReaderError(error) - } - } - - static func createArguments( - name: String, - proxyHost: String, - proxyPort: Int, - proxyToken: String - ) -> [String] { - [ - "create", - ] + DerrickDockerRuntimeIdentity.createLabelArguments + [ - "--network", dockerNetwork, - "--read-only", - "--tmpfs", "/tmp:rw,exec,nosuid,size=64m", - "--pids-limit", "128", - "--cpus", "1.0", - "--memory", "512m", - "--name", name, - "--env", "DERRICK_EGRESS_PROXY_HOST=\(proxyHost)", - "--env", "DERRICK_EGRESS_PROXY_PORT=\(proxyPort)", - "--env", "DERRICK_EGRESS_PROXY_TOKEN=\(proxyToken)", - "--entrypoint", "/bin/sleep", - image, - "infinity", - ] - } - - private func mappedNewsReaderError(_ error: OneshotDockerContainerError) -> Error { - switch error { - case .commandFailed(let step, let detail): - return NewsReaderDockerExecutorError.commandFailed(step, detail) - case .imageUnavailable(let detail): - return NewsReaderDockerExecutorError.imageUnavailable(detail) - } - } -} - -public enum NewsReaderDockerExecutorError: Error, LocalizedError, Sendable, Equatable { - case commandFailed(String, String) - case imageUnavailable(String) - - public var errorDescription: String? { - switch self { - case .commandFailed(let step, let detail): - return "\(step) failed: \(detail)" - case .imageUnavailable(let image): - return "News reader image is not installed: \(image)." - } - } -} diff --git a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift b/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift deleted file mode 100644 index a894dbe..0000000 --- a/packages/MCPServer/Sources/MCPServer/NewsReaderDockerInputPreparer.swift +++ /dev/null @@ -1,43 +0,0 @@ -import Foundation -import Structure -import WebCrawler - -/// Host-side helpers for preparing news reader Docker input and egress policy. -enum NewsReaderDockerInputPreparer { - static func enrich(_ input: Data) async throws -> (data: Data, leaseHosts: [String]) { - let request: NewsReaderWorkerRequest - do { - request = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: input) - } catch { - throw NewsReaderDockerExecutorError.commandFailed( - "prepare news reader container", - "News reader input was not valid JSON." - ) - } - guard !request.sources.isEmpty else { - throw NewsReaderDockerExecutorError.commandFailed( - "prepare news reader container", - "News reader input did not include any sources." - ) - } - - var hosts = Set() - for source in request.sources { - guard let url = URL(string: source.url) else { - throw NewsReaderDockerExecutorError.commandFailed( - "prepare news reader container", - "Source URL is not valid: \(source.url)" - ) - } - let chain = await WebCrawlerRedirectResolver.hostsInRedirectChain(from: url) - hosts.formUnion(chain) - } - guard !hosts.isEmpty else { - throw NewsReaderDockerExecutorError.commandFailed( - "prepare news reader container", - "Could not determine hosts to fetch from the configured sources." - ) - } - return (input, Array(hosts).sorted()) - } -} diff --git a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift index 25576ca..c9dd3a5 100644 --- a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift +++ b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift @@ -282,7 +282,7 @@ import WebCrawler #expect(allowed?.contains("docs.slack.dev") == true) } - @Test func workerImageLabelDetectsMissingNewsReaderBinary() async { + @Test func workerImageLabelDetectsMissingBinaries() async { let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in await recorder.append(args) @@ -295,47 +295,6 @@ import WebCrawler #expect(!current) } - @Test func newsReaderContainerCreateAndExecPassDockerValidator() { - let createArgs = NewsReaderDockerExecutor.createArguments( - name: "derrick-news-reader-test", - proxyHost: "172.17.0.1", - proxyPort: 3128, - proxyToken: "token" - ) - #expect( - DockerRunRequestValidator.validate( - DockerHostLaunch.makeRequest(dockerArguments: createArgs, timeoutSeconds: 60) - ) == nil - ) - let execArgs = DockerHostLaunch.dockerCLIArguments([ - "exec", "-i", "derrick-news-reader-test", NewsReaderDockerExecutor.binaryPath, - ]) - #expect( - DockerRunRequestValidator.validate( - DockerHostLaunch.makeRequest(dockerArguments: execArgs, timeoutSeconds: 60) - ) == nil - ) - } - - @Test func newsReaderInputPreparerUsesSourceHostsNotCrawlerStartURL() async throws { - let input = try JSONEncoder.service.encode( - NewsReaderWorkerRequest( - mode: .rss, - sources: [ - NewsSource(label: "Google News", url: "https://news.google.com/rss?hl=en-US"), - ], - topics: ["Tech"], - maxCount: 20 - ) - ) - - let prepared = try await NewsReaderDockerInputPreparer.enrich(input) - #expect(prepared.leaseHosts.contains("news.google.com")) - let decoded = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: prepared.data) - #expect(decoded.sources.count == 1) - #expect(decoded.sources[0].url.contains("news.google.com")) - } - @Test func dockerProductImagePrewarmerSkipsBuildWhenImagePresent() async throws { let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in diff --git a/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift b/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift index eab2002..d8e8e74 100644 --- a/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift +++ b/packages/Structure/Sources/AppLayerServices/JobKeepAlive/InProcessServiceBridges.swift @@ -38,7 +38,4 @@ public enum InProcessServiceBridges: Sendable { public typealias RouteMessagingAgent = @Sendable (MessagingAgentRoute) async throws -> Void nonisolated(unsafe) public static var messagingAgentRoute: RouteMessagingAgent? - - public typealias RunNewsReader = @Sendable (Data) async throws -> NewsReaderRunResult - nonisolated(unsafe) public static var runNewsReader: RunNewsReader? } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift index 9d73214..5d3808f 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPServiceXPC.swift @@ -19,8 +19,6 @@ import CryptoKit func callTool(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) /// Signed `searchTools` envelope. Reply is `MCPToolSearchResultDTO`. func searchTools(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) - /// Runs the Docker news reader worker (stdin JSON request). Reply is `NewsReaderRunResult`. - func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) } public struct MCPServiceBootstrapResult: Codable, Sendable, Hashable { @@ -350,12 +348,4 @@ public enum MCPServiceXPCCodec { public static func decodeString(_ data: Data) -> String { String(data: data, encoding: .utf8) ?? "" } - - public static func encodeNewsReaderRunResult(_ result: NewsReaderRunResult) throws -> Data { - try JSONEncoder.service.encode(result) - } - - public static func decodeNewsReaderRunResult(_ data: Data) throws -> NewsReaderRunResult { - try JSONDecoder.service.decode(NewsReaderRunResult.self, from: data) - } } diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift b/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift index ed48b99..94eec53 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/MCPToolCallTimeouts.swift @@ -12,9 +12,6 @@ public enum MCPToolCallTimeouts { /// Must be at least as long as `MCPServiceDockerHelperRunner` call timeout. public static let pluginInvokeNanoseconds: UInt64 = 120_000_000_000 - /// News reader Docker worker (RSS fetch + optional summary prep). - public static let newsReaderNanoseconds: UInt64 = 200_000_000_000 - public static func nanoseconds(forToolName toolName: String) -> UInt64 { switch toolName { case "web.crawl", "plugin_factory_build", "script_exec": diff --git a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift index 923fdff..a152366 100644 --- a/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift +++ b/packages/Structure/Sources/AppLayerServices/MCPService/PluginFactoryCreateInput.swift @@ -4,7 +4,6 @@ import Foundation public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { public enum PluginType: String, Codable, Sendable, CaseIterable { case connector - case newsReader = "news_reader" case custom } @@ -148,8 +147,6 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { pluginID: pluginID, skillMarkdown: skillMarkdown ) - case .newsDigest: - throw PluginSkillDraftError.newsUsesReaderPath } } @@ -332,7 +329,6 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { case preview case credentials case build - case news } public static func failureStep(forStage stage: String?) -> FailureStep { @@ -345,8 +341,6 @@ public struct PluginFactoryCreateInput: Codable, Sendable, Hashable { return .preview case "auth", "discover", "credentials": return .credentials - case "news", "paywall": - return .news case "crawl", "docs", "factory", "build", "review": return .build default: diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift deleted file mode 100644 index 57ce652..0000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderModels.swift +++ /dev/null @@ -1,231 +0,0 @@ -import Foundation - -public enum NewsReaderMode: String, Codable, Sendable, Hashable, CaseIterable { - case rss - case list - case summary - - public var displayName: String { - switch self { - case .rss: return "RSS feed" - case .list: return "Crawl site" - case .summary: return "AI summary" - } - } - - public init(from decoder: Decoder) throws { - let raw = try decoder.singleValueContainer().decode(String.self) - switch raw { - case "rss": self = .rss - case "list": self = .list - case "summary", "summaries": self = .summary - default: - self = .rss - } - } -} - -public enum NewsReaderSchedule: String, Codable, Sendable, Hashable, CaseIterable { - case off - case hourly - case daily - - public var displayName: String { - switch self { - case .off: return "Only when opened" - case .hourly: return "Every hour" - case .daily: return "Every day" - } - } -} - -public struct NewsSource: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var label: String - public var url: String - - public init(id: String = UUID().uuidString, label: String, url: String) { - self.id = id - self.label = label - self.url = url.trimmingCharacters(in: .whitespacesAndNewlines) - } -} - -public struct NewsReaderSpec: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var name: String - public var topics: [String] - public var sources: [NewsSource] - public var mode: NewsReaderMode - public var maxCount: Int - public var schedule: NewsReaderSchedule - public var summaryText: String? - public var lastError: String? - public var lastFetchedAt: Date? - public var createdAt: Date - public var updatedAt: Date - - public init( - id: String = UUID().uuidString, - name: String, - topics: [String], - sources: [NewsSource], - mode: NewsReaderMode = .rss, - maxCount: Int = 20, - schedule: NewsReaderSchedule = .off, - summaryText: String? = nil, - lastError: String? = nil, - lastFetchedAt: Date? = nil, - createdAt: Date = .now, - updatedAt: Date = .now - ) { - self.id = id - self.name = name.trimmingCharacters(in: .whitespacesAndNewlines) - self.topics = topics.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty } - self.sources = sources.filter { !$0.url.isEmpty } - self.mode = mode - self.maxCount = min(50, max(1, maxCount)) - self.schedule = schedule - self.summaryText = summaryText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty - self.lastError = lastError - self.lastFetchedAt = lastFetchedAt - self.createdAt = createdAt - self.updatedAt = updatedAt - } -} - -public struct NewsItem: Codable, Sendable, Hashable, Identifiable { - public var id: String - public var readerID: String - public var title: String - public var sourceURL: String - public var sourceLabel: String - public var summary: String? - public var publishedAt: Date? - public var fetchedAt: Date - - public init( - id: String = UUID().uuidString, - readerID: String, - title: String, - sourceURL: String, - sourceLabel: String, - summary: String? = nil, - publishedAt: Date? = nil, - fetchedAt: Date = .now - ) { - self.id = id - self.readerID = readerID - self.title = title.trimmingCharacters(in: .whitespacesAndNewlines) - self.sourceURL = sourceURL.trimmingCharacters(in: .whitespacesAndNewlines) - self.sourceLabel = sourceLabel - self.summary = summary?.trimmingCharacters(in: .whitespacesAndNewlines) - self.publishedAt = publishedAt - self.fetchedAt = fetchedAt - } -} - -public enum NewsPresetTopic: String, Sendable, CaseIterable, Identifiable { - case financial - case tech - case international - case politics - case science - case sports - - public var id: String { rawValue } - - public var displayName: String { - switch self { - case .financial: return "Financial" - case .tech: return "Tech" - case .international: return "International" - case .politics: return "Politics" - case .science: return "Science" - case .sports: return "Sports" - } - } -} - -public enum NewsPresetSource: String, Sendable, CaseIterable, Identifiable { - case googleNews - case foxNews - case newsmax - case nationalReview - case wsj - - public var id: String { rawValue } - - public var source: NewsSource { - switch self { - case .googleNews: - return NewsSource( - id: rawValue, - label: "Google News", - url: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" - ) - case .foxNews: - return NewsSource( - id: rawValue, - label: "Fox News", - url: "https://moxie.foxnews.com/google-publisher/latest.xml" - ) - case .newsmax: - return NewsSource( - id: rawValue, - label: "Newsmax", - url: "https://www.newsmax.com/rss/Newsfront/" - ) - case .nationalReview: - return NewsSource( - id: rawValue, - label: "National Review", - url: "https://www.nationalreview.com/feed/" - ) - case .wsj: - return NewsSource( - id: rawValue, - label: "Wall Street Journal", - url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" - ) - } - } -} - -public enum NewsReaderError: Error, Sendable, Equatable, LocalizedError { - case paywalled(url: String, detail: String) - case invalidURL(String) - case emptySources - case emptyName - case fetchFailed(url: String, detail: String) - case notReady - case summarizerUnavailable - case workerUnavailable(String) - - public var errorDescription: String? { - switch self { - case .paywalled(let url, let detail): - return "This source is behind a paywall, which is not supported yet. \(detail) (\(url))" - case .invalidURL(let url): - return "That is not a usable web address: \(url)" - case .emptySources: - return "Add at least one source or URL." - case .emptyName: - return "Give this news list a name." - case .fetchFailed(let url, let detail): - return "Could not read \(url). \(detail)" - case .notReady: - return "News lists are not ready yet. Try again in a moment." - case .summarizerUnavailable: - return "Add an API key in Settings before creating an AI summary list." - case .workerUnavailable(let detail): - return "News reader worker is not available. \(detail)" - } - } -} - -private extension String { - var nilIfEmpty: String? { - isEmpty ? nil : self - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift deleted file mode 100644 index aba438e..0000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderRefresh.swift +++ /dev/null @@ -1,108 +0,0 @@ -import Foundation - -public enum NewsReaderRefresh { - public static func validateAndFetch( - spec: NewsReaderSpec, - worker: any NewsWorkerRunning, - summarizer: NewsSummaryGenerating? = nil - ) async throws -> NewsReaderFetchResult { - let name = spec.name.trimmingCharacters(in: .whitespacesAndNewlines) - guard !name.isEmpty else { throw NewsReaderError.emptyName } - guard !spec.sources.isEmpty else { throw NewsReaderError.emptySources } - - for source in spec.sources { - guard let url = URL(string: source.url), url.scheme == "http" || url.scheme == "https" else { - throw NewsReaderError.invalidURL(source.url) - } - if let reason = NewsPaywall.preflightRejection(url: url) { - throw NewsReaderError.paywalled(url: source.url, detail: reason) - } - } - - let request = NewsReaderWorkerRequest( - mode: spec.mode, - sources: spec.sources, - topics: spec.topics, - maxCount: spec.maxCount, - contextHint: ([spec.name] + spec.topics).joined(separator: " ") - ) - let requestJSON = try request.encodedJSON() - let stdout = try await worker.run(requestJSON: requestJSON) - let result = try JSONDecoder.service.decode(NewsReaderWorkerResult.self, from: stdout) - guard result.ok else { - let detail = result.diagnostics.joined(separator: " ") - throw NewsReaderError.fetchFailed( - url: spec.sources.first?.url ?? name, - detail: detail.isEmpty ? "News reader returned no articles." : detail - ) - } - - let items = result.articles.map { article in - NewsItem( - readerID: spec.id, - title: article.title, - sourceURL: article.url, - sourceLabel: sourceLabel(for: article.url, sources: spec.sources), - summary: article.detail?.nilIfEmpty, - publishedAt: parsePublishedAt(article.publishedAt) - ) - } - - var summaryText: String? - if spec.mode == .summary { - guard let summarizer else { - throw NewsReaderError.summarizerUnavailable - } - summaryText = try await summarizer.summarize( - listName: spec.name, - topics: spec.topics, - articles: items - ) - } - - return NewsReaderFetchResult(items: items, summaryText: summaryText) - } - - private static func sourceLabel(for url: String, sources: [NewsSource]) -> String { - if let host = URL(string: url)?.host { - if let match = sources.first(where: { URL(string: $0.url)?.host == host }) { - return match.label - } - return host - } - return sources.first?.label ?? "Source" - } - - private static func parsePublishedAt(_ raw: String?) -> Date? { - guard let raw, !raw.isEmpty else { return nil } - let rfc = DateFormatter() - rfc.locale = Locale(identifier: "en_US_POSIX") - rfc.dateFormat = "EEE, dd MMM yyyy HH:mm:ss Z" - if let date = rfc.date(from: raw) { return date } - let iso = ISO8601DateFormatter() - iso.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - if let date = iso.date(from: raw) { return date } - iso.formatOptions = [.withInternetDateTime] - return iso.date(from: raw) - } -} - -public struct NewsReaderFetchResult: Sendable, Hashable { - public var items: [NewsItem] - public var summaryText: String? - - public init(items: [NewsItem], summaryText: String? = nil) { - self.items = items - self.summaryText = summaryText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty - } -} - -public protocol NewsSummaryGenerating: Sendable { - func summarize(listName: String, topics: [String], articles: [NewsItem]) async throws -> String -} - -private extension String { - var nilIfEmpty: String? { - isEmpty ? nil : self - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift b/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift deleted file mode 100644 index 42ddc0e..0000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsReaderWorkerTypes.swift +++ /dev/null @@ -1,72 +0,0 @@ -import Foundation - -public protocol NewsWorkerRunning: Sendable { - func run(requestJSON: Data) async throws -> Data -} - -public struct NewsReaderWorkerRequest: Codable, Sendable, Hashable { - public var mode: NewsReaderMode - public var sources: [NewsSource] - public var topics: [String] - public var maxCount: Int - public var contextHint: String? - - public init( - mode: NewsReaderMode, - sources: [NewsSource], - topics: [String], - maxCount: Int, - contextHint: String? = nil - ) { - self.mode = mode - self.sources = sources - self.topics = topics - self.maxCount = maxCount - self.contextHint = contextHint?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty - } - - public func encodedJSON() throws -> Data { - try JSONEncoder.service.encode(self) - } -} - -public struct NewsReaderWorkerArticle: Codable, Sendable, Hashable { - public var title: String - public var url: String - public var detail: String? - public var publishedAt: String? - - enum CodingKeys: String, CodingKey { - case title - case url - case detail - case publishedAt = "published_at" - } -} - -public struct NewsReaderWorkerResult: Codable, Sendable, Hashable { - public var ok: Bool - public var mode: NewsReaderMode - public var articles: [NewsReaderWorkerArticle] - public var diagnostics: [String] -} - -public struct NewsReaderRunResult: Codable, Sendable, Hashable { - public var ok: Bool - public var stdout: Data - public var stderr: Data - public var message: String - - public init(ok: Bool, stdout: Data = Data(), stderr: Data = Data(), message: String = "") { - self.ok = ok - self.stdout = stdout - self.stderr = stderr - self.message = message - } -} - -private extension String { - var nilIfEmpty: String? { - isEmpty ? nil : self - } -} diff --git a/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift b/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift deleted file mode 100644 index 0d082d2..0000000 --- a/packages/Structure/Sources/AppLayerServices/News/NewsWorkerBridge.swift +++ /dev/null @@ -1,30 +0,0 @@ -import Foundation - -public enum NewsWorkerBridge: Sendable { - public typealias Runner = @Sendable (Data) async throws -> Data - - private final class Storage: @unchecked Sendable { - var runner: Runner? - } - - private static let storage = Storage() - - public static func install(_ runner: @escaping Runner) { - storage.runner = runner - } - - public static func run(requestJSON: Data) async throws -> Data { - guard let runner = storage.runner else { - throw NewsReaderError.workerUnavailable("Docker news reader is not ready yet.") - } - return try await runner(requestJSON) - } -} - -public struct BridgedNewsWorker: NewsWorkerRunning { - public init() {} - - public func run(requestJSON: Data) async throws -> Data { - try await NewsWorkerBridge.run(requestJSON: requestJSON) - } -} diff --git a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift index bcdc13f..f6b7d28 100644 --- a/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift +++ b/packages/Structure/Sources/AppLayerServices/Plugin/PluginSkillDraft.swift @@ -32,7 +32,6 @@ public struct PluginSkillDraft: Sendable, Hashable { public enum PlannedKind: String, Sendable, Hashable { case messagingConnector - case newsDigest case customCapability } @@ -41,25 +40,19 @@ public struct PluginSkillDraft: Sendable, Hashable { public var triggers: Set public var examples: [Example] public var pluginName: String - public var newsTopics: [String] - public var newsSourceURLs: [String] public init( goal: String = "", purpose: String = "", triggers: Set = [.chat], examples: [Example] = [], - pluginName: String = "", - newsTopics: [String] = [], - newsSourceURLs: [String] = [] + pluginName: String = "" ) { self.goal = goal self.purpose = purpose self.triggers = triggers self.examples = examples self.pluginName = pluginName - self.newsTopics = newsTopics - self.newsSourceURLs = newsSourceURLs } public var plannedKind: PlannedKind { @@ -74,7 +67,7 @@ public struct PluginSkillDraft: Sendable, Hashable { switch plannedKind { case .messagingConnector: return inferredConnectorVendor?.isSelectableInWizard == true - case .newsDigest, .customCapability: + case .customCapability: return true } } @@ -104,12 +97,6 @@ public struct PluginSkillDraft: Sendable, Hashable { public func packageOutline() -> [String] { switch plannedKind { - case .newsDigest: - return [ - "plugin.json — name, schedule, and reader settings", - "skills/\(normalizedPluginFolderName())/SKILL.md — when Derrick uses this list", - "News fetcher — loads articles with source links", - ] case .messagingConnector, .customCapability: return [ "plugin.json — name, permissions, and secrets", @@ -143,8 +130,6 @@ public enum PluginSkillDraftPlanner { for kind: PluginSkillDraft.PlannedKind ) -> Set { switch kind { - case .newsDigest: - return [.chat, .schedule] case .messagingConnector: return [.chat, .messaging, .mention] case .customCapability: @@ -160,22 +145,8 @@ public enum PluginSkillDraftPlanner { } } - public static func inferNewsMode(from draft: PluginSkillDraft) -> NewsReaderMode { - let text = combinedText(draft) - if ["summary", "summarize", "summaries", "digest", "brief", "overview"] - .contains(where: { text.contains($0) }) { - return .summary - } - if ["crawl", "website", "homepage", "web page", "webpage", "site"] - .contains(where: { text.contains($0) }) { - return .list - } - return .rss - } - public static func inferKind(from draft: PluginSkillDraft) -> PluginSkillDraft.PlannedKind { let text = combinedText(draft) - if looksLikeNews(text) { return .newsDigest } if looksLikeMessaging(text) { return .messagingConnector } return .customCapability } @@ -208,12 +179,6 @@ public enum PluginSkillDraftPlanner { draft.triggers = defaultTriggers(for: draft) } sanitizeTriggers(in: &draft) - if inferKind(from: draft) == .newsDigest, draft.newsTopics.isEmpty { - draft.newsTopics = defaultNewsTopics(from: trimmed) - } - if inferKind(from: draft) == .newsDigest, draft.newsSourceURLs.isEmpty { - draft.newsSourceURLs = [NewsPresetSource.googleNews.source.url] - } } public static func skillMarkdown(for draft: PluginSkillDraft) -> String { @@ -259,15 +224,6 @@ public enum PluginSkillDraftPlanner { Confirmed behavior: \(examples) """ - case .newsDigest: - return """ - \(purpose) - - News reader that fetches articles from configured sources and includes source links. - - Topics: \(draft.newsTopics.joined(separator: ", ")) - Sources: \(draft.newsSourceURLs.joined(separator: ", ")) - """ case .customCapability: return """ \(purpose) @@ -313,8 +269,6 @@ public enum PluginSkillDraftPlanner { Return go_source, test_input_json, and skill_files. The host writes plugin.json when a host manifest is supplied; otherwise include a valid manifest in your output path via the builder contract. """ - case .newsDigest: - throw PluginSkillDraftError.newsUsesReaderPath } } @@ -324,10 +278,6 @@ public enum PluginSkillDraftPlanner { .lowercased() } - private static func looksLikeNews(_ text: String) -> Bool { - ["news", "rss", "headline", "digest", "articles", "reader"].contains { text.contains($0) } - } - private static func looksLikeMessaging(_ text: String) -> Bool { ["slack", "telegram", "whatsapp", "discord", "messaging", "channel", "inbox", "dm", "chat app"] .contains { text.contains($0) } @@ -340,12 +290,6 @@ public enum PluginSkillDraftPlanner { return ConnectorPluginNaming.defaultPluginID(vendor: vendor, existingIDs: existingIDs) } return ConnectorPluginNaming.defaultPluginID(vendor: .slack, existingIDs: existingIDs) - case .newsDigest: - let base = "news-list" - if !existingIDs.contains(base) { return base } - var index = 2 - while existingIDs.contains("\(base)-\(index)") { index += 1 } - return "\(base)-\(index)" case .customCapability: let words = draft.goal .lowercased() @@ -362,8 +306,6 @@ public enum PluginSkillDraftPlanner { switch inferKind(from: draft) { case .messagingConnector: return [.messaging, .chat] - case .newsDigest: - return [.schedule, .chat] case .customCapability: return [.chat] } @@ -383,13 +325,6 @@ public enum PluginSkillDraftPlanner { pluginDoes: "post the message in that channel" ), ] - case .newsDigest: - return [ - PluginSkillDraft.Example( - userSays: "What's in my news list?", - pluginDoes: "fetch the latest articles with links to the original sources" - ), - ] case .customCapability: let snippet = draft.goal.trimmingCharacters(in: .whitespacesAndNewlines) return [ @@ -400,46 +335,16 @@ public enum PluginSkillDraftPlanner { ] } } - - private static func defaultNewsTopics(from goal: String) -> [String] { - let lower = goal.lowercased() - var topics: [String] = [] - if lower.contains("financ") || lower.contains("market") { - topics.append(NewsPresetTopic.financial.displayName) - } - if lower.contains("tech") { - topics.append(NewsPresetTopic.tech.displayName) - } - if lower.contains("world") || lower.contains("international") { - topics.append(NewsPresetTopic.international.displayName) - } - if lower.contains("politic") { - topics.append(NewsPresetTopic.politics.displayName) - } - if lower.contains("science") { - topics.append(NewsPresetTopic.science.displayName) - } - if lower.contains("sport") { - topics.append(NewsPresetTopic.sports.displayName) - } - if topics.isEmpty { - topics.append(NewsPresetTopic.tech.displayName) - } - return topics - } } public enum PluginSkillDraftError: Error, LocalizedError { case missingConnectorVendor - case newsUsesReaderPath case invalidPluginName public var errorDescription: String? { switch self { case .missingConnectorVendor: return "Could not determine which messaging service this plugin targets." - case .newsUsesReaderPath: - return "News lists use the reader path, not the plugin factory." case .invalidPluginName: return "Choose a valid plugin name using letters, numbers, and hyphens." } diff --git a/packages/Structure/Sources/Contract/GuestContract.swift b/packages/Structure/Sources/Contract/GuestContract.swift index 4385f61..fdf034a 100644 --- a/packages/Structure/Sources/Contract/GuestContract.swift +++ b/packages/Structure/Sources/Contract/GuestContract.swift @@ -15,7 +15,6 @@ public enum GuestContract: Sendable { case workerProduct = "worker-product.schema.json" case webCrawlerResult = "web-crawler-result.schema.json" case fileExtractorResult = "file-extractor-result.schema.json" - case newsReaderResult = "news-reader-result.schema.json" case scriptExecContract = "script-exec-contract.schema.json" } diff --git a/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json deleted file mode 100644 index 6632851..0000000 --- a/packages/Structure/Sources/Contract/Resources/schemas/news-reader-result.schema.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://derrick.local/schemas/news-reader-result.json", - "title": "News reader worker stdout", - "description": "JSON object written to stdout by derrick-news-reader.", - "$ref": "worker-product.schema.json#/$defs/news_reader_result" -} diff --git a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json index 611ef5f..874f6bf 100644 --- a/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json +++ b/packages/Structure/Sources/Contract/Resources/schemas/worker-product.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://derrick.local/schemas/worker-product.json", "title": "Derrick trusted worker product contracts", - "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler, file extractor, and news reader). Swift host and Go workers must match these schemas.", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", "$defs": { "string_list": { "type": "array", @@ -92,35 +92,6 @@ }, "diagnostics": { "$ref": "#/$defs/string_list" } } - }, - "news_reader_mode": { - "type": "string", - "enum": ["rss", "list", "summary"] - }, - "news_reader_article": { - "type": "object", - "required": ["title", "url"], - "additionalProperties": false, - "properties": { - "title": { "type": "string" }, - "url": { "type": "string" }, - "detail": { "type": "string" }, - "published_at": { "type": "string" } - } - }, - "news_reader_result": { - "type": "object", - "required": ["ok", "mode", "articles", "diagnostics"], - "additionalProperties": false, - "properties": { - "ok": { "type": "boolean" }, - "mode": { "$ref": "#/$defs/news_reader_mode" }, - "articles": { - "type": "array", - "items": { "$ref": "#/$defs/news_reader_article" } - }, - "diagnostics": { "$ref": "#/$defs/string_list" } - } } } } diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift index 040be5a..ca2f57e 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickDockerRuntimeIdentity.swift @@ -13,9 +13,8 @@ public enum DerrickDockerRuntimeIdentity: Sendable { /// Name prefixes for current and unlabeled leftover containers. public static let namePrefixes = [ - "derrick-web-crawler", - "derrick-news-reader", - "derrick-guest-runtime", + "derrick-web-crawler", + "derrick-guest-runtime", "derrick-swift-runtime", "derrick-file-extractor", ] diff --git a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift index 9a5c9d5..248376c 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DockerWorkerRuntime.swift @@ -8,12 +8,10 @@ public enum DockerWorkerRuntime: Sendable { public static let crawlerBinary = "/usr/local/bin/derrick-web-crawler" public static let extractorBinary = "/usr/local/bin/derrick-file-extractor" - public static let newsReaderBinary = "/usr/local/bin/derrick-news-reader" /// Binaries that must exist in the unified worker image. public static let requiredBinaries: [String] = [ crawlerBinary, extractorBinary, - newsReaderBinary, ] public static let guestBinaryPath = "/tmp/guest" @@ -30,5 +28,5 @@ public enum DockerWorkerRuntime: Sendable { /// OCI label written by `docker/worker/Dockerfile`; used to detect stale local images. public static let binariesLabelKey = "derrick.worker.binaries" - public static let binariesLabelValue = "crawler,extractor,news-reader" + public static let binariesLabelValue = "crawler,extractor" } diff --git a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift index b399687..4041344 100644 --- a/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift +++ b/packages/Structure/Tests/StructureTests/AppLayerServicesWireTests.swift @@ -1205,7 +1205,6 @@ import Testing #expect(DerrickDockerRuntimeIdentity.createLabelArguments == ["--label", "app.derrick=runtime"]) #expect(DerrickDockerRuntimeIdentity.namePrefixes == [ "derrick-web-crawler", - "derrick-news-reader", "derrick-guest-runtime", "derrick-swift-runtime", "derrick-file-extractor", @@ -1486,8 +1485,6 @@ import Testing #expect(PluginFactoryCreateInput.failureStep(forStage: "name") == .skill) #expect(PluginFactoryCreateInput.failureStep(forStage: "auth") == .credentials) #expect(PluginFactoryCreateInput.failureStep(forStage: "discover") == .credentials) - #expect(PluginFactoryCreateInput.failureStep(forStage: "paywall") == .news) - #expect(PluginFactoryCreateInput.failureStep(forStage: "news") == .news) #expect(PluginFactoryCreateInput.failureStep(forStage: "goal") == .goal) } diff --git a/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift b/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift new file mode 100644 index 0000000..0659531 --- /dev/null +++ b/packages/Structure/Tests/StructureTests/NewsPaywallTests.swift @@ -0,0 +1,29 @@ +import Foundation +import Testing +import Structure + +@Suite struct NewsPaywallTests { + @Test func paywallWarningIsUserFacing() { + #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("paywall")) + #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("not supported")) + } + + @Test func preflightRejectsPaywalledArticleAllowsFeed() { + #expect( + NewsPaywall.preflightRejection( + url: URL(string: "https://www.nytimes.com/2024/01/01/world.html")! + ) != nil + ) + #expect( + NewsPaywall.preflightRejection( + url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")! + ) == nil + ) + } + + @Test func canonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint() { + let url = URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! + let canonical = NewsSourceURL.canonicalFetchURL(url, contextHint: "tech-news Tech") + #expect(canonical.path.contains("/headlines/section/topic/TECHNOLOGY")) + } +} diff --git a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift b/packages/Structure/Tests/StructureTests/NewsReaderTests.swift deleted file mode 100644 index 7077ce0..0000000 --- a/packages/Structure/Tests/StructureTests/NewsReaderTests.swift +++ /dev/null @@ -1,74 +0,0 @@ -import Foundation -import Testing -import Structure - -@Suite struct NewsReaderTests { - @Test func paywallWarningIsUserFacing() { - #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("paywall")) - #expect(NewsPaywall.userWarning.localizedCaseInsensitiveContains("not supported")) - } - - @Test func preflightRejectsPaywalledArticleAllowsFeed() { - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://www.nytimes.com/2024/01/01/world.html")! - ) != nil - ) - #expect( - NewsPaywall.preflightRejection( - url: URL(string: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")! - ) == nil - ) - } - - @Test func inferNewsModeMapsSummaryCrawlAndRSS() { - let summary = PluginSkillDraft(goal: "Give me a summary of tech news") - #expect(PluginSkillDraftPlanner.inferNewsMode(from: summary) == .summary) - - let crawl = PluginSkillDraft(goal: "Crawl the BBC homepage for headlines") - #expect(PluginSkillDraftPlanner.inferNewsMode(from: crawl) == .list) - - let rss = PluginSkillDraft(goal: "Fetch tech headlines from Google News RSS") - #expect(PluginSkillDraftPlanner.inferNewsMode(from: rss) == .rss) - } - - @Test func legacySummariesModeDecodesToSummary() throws { - struct Wrapper: Decodable { let mode: NewsReaderMode } - let wrapper = try JSONDecoder.service.decode( - Wrapper.self, - from: Data(#"{"mode":"summaries"}"#.utf8) - ) - #expect(wrapper.mode == .summary) - } - - @Test func presetSourcesUsePublicFeedsNotPaywalledArticlePages() { - for preset in NewsPresetSource.allCases { - let url = URL(string: preset.source.url)! - #expect( - NewsPaywall.preflightRejection(url: url) == nil, - "Expected \(preset.source.label) feed to pass paywall preflight" - ) - } - } - - @Test func canonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint() { - let url = URL(string: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en")! - let canonical = NewsSourceURL.canonicalFetchURL(url, contextHint: "tech-news Tech") - #expect(canonical.path.contains("/headlines/section/topic/TECHNOLOGY")) - } - - @Test func workerRequestEncodesModeAndSources() throws { - let request = NewsReaderWorkerRequest( - mode: .rss, - sources: [NewsSource(label: "BBC", url: "https://feeds.bbci.co.uk/news/rss.xml")], - topics: ["Tech"], - maxCount: 10, - contextHint: "tech-news" - ) - let json = try request.encodedJSON() - let decoded = try JSONDecoder.service.decode(NewsReaderWorkerRequest.self, from: json) - #expect(decoded.mode == .rss) - #expect(decoded.sources.count == 1) - #expect(decoded.topics == ["Tech"]) - } -} diff --git a/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift index 561a0a5..a2bb3eb 100644 --- a/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift +++ b/packages/Structure/Tests/StructureTests/PluginSkillDraftTests.swift @@ -3,18 +3,10 @@ import Testing @testable import Structure @Suite struct PluginSkillDraftTests { - @Test func infersSummariesModeFromSummaryGoal() { - var draft = PluginSkillDraft(goal: "Give me a summary of tech headlines") - PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) - #expect(PluginSkillDraftPlanner.inferNewsMode(from: draft) == .summary) - } - - @Test func infersNewsFromGoal() { + @Test func newsGoalIsCustomCapabilityNotACoreProduct() { var draft = PluginSkillDraft(goal: "Fetch tech news headlines daily") PluginSkillDraftPlanner.applyGoal(draft.goal, to: &draft, existingPluginIDs: []) - #expect(draft.plannedKind == .newsDigest) - #expect(!draft.newsTopics.isEmpty) - #expect(!draft.newsSourceURLs.isEmpty) + #expect(draft.plannedKind == .customCapability) } @Test func infersSlackConnectorFromGoal() { @@ -78,18 +70,6 @@ import Testing #expect(input.pluginID == "summarizer") } - @Test func newsDigestOnlyAllowsChatAndScheduleTriggers() { - let allowed = PluginSkillDraftPlanner.availableTriggers(for: .newsDigest) - #expect(allowed == Set([.chat, .schedule])) - var draft = PluginSkillDraft( - goal: "tech news", - triggers: [.chat, .schedule, .mention, .messaging], - pluginName: "tech-news" - ) - PluginSkillDraftPlanner.sanitizeTriggers(in: &draft) - #expect(draft.triggers == Set([.chat, .schedule])) - } - @Test func messagingConnectorDisallowsScheduleTrigger() { let allowed = PluginSkillDraftPlanner.availableTriggers(for: .messagingConnector) #expect(allowed == Set([.chat, .messaging, .mention])) @@ -101,18 +81,4 @@ import Testing PluginSkillDraftPlanner.sanitizeTriggers(in: &draft) #expect(draft.triggers == Set([.chat, .messaging])) } - - @Test func newsDraftRejectsFactoryInput() { - let draft = PluginSkillDraft( - goal: "news digest", - purpose: "News", - examples: [PluginSkillDraft.Example(userSays: "news", pluginDoes: "fetch")], - pluginName: "news-list", - newsTopics: ["Tech"], - newsSourceURLs: ["https://news.google.com/rss"] - ) - #expect(throws: PluginSkillDraftError.newsUsesReaderPath) { - try PluginFactoryCreateInput.makeFromSkillDraft(draft) - } - } } diff --git a/ui/JobKeepAlive/DaemonModuleBootstrap.swift b/ui/JobKeepAlive/DaemonModuleBootstrap.swift index df0c8d8..8595654 100644 --- a/ui/JobKeepAlive/DaemonModuleBootstrap.swift +++ b/ui/JobKeepAlive/DaemonModuleBootstrap.swift @@ -37,9 +37,6 @@ enum DaemonModuleBootstrap { let tools = try await MCPServiceToolHost.shared.searchTools(query: query, principal: principal) return MCPToolSearchResultDTO(ok: true, tools: tools, message: "ok") } - InProcessServiceBridges.runNewsReader = { requestJSON in - try await MCPServiceToolHost.shared.runNewsReader(requestJSON: requestJSON) - } InProcessServiceBridges.jobLocalProxy = JobServiceExportedObject() InProcessServiceBridges.jobNetworkPreflight = { toolName, argumentsJSON, jobID in let repo = try await JobServiceStore.shared.sharedRepository() diff --git a/ui/JobKeepAlive/DaemonUnifiedXPC.swift b/ui/JobKeepAlive/DaemonUnifiedXPC.swift index 1bd52e6..4806931 100644 --- a/ui/JobKeepAlive/DaemonUnifiedXPC.swift +++ b/ui/JobKeepAlive/DaemonUnifiedXPC.swift @@ -234,10 +234,6 @@ final class DaemonUnifiedExportedObject: NSObject, DerrickDaemonServiceXPC, @unc mcp.searchTools(requestJSON: requestJSON, withReply: reply) } - func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { - mcp.runNewsReader(requestJSON: requestJSON, withReply: reply) - } - // MARK: - Workflow runtime func startWorkflow(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { diff --git a/ui/MCPService/MCPServiceExportedObject.swift b/ui/MCPService/MCPServiceExportedObject.swift index d7dc754..e1a90b7 100644 --- a/ui/MCPService/MCPServiceExportedObject.swift +++ b/ui/MCPService/MCPServiceExportedObject.swift @@ -177,21 +177,4 @@ final class MCPServiceExportedObject: NSObject, MCPServiceXPC { } } } - - func runNewsReader(requestJSON: NSData, withReply reply: @escaping @Sendable (NSData) -> Void) { - let data = requestJSON as Data - Task { - do { - let result = try await MCPServiceToolHost.shared.runNewsReader(requestJSON: data) - reply((try MCPServiceXPCCodec.encodeNewsReaderRunResult(result)) as NSData) - } catch { - fputs("[MCPService] runNewsReader failed: \(error.localizedDescription)\n", stderr) - let result = NewsReaderRunResult(ok: false, message: error.localizedDescription) - reply( - (try? MCPServiceXPCCodec.encodeNewsReaderRunResult(result)) as NSData? - ?? Data("{}".utf8) as NSData - ) - } - } - } } diff --git a/ui/MCPService/MCPServiceToolHost.swift b/ui/MCPService/MCPServiceToolHost.swift index 7a7e533..d3aa0af 100644 --- a/ui/MCPService/MCPServiceToolHost.swift +++ b/ui/MCPService/MCPServiceToolHost.swift @@ -195,36 +195,6 @@ actor MCPServiceToolHost { return made } - func runNewsReader(requestJSON: Data, timeoutSeconds: Int = 180) async throws -> NewsReaderRunResult { - _ = try await ensureReady() - let executor = NewsReaderDockerExecutor( - executor: MCPServiceDockerHelperRunner.shared.makeStdinCLIExecutor() - ) - let result = try await executor.run(input: requestJSON, timeoutSeconds: timeoutSeconds) - let stderrText = String(decoding: result.stderr, as: UTF8.self) - if result.exitCode != 0 { - let detail = newsReaderFailureDetail(exitCode: result.exitCode, stderr: stderrText) - await MCPServiceStore.shared.log( - level: .error, - message: "news reader failed: \(detail)", - code: "news_reader_failed" - ) - return NewsReaderRunResult( - ok: false, - stdout: result.stdout, - stderr: result.stderr, - message: detail - ) - } - guard !result.stdout.isEmpty else { - return NewsReaderRunResult( - ok: false, - message: "News reader returned no output." - ) - } - return NewsReaderRunResult(ok: true, stdout: result.stdout, stderr: result.stderr) - } - func searchTools(query: String, principal: ServicePrincipal) async throws -> [MCPToolDescriptorDTO] { let client = try await ensureReady().client await MCPServiceStore.shared.log( @@ -388,15 +358,3 @@ private func pluginFactoryFailureDetail(for error: Error) -> String { } return error.localizedDescription } - -private func newsReaderFailureDetail(exitCode: Int32, stderr: String) -> String { - if exitCode == 126 { - return """ - The worker image on this Mac is missing the news reader binary. Quit Derrick completely and reopen it so the worker image can rebuild, then try again. - """ - } - if !stderr.isEmpty { - return stderr - } - return "exit \(exitCode)" -} diff --git a/ui/SharedAgentRuntime/Services/MCPServiceClient.swift b/ui/SharedAgentRuntime/Services/MCPServiceClient.swift index 4f4dbd1..048f7ce 100644 --- a/ui/SharedAgentRuntime/Services/MCPServiceClient.swift +++ b/ui/SharedAgentRuntime/Services/MCPServiceClient.swift @@ -176,25 +176,6 @@ public final class MCPServiceClient: @unchecked Sendable { } } - public func runNewsReader(requestJSON: Data, timeoutSeconds: Int = 180) async throws -> NewsReaderRunResult { - if DerrickProcessRole.isDaemon, let run = InProcessServiceBridges.runNewsReader { - return try await run(requestJSON) - } - nonisolated(unsafe) let proxy = try remoteProxy() - let payload = requestJSON as NSData - return try await invoke(timeout: MCPToolCallTimeouts.newsReaderNanoseconds) { - try await withCheckedThrowingContinuation { cont in - proxy.runNewsReader(requestJSON: payload) { data in - do { - cont.resume(returning: try MCPServiceXPCCodec.decodeNewsReaderRunResult(data as Data)) - } catch { - cont.resume(throwing: error) - } - } - } - } - } - public func searchTools(principal: ServicePrincipal, query: String = "") async throws -> MCPToolSearchResultDTO { if DerrickProcessRole.isDaemon, let search = InProcessServiceBridges.mcpSearchTools { return try await search(principal, query) diff --git a/ui/ui/Messaging/AppWorkspace.swift b/ui/ui/Messaging/AppWorkspace.swift index 0809553..872c89b 100644 --- a/ui/ui/Messaging/AppWorkspace.swift +++ b/ui/ui/Messaging/AppWorkspace.swift @@ -4,6 +4,5 @@ enum AppWorkspace: Equatable { case chats case plugins case messaging - case news case debugLogs } diff --git a/ui/ui/News/MCPServiceNewsWorker.swift b/ui/ui/News/MCPServiceNewsWorker.swift deleted file mode 100644 index 54c8277..0000000 --- a/ui/ui/News/MCPServiceNewsWorker.swift +++ /dev/null @@ -1,23 +0,0 @@ -import Foundation -import Structure - -/// Runs the Docker news reader in MCPService where the egress proxy can bind. -struct MCPServiceNewsWorker: NewsWorkerRunning { - func run(requestJSON: Data) async throws -> Data { - _ = try await MCPServiceClient.shared.ensureUpAndHealth(retries: 2) - let result = try await MCPServiceClient.shared.runNewsReader(requestJSON: requestJSON) - guard result.ok else { - let detail = result.message.nilIfEmpty - ?? String(decoding: result.stderr, as: UTF8.self).nilIfEmpty - ?? "News reader failed." - throw NewsReaderError.workerUnavailable(detail) - } - return result.stdout - } -} - -private extension String { - var nilIfEmpty: String? { - isEmpty ? nil : self - } -} diff --git a/ui/ui/News/NewsReaderStore.swift b/ui/ui/News/NewsReaderStore.swift deleted file mode 100644 index dde776d..0000000 --- a/ui/ui/News/NewsReaderStore.swift +++ /dev/null @@ -1,148 +0,0 @@ -import Combine -import DBRepository -import Foundation -import Structure - -@MainActor -final class NewsReaderStore: ObservableObject { - static let shared = NewsReaderStore() - - @Published private(set) var readers: [NewsReaderSpec] = [] - @Published private(set) var items: [NewsItem] = [] - @Published var selectedReaderID: String? - @Published private(set) var isRefreshing = false - @Published private(set) var lastError: String? - - private var repository: DBRepository? - private let worker: any NewsWorkerRunning - private var summarizer: NewsReaderSummarizer? - - init( - worker: any NewsWorkerRunning = MCPServiceNewsWorker(), - summarizer: NewsReaderSummarizer? = nil - ) { - self.worker = worker - self.summarizer = summarizer - } - - func attachSummarizer(_ settings: LLMModelSettings) { - self.summarizer = NewsReaderSummarizer(settings: settings) - } - - var selectedReader: NewsReaderSpec? { - readers.first { $0.id == selectedReaderID } - } - - func configure(repository: DBRepository, summarizerSettings: LLMModelSettings? = nil) async { - self.repository = repository - if let summarizerSettings { - attachSummarizer(summarizerSettings) - } - await reload() - } - - func reload() async { - guard let repository else { return } - do { - readers = try await repository.listNewsReaders() - if selectedReaderID == nil { - selectedReaderID = readers.first?.id - } - if let id = selectedReaderID { - items = try await repository.listNewsItems(readerID: id) - } else { - items = [] - } - lastError = nil - } catch { - lastError = error.localizedDescription - } - } - - func select(id: String) async { - selectedReaderID = id - await reloadItems() - if let reader = selectedReader, shouldRefreshForSchedule(reader) { - await refreshSelected() - } - } - - @discardableResult - func create(_ spec: NewsReaderSpec) async throws -> NewsReaderSpec { - guard let repository else { - throw NewsReaderError.notReady - } - var next = spec - next.updatedAt = .now - let fetched = try await NewsReaderRefresh.validateAndFetch( - spec: next, - worker: worker, - summarizer: summarizer - ) - next.lastFetchedAt = .now - next.lastError = nil - next.summaryText = fetched.summaryText - try await repository.upsertNewsReader(next) - try await repository.replaceNewsItems(readerID: next.id, items: fetched.items) - selectedReaderID = next.id - self.items = fetched.items - await reload() - return next - } - - func refreshSelected() async { - guard let repository, var reader = selectedReader else { return } - isRefreshing = true - defer { isRefreshing = false } - do { - let fetched = try await NewsReaderRefresh.validateAndFetch( - spec: reader, - worker: worker, - summarizer: summarizer - ) - reader.lastFetchedAt = .now - reader.lastError = nil - reader.updatedAt = .now - reader.summaryText = fetched.summaryText - try await repository.upsertNewsReader(reader) - try await repository.replaceNewsItems(readerID: reader.id, items: fetched.items) - items = fetched.items - lastError = nil - await reload() - } catch { - reader.lastError = error.localizedDescription - reader.updatedAt = .now - try? await repository.upsertNewsReader(reader) - lastError = error.localizedDescription - await reload() - } - } - - func deleteSelected() async { - guard let repository, let id = selectedReaderID else { return } - try? await repository.deleteNewsReader(id: id) - selectedReaderID = nil - await reload() - } - - private func reloadItems() async { - guard let repository, let id = selectedReaderID else { - items = [] - return - } - items = (try? await repository.listNewsItems(readerID: id)) ?? [] - } - - private func shouldRefreshForSchedule(_ reader: NewsReaderSpec) -> Bool { - guard reader.schedule != .off else { return false } - guard let last = reader.lastFetchedAt else { return true } - switch reader.schedule { - case .off: - return false - case .hourly: - return Date().timeIntervalSince(last) >= 3_600 - case .daily: - return Date().timeIntervalSince(last) >= 86_400 - } - } -} diff --git a/ui/ui/News/NewsReaderSummarizer.swift b/ui/ui/News/NewsReaderSummarizer.swift deleted file mode 100644 index e0b0404..0000000 --- a/ui/ui/News/NewsReaderSummarizer.swift +++ /dev/null @@ -1,67 +0,0 @@ -import Foundation -import LLMAgentClient -import Structure - -struct NewsReaderSummarizer: NewsSummaryGenerating { - let settings: LLMModelSettings - - func summarize(listName: String, topics: [String], articles: [NewsItem]) async throws -> String { - let model = await MainActor.run { settings.summarizerModel } - guard let apiKey = await LLMProviderCredentialGate.resolveAPIKey(for: model) else { - throw NewsReaderError.summarizerUnavailable - } - - let prompt = Self.prompt(listName: listName, topics: topics, articles: articles) - let request = AgentRequest.prompt( - prompt, - system: """ - You summarize news for the user. Write clear prose in plain English. - Use bullet points. Include markdown links to the original articles using the URLs provided. - Do not add a title or markdown heading for the list name. - Cover every article provided when possible. - Do not invent stories or URLs. - """, - temperature: 0.2 - ) - let text: String - switch model { - case .gemini(let geminiModel): - let client = GeminiAgentClient(provider: GeminiProvider(apiKey: apiKey)) - text = try await collect(client.stream(request, model: geminiModel)) - case .openai(let openAIModel): - let client = OpenAIAgentClient(provider: OpenAIProvider(apiKey: apiKey)) - text = try await collect(client.stream(request, model: openAIModel)) - } - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { - throw NewsReaderError.fetchFailed(url: listName, detail: "Summarizer returned an empty response.") - } - return trimmed - } - - private func collect(_ stream: AsyncThrowingStream) async throws -> String { - let (text, usage) = try await collectAgentStream(stream) - if let usage { - _ = await UsageLimitsService.shared.recordAPIUsage(usage) - } - return text - } - - private static func prompt(listName: String, topics: [String], articles: [NewsItem]) -> String { - var lines = [ - "Summarize these articles for the list \"\(listName)\".", - ] - if !topics.isEmpty { - lines.append("Topics: \(topics.joined(separator: ", "))") - } - lines.append("Articles:") - for article in articles.prefix(12) { - var entry = "- \(article.title) (\(article.sourceURL))" - if let detail = article.summary, !detail.isEmpty { - entry += "\n \(detail)" - } - lines.append(entry) - } - return lines.joined(separator: "\n") - } -} diff --git a/ui/ui/News/NewsWorkspaceView.swift b/ui/ui/News/NewsWorkspaceView.swift deleted file mode 100644 index 8228570..0000000 --- a/ui/ui/News/NewsWorkspaceView.swift +++ /dev/null @@ -1,171 +0,0 @@ -import SwiftUI -import Structure - -struct NewsWorkspaceView: View { - @ObservedObject var store: NewsReaderStore - - var body: some View { - VStack(spacing: 0) { - header - Divider() - if store.selectedReader == nil { - emptyState - } else { - itemList - } - } - .background(Color(red: 252.0 / 255.0, green: 252.0 / 255.0, blue: 250.0 / 255.0)) - } - - private var header: some View { - HStack(spacing: 12) { - VStack(alignment: .leading, spacing: 2) { - Text(store.selectedReader?.name ?? "News") - .font(.headline) - if let reader = store.selectedReader { - Text(headerSubtitle(reader)) - .font(.caption) - .foregroundStyle(.secondary) - } - } - Spacer() - if store.selectedReader != nil { - Button { - Task { await store.refreshSelected() } - } label: { - if store.isRefreshing { - ProgressView() - .controlSize(.small) - } else { - Text("Refresh") - } - } - .disabled(store.isRefreshing) - .buttonStyle(.bordered) - } - } - .padding(.horizontal, 24) - .padding(.vertical, 14) - } - - private var emptyState: some View { - VStack(spacing: 10) { - Spacer() - Image(systemName: "newspaper") - .font(.system(size: 36)) - .foregroundStyle(.secondary) - Text("No news lists yet") - .font(.title3.weight(.semibold)) - Text("Create a news list from Plugins. Derrick fetches articles in Docker, then shows them here.") - .font(.subheadline) - .foregroundStyle(.secondary) - .multilineTextAlignment(.center) - .frame(maxWidth: 420) - Spacer() - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - } - - private var itemList: some View { - ScrollView { - VStack(alignment: .leading, spacing: 16) { - if let error = store.selectedReader?.lastError ?? store.lastError, !error.isEmpty { - Text(error) - .font(.caption) - .foregroundStyle(.red) - .padding(12) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.red.opacity(0.08), in: RoundedRectangle(cornerRadius: 10)) - } - - if store.selectedReader?.mode == .summary, - let summary = store.selectedReader?.summaryText, - !summary.isEmpty { - VStack(alignment: .leading, spacing: 8) { - Text("AI summary") - .font(.subheadline.weight(.semibold)) - Text("A digest of the articles below.") - .font(.caption) - .foregroundStyle(.secondary) - MarkdownResponseView(text: summary, allowsCSVExport: false) - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) - } - - if store.selectedReader?.mode != .summary { - ForEach(store.items) { item in - articleCard(item) - } - } else if !store.items.isEmpty { - VStack(alignment: .leading, spacing: 8) { - Text("Articles (\(store.items.count))") - .font(.subheadline.weight(.semibold)) - Text("Stories fetched from your feeds. The header feed count is how many RSS sources are configured, not how many articles appear here.") - .font(.caption) - .foregroundStyle(.secondary) - ForEach(store.items) { item in - HStack(alignment: .top, spacing: 8) { - Text("•") - VStack(alignment: .leading, spacing: 2) { - Link(item.title, destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) - .font(.caption.weight(.semibold)) - Text(item.sourceLabel) - .font(.caption2) - .foregroundStyle(.secondary) - } - } - } - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) - } - - if store.items.isEmpty, store.lastError == nil, store.selectedReader?.lastError == nil { - Text("No articles yet. Refresh this list.") - .font(.subheadline) - .foregroundStyle(.secondary) - } - } - .padding(24) - } - } - - @ViewBuilder - private func articleCard(_ item: NewsItem) -> some View { - VStack(alignment: .leading, spacing: 6) { - Link(destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) { - Text(item.title) - .font(.body.weight(.semibold)) - .multilineTextAlignment(.leading) - } - HStack(spacing: 8) { - Text(item.sourceLabel) - .font(.caption) - .foregroundStyle(.secondary) - if let host = URL(string: item.sourceURL)?.host { - Link(host, destination: URL(string: item.sourceURL) ?? URL(string: "https://example.com")!) - .font(.caption.weight(.semibold)) - .lineLimit(1) - } - } - if let summary = item.summary, !summary.isEmpty { - Text(summary) - .font(.caption) - .foregroundStyle(.secondary) - .lineLimit(4) - } - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.white, in: RoundedRectangle(cornerRadius: 12)) - } - - private func headerSubtitle(_ reader: NewsReaderSpec) -> String { - let topics = reader.topics.isEmpty ? "All topics" : reader.topics.joined(separator: ", ") - let sources = "\(reader.sources.count) source\(reader.sources.count == 1 ? "" : "s")" - return "\(topics) · \(sources) · \(reader.mode.displayName) · \(reader.schedule.displayName)" - } -} diff --git a/ui/ui/Plugins/PluginCreationController.swift b/ui/ui/Plugins/PluginCreationController.swift index fb65491..c4228f2 100644 --- a/ui/ui/Plugins/PluginCreationController.swift +++ b/ui/ui/Plugins/PluginCreationController.swift @@ -20,7 +20,6 @@ final class PluginCreationController: ObservableObject { enum SuccessOutcome: Equatable { case plugin - case newsList } struct ProgressStepState: Identifiable, Equatable { @@ -70,18 +69,12 @@ final class PluginCreationController: ObservableObject { !$0.userSays.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && !$0.pluginDoes.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }) else { return false } - if skillDraft.plannedKind == .newsDigest { - return !skillDraft.newsTopics.isEmpty && !skillDraft.newsSourceURLs.isEmpty - } return skillDraft.buildBlockedReason == nil } var canConfirmPluginName: Bool { let trimmed = skillDraft.pluginName.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return false } - if skillDraft.plannedKind == .newsDigest { - return true - } return (try? PluginID.normalized(trimmed)) != nil } @@ -188,48 +181,6 @@ final class PluginCreationController: ObservableObject { } } - func addNewsTopic() { - let topic = skillDraft.goal.trimmingCharacters(in: .whitespacesAndNewlines) - guard !topic.isEmpty else { return } - mutateSkillDraft { draft in - if !draft.newsTopics.contains(where: { $0.compare(topic, options: .caseInsensitive) == .orderedSame }) { - draft.newsTopics.append(topic) - } - } - } - - func removeNewsTopic(_ topic: String) { - mutateSkillDraft { $0.newsTopics.removeAll { $0 == topic } } - } - - func addNewsTopicFromPreset(_ topic: String) { - let trimmed = topic.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return } - mutateSkillDraft { draft in - if !draft.newsTopics.contains(trimmed) { - draft.newsTopics.append(trimmed) - } - } - } - - func addNewsURL(_ raw: String) { - let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return } - let normalized = NewsSourceURL.canonicalFetchURL( - URL(string: trimmed.contains("://") ? trimmed : "https://\(trimmed)") - ?? URL(string: "https://news.google.com/rss")! - ).absoluteString - mutateSkillDraft { draft in - if !draft.newsSourceURLs.contains(normalized) { - draft.newsSourceURLs.append(normalized) - } - } - } - - func removeNewsURL(_ url: String) { - mutateSkillDraft { $0.newsSourceURLs.removeAll { $0 == url } } - } - private func mutateSkillDraft(_ transform: (inout PluginSkillDraft) -> Void) { var draft = skillDraft transform(&draft) @@ -245,11 +196,6 @@ final class PluginCreationController: ObservableObject { creationAPIKey = helperAPIKey creationReviewerModelJSON = helperReviewerModelJSON - if skillDraft.plannedKind == .newsDigest { - startNewsCreation() - return - } - if skillDraft.plannedKind == .messagingConnector { phase = .discoveringAuth statusMessage = "Reading how this service authenticates…" @@ -284,7 +230,7 @@ final class PluginCreationController: ObservableObject { case .failed(let step, _, _): switch step { case .goal: phase = .goal - case .skill, .news: phase = .skill + case .skill: phase = .skill case .preview: phase = .preview case .credentials: phase = .preview case .build: phase = .preview @@ -298,66 +244,6 @@ final class PluginCreationController: ObservableObject { showIntro() } - private func startNewsCreation() { - let sources = skillDraft.newsSourceURLs.map { url in - NewsSource(label: URL(string: url)?.host ?? url, url: url) - } - let spec = NewsReaderSpec( - name: skillDraft.pluginName, - topics: skillDraft.newsTopics, - sources: sources, - mode: PluginSkillDraftPlanner.inferNewsMode(from: skillDraft), - maxCount: 20, - schedule: .off - ) - if let blocked = spec.sources.compactMap({ source -> (NewsSource, String)? in - guard let url = URL(string: source.url), - let reason = NewsPaywall.preflightRejection(url: url) else { return nil } - return (source, reason) - }).first { - phase = .failed( - step: .news, - message: NewsReaderError.paywalled(url: blocked.0.url, detail: blocked.1).errorDescription - ?? "This source is behind a paywall, which is not supported yet." - ) - return - } - phase = .creating - statusMessage = "Starting news reader in Docker…" - progressSteps = [ - ProgressStepState(id: "skill", title: "Write SKILL.md", status: .completed), - ProgressStepState(id: "sources", title: "Check sources", status: .active), - ProgressStepState(id: "fetch", title: "Run news reader", status: .pending), - ] - if spec.mode == .summary { - progressSteps.append( - ProgressStepState(id: "summary", title: "Summarize with AI", status: .pending) - ) - } - pollTask?.cancel() - pollTask = Task { @MainActor in - do { - let saved = try await NewsReaderStore.shared.create(spec) - setProgressStep("sources", status: .completed) - setProgressStep("fetch", status: .completed) - if spec.mode == .summary { - setProgressStep("summary", status: .completed) - } - phase = .succeeded(pluginID: saved.id, outcome: .newsList) - } catch let error as NewsReaderError { - setProgressStep("sources", status: .failed) - phase = .failed( - step: .news, - message: error.localizedDescription, - technicalDetail: String(describing: error) - ) - } catch { - setProgressStep("sources", status: .failed) - phase = .failed(step: .news, message: error.localizedDescription) - } - } - } - private func startFactoryCreation() { guard let creationAPIKey, !creationAPIKey.isEmpty else { phase = .failed( diff --git a/ui/ui/Plugins/PluginsWorkspaceView.swift b/ui/ui/Plugins/PluginsWorkspaceView.swift index 11467e6..a3e8a3b 100644 --- a/ui/ui/Plugins/PluginsWorkspaceView.swift +++ b/ui/ui/Plugins/PluginsWorkspaceView.swift @@ -8,10 +8,6 @@ struct PluginsWorkspaceView: View { let helperReviewerModelJSON: String? let sessionID: String let onOpenMessagingConnector: (String) -> Void - var onOpenNewsReader: (String) -> Void = { _ in } - - @State private var newsTopicDraft = "" - @State private var newsURLDraft = "" var body: some View { ZStack { @@ -85,22 +81,18 @@ struct PluginsWorkspaceView: View { case .collectCredentials: return "Plugin credentials" case .failed: return failureTitle case .succeeded(_, let outcome): - return outcome == .newsList ? "News list ready" : "Plugin ready" + return "Plugin ready" } } private var creatingTitle: String { switch controller.skillDraft.plannedKind { - case .newsDigest: return "Creating news list" case .messagingConnector: return "Creating connector" case .customCapability: return "Building plugin" } } private var failureTitle: String { - if case .failed(let step, _, _) = controller.phase, step == .news { - return "Could not create news list" - } return "Could not create plugin" } @@ -252,12 +244,6 @@ struct PluginsWorkspaceView: View { } Spacer() switch outcome { - case .newsList: - Button("Open news list") { - onOpenNewsReader(pluginID) - } - .buttonStyle(ModalPrimaryButtonStyle()) - .keyboardShortcut(.defaultAction) case .plugin where controller.skillDraft.plannedKind == .messagingConnector: Button("Open connector") { onOpenMessagingConnector(pluginID) @@ -275,10 +261,7 @@ struct PluginsWorkspaceView: View { } private var buildButtonTitle: String { - switch controller.skillDraft.plannedKind { - case .newsDigest: return "Create" - default: return "Build plugin" - } + "Build plugin" } private var skillBuilderForm: some View { @@ -308,10 +291,6 @@ struct PluginsWorkspaceView: View { examplesSection - if controller.skillDraft.plannedKind == .newsDigest { - newsFieldsSection - } - nameFieldSection if let blocked = controller.skillDraft.buildBlockedReason { @@ -339,8 +318,6 @@ struct PluginsWorkspaceView: View { case .messagingConnector: let vendor = controller.skillDraft.inferredConnectorVendor?.displayName ?? "Messaging" return ("\(vendor) connector", "bubble.left.and.bubble.right") - case .newsDigest: - return ("News list", "newspaper") case .customCapability: return ("Custom capability", "wand.and.stars") } @@ -395,98 +372,6 @@ struct PluginsWorkspaceView: View { ) } - private var newsFieldsSection: some View { - VStack(alignment: .leading, spacing: 12) { - Text("Topics") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 110), spacing: 8)], spacing: 8) { - ForEach(NewsPresetTopic.allCases) { topic in - let on = controller.skillDraft.newsTopics.contains(topic.displayName) - Button { - if on { - controller.removeNewsTopic(topic.displayName) - } else if !controller.skillDraft.newsTopics.contains(topic.displayName) { - controller.addNewsTopicFromPreset(topic.displayName) - } - } label: { - Text(topic.displayName) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("Add a custom topic", text: $newsTopicDraft) - .textFieldStyle(.roundedBorder) - .onSubmit { addNewsTopicFromDraft() } - Button("Add") { addNewsTopicFromDraft() } - } - Text("Sources") - .font(.caption) - .foregroundStyle(.secondary) - LazyVGrid(columns: [GridItem(.adaptive(minimum: 130), spacing: 8)], spacing: 8) { - ForEach(NewsPresetSource.allCases) { source in - let url = source.source.url - let on = controller.skillDraft.newsSourceURLs.contains(url) - Button { - if on { - controller.removeNewsURL(url) - } else { - controller.addNewsURL(url) - } - } label: { - Text(source.source.label) - .font(.caption.weight(.medium)) - .frame(maxWidth: .infinity) - .padding(.vertical, 8) - .background(on ? Color.accentColor.opacity(0.15) : Color.primary.opacity(0.05)) - .clipShape(RoundedRectangle(cornerRadius: 8)) - } - .buttonStyle(.plain) - } - } - HStack { - TextField("https://…", text: $newsURLDraft) - .textFieldStyle(.roundedBorder) - .accessibilityIdentifier("news-url-field") - .onSubmit { addNewsURLFromDraft() } - Button("Add URL") { addNewsURLFromDraft() } - } - ForEach(controller.skillDraft.newsSourceURLs, id: \.self) { url in - HStack { - Text(url) - .font(.caption) - .lineLimit(1) - Spacer() - Button("Remove") { controller.removeNewsURL(url) } - .font(.caption) - } - } - HStack(alignment: .top, spacing: 8) { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundStyle(.orange) - Text(NewsPaywall.userWarning) - .fixedSize(horizontal: false, vertical: true) - } - .font(.caption) - .padding(10) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.orange.opacity(0.12), in: RoundedRectangle(cornerRadius: 8)) - } - } - - private func addNewsTopicFromDraft() { - let topic = newsTopicDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !topic.isEmpty else { return } - controller.addNewsTopicFromPreset(topic) - newsTopicDraft = "" - } - private func triggerChip(_ trigger: PluginSkillDraft.Trigger) -> some View { let available = controller.skillDraft.isTriggerAvailable(trigger) let on = available && controller.skillDraft.triggers.contains(trigger) @@ -517,15 +402,6 @@ struct PluginsWorkspaceView: View { ) -> String { guard !available else { return "" } switch controller.skillDraft.plannedKind { - case .newsDigest: - switch trigger { - case .mention: - return "News lists live in the sidebar, not as /slash commands." - case .messaging: - return "News lists are not messaging connectors." - default: - return "Not available for news lists." - } case .messagingConnector: switch trigger { case .schedule: @@ -544,19 +420,14 @@ struct PluginsWorkspaceView: View { } private var nameFieldSection: some View { - let isNews = controller.skillDraft.plannedKind == .newsDigest - return VStack(alignment: .leading, spacing: 6) { - Text(isNews ? "List name" : "Plugin name") + VStack(alignment: .leading, spacing: 6) { + Text("Plugin name") .font(.caption) .foregroundStyle(.secondary) - TextField(isNews ? "List name" : "Plugin name", text: controller.skillDraftBinding(\.pluginName)) + TextField("Plugin name", text: controller.skillDraftBinding(\.pluginName)) .textFieldStyle(.roundedBorder) .accessibilityIdentifier("connector-plugin-name") - if isNews { - Text("This name appears in the sidebar. News lists are not slash commands.") - .font(.caption2) - .foregroundStyle(.tertiary) - } else if controller.canConfirmPluginName { + if controller.canConfirmPluginName { Text("Invoke this plugin in chat as /\(normalizedPluginID()).") .font(.caption2) .foregroundStyle(.tertiary) @@ -576,11 +447,6 @@ struct PluginsWorkspaceView: View { return trimmed.isEmpty ? "plugin" : trimmed } - private func addNewsURLFromDraft() { - controller.addNewsURL(newsURLDraft) - newsURLDraft = "" - } - private var previewForm: some View { ScrollView { VStack(alignment: .leading, spacing: 16) { @@ -588,14 +454,6 @@ struct PluginsWorkspaceView: View { nameFieldSection - if controller.skillDraft.plannedKind == .newsDigest { - Text( - "Display mode: \(PluginSkillDraftPlanner.inferNewsMode(from: controller.skillDraft).displayName)" - ) - .font(.caption) - .foregroundStyle(.secondary) - } - VStack(alignment: .leading, spacing: 6) { Text("Scenarios") .font(.caption) @@ -637,23 +495,12 @@ struct PluginsWorkspaceView: View { @ViewBuilder private func failureBody(message: String, technicalDetail: String?) -> some View { VStack(alignment: .leading, spacing: 10) { - if case .failed(let step, _, _) = controller.phase, step == .news { - Label( - message.localizedCaseInsensitiveContains("paywall") - ? "Blocked because of a paywall" - : "News list was not created", - systemImage: "exclamationmark.triangle.fill" - ) + Label("Nothing was installed", systemImage: "minus.circle") .font(.subheadline.weight(.semibold)) - .foregroundStyle(message.localizedCaseInsensitiveContains("paywall") ? Color.orange : Color.secondary) - } else { - Label("Nothing was installed", systemImage: "minus.circle") - .font(.subheadline.weight(.semibold)) - .foregroundStyle(.secondary) - Text("Your sidebar and Messaging are unchanged.") - .font(.caption) - .foregroundStyle(.secondary) - } + .foregroundStyle(.secondary) + Text("Your sidebar and Messaging are unchanged.") + .font(.caption) + .foregroundStyle(.secondary) Text(message) .font(.body) .fixedSize(horizontal: false, vertical: true) @@ -673,10 +520,6 @@ struct PluginsWorkspaceView: View { @ViewBuilder private func successBody(pluginID: String, outcome: PluginCreationController.SuccessOutcome) -> some View { switch outcome { - case .newsList: - Text("Your news list is ready. Every article includes a source link.") - .font(.body) - .fixedSize(horizontal: false, vertical: true) case .plugin: if controller.skillDraft.plannedKind == .messagingConnector { Text("Your connector /\(pluginID) is ready. Open it to start talking in Messaging.") diff --git a/ui/ui/Views/ContentView.swift b/ui/ui/Views/ContentView.swift index 9254ea3..46013e3 100644 --- a/ui/ui/Views/ContentView.swift +++ b/ui/ui/Views/ContentView.swift @@ -234,7 +234,6 @@ struct ContentView: View { @ObservedObject private var bootstrapStatus = AppBootstrapStatus.shared @StateObject private var chatSessions = ChatSessionStore() @StateObject private var messaging = MessagingStore() - @ObservedObject private var news = NewsReaderStore.shared @State private var workspace: AppWorkspace = .chats private var secretStore: SecretStore { @@ -389,7 +388,6 @@ struct ContentView: View { modelThinkingSettings: modelThinkingSettings ?? LLMModelThinkingSettings(repository: helperModelSettings.settingsRepository), chatSessions: chatSessions, messaging: messaging, - news: news, workspace: $workspace, isDebugEnabled: isDebugEnabled ) @@ -403,14 +401,12 @@ struct ContentView: View { VStack(spacing: 0) { if workspace == .messaging { MessagingTabBarView(store: messaging) - } else if workspace != .debugLogs && workspace != .plugins && workspace != .news { + } else if workspace != .debugLogs && workspace != .plugins { ChatTabBarView(store: chatSessions) } switch workspace { case .messaging: MessagingConversationView(store: messaging) - case .news: - NewsWorkspaceView(store: news) case .debugLogs: DebugLogsView(repository: repository) case .plugins: @@ -428,13 +424,6 @@ struct ContentView: View { } Task { await pluginFactoryList.reload() } } - }, - onOpenNewsReader: { readerID in - Task { @MainActor in - workspace = .news - await news.select(id: readerID) - pluginCreationController.dismissSuccess() - } } ) default: @@ -846,10 +835,6 @@ struct ContentView: View { await ContainerLifecycleSettingsService.shared.configure(repository: repository) await OrchestrationLimitsSettingsService.shared.configure(repository: repository) await PluginFactoryListStore.shared.configure(repository: repository) - await NewsReaderStore.shared.configure( - repository: repository, - summarizerSettings: helperModelSettings - ) await AgentProfileStore.shared.configure(repository: repository) pluginCreationController.configure(repository: repository) } diff --git a/ui/ui/Views/SidebarView.swift b/ui/ui/Views/SidebarView.swift index 20b340f..b719f13 100644 --- a/ui/ui/Views/SidebarView.swift +++ b/ui/ui/Views/SidebarView.swift @@ -10,7 +10,6 @@ struct SidebarView: View { @ObservedObject var modelThinkingSettings: LLMModelThinkingSettings @ObservedObject var chatSessions: ChatSessionStore @ObservedObject var messaging: MessagingStore - @ObservedObject var news: NewsReaderStore @Binding var workspace: AppWorkspace var isDebugEnabled: Bool = false /// Reference type must not be recreated every `View` value; hold via `@State`. @@ -81,17 +80,6 @@ struct SidebarView: View { workspace = .messaging Task { await messaging.syncConnectorsFromFactory() } } - SidebarActionRow( - row: SidebarRow( - id: "news", - icon: "newspaper.fill", - title: "News", - isProminent: workspace == .news - ) - ) { - workspace = .news - Task { await news.reload() } - } if isDebugEnabled { SidebarActionRow( row: SidebarRow( @@ -110,8 +98,6 @@ struct SidebarView: View { pluginsList } else if workspace == .messaging { messagingList - } else if workspace == .news { - newsList } else if workspace == .debugLogs { debugLogsHint } else { @@ -289,46 +275,6 @@ struct SidebarView: View { } } - private var newsList: some View { - VStack(alignment: .leading, spacing: 8) { - HStack { - Text("Saved lists") - .font(.caption) - .foregroundStyle(.secondary) - Spacer() - } - .padding(.top, 4) - ScrollView { - LazyVStack(alignment: .leading, spacing: 10) { - if news.readers.isEmpty { - Text("No news lists yet") - .font(.system(size: sideMenuRecentsFontSize)) - .foregroundStyle(.secondary) - } else { - ForEach(news.readers) { reader in - Button { - workspace = .news - Task { await news.select(id: reader.id) } - } label: { - Text(reader.name) - .font(.system(size: sideMenuRecentsFontSize)) - .lineLimit(1) - .frame(maxWidth: .infinity, alignment: .leading) - .foregroundStyle( - news.selectedReaderID == reader.id - ? Color.primary - : Color.primary.opacity(0.9) - ) - } - .buttonStyle(.plain) - } - } - } - .frame(maxWidth: .infinity, alignment: .leading) - } - } - } - private var pluginsList: some View { VStack(alignment: .leading, spacing: 8) { ScrollView { @@ -439,7 +385,6 @@ struct SidebarView: View { modelThinkingSettings: LLMModelThinkingSettings(repository: repo), chatSessions: store, messaging: MessagingStore(), - news: NewsReaderStore.shared, workspace: .constant(.chats) ) } diff --git a/ui/uiTests/NewsReaderFlowTests.swift b/ui/uiTests/NewsReaderFlowTests.swift deleted file mode 100644 index 6ee5364..0000000 --- a/ui/uiTests/NewsReaderFlowTests.swift +++ /dev/null @@ -1,58 +0,0 @@ -import Foundation -import Testing -import Structure - -@Suite struct NewsReaderFlowTests { - @Test func skillDraftForTechNewsSummaryRequest() { - var draft = PluginSkillDraft() - PluginSkillDraftPlanner.applyGoal( - "Give me summaries of today's tech news", - to: &draft, - existingPluginIDs: [] - ) - draft.pluginName = "tech-news" - draft.newsSourceURLs = [ - NewsPresetSource.googleNews.source.url, - NewsPresetSource.wsj.source.url, - ] - draft.newsTopics = ["Tech"] - draft.examples = [ - PluginSkillDraft.Example( - userSays: "Give me summaries of tech news", - pluginDoes: "fetch headlines and summarize them with source links" - ), - ] - - #expect(draft.plannedKind == .newsDigest) - #expect(PluginSkillDraftPlanner.inferNewsMode(from: draft) == .summary) - #expect(draft.pluginName == "tech-news") - #expect(draft.newsSourceURLs.contains(NewsPresetSource.googleNews.source.url)) - #expect(draft.newsSourceURLs.contains(NewsPresetSource.wsj.source.url)) - } - - @Test func newsReaderWorkerRequestMatchesWizardSpec() throws { - let sources = [ - NewsPresetSource.googleNews.source, - NewsPresetSource.wsj.source, - ] - let spec = NewsReaderSpec( - name: "tech-news", - topics: ["Tech"], - sources: sources, - mode: .summary, - maxCount: 20, - schedule: .off - ) - let request = NewsReaderWorkerRequest( - mode: spec.mode, - sources: spec.sources, - topics: spec.topics, - maxCount: spec.maxCount, - contextHint: spec.name - ) - let json = try request.encodedJSON() - let object = try JSONSerialization.jsonObject(with: json) as? [String: Any] - #expect(object?["mode"] as? String == "summary") - #expect((object?["sources"] as? [[String: Any]])?.count == 2) - } -} diff --git a/workers/go/cmd/derrick-news-reader/main.go b/workers/go/cmd/derrick-news-reader/main.go deleted file mode 100644 index a078c1d..0000000 --- a/workers/go/cmd/derrick-news-reader/main.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "io" - "os" - "strconv" - - "github.com/jsoneaday/derrick/workers/internal/contract" - "github.com/jsoneaday/derrick/workers/internal/newsreader" -) - -func main() { - input, err := io.ReadAll(os.Stdin) - if err != nil { - writeResult(blockedResult(newsreader.ModeRSS, "News reader input must be a valid JSON object.")) - return - } - - var req newsreader.Request - if err := json.Unmarshal(input, &req); err != nil { - writeResult(blockedResult(newsreader.ModeRSS, "News reader input must be a valid JSON object.")) - return - } - - validated, err := newsreader.Validate(req) - if err != nil { - writeResult(blockedResult(req.Mode, err.Error())) - return - } - - result := newsreader.Run(context.Background(), validated, readProxy()) - writeResult(result) -} - -func readProxy() *newsreader.ProxyConfig { - host := os.Getenv("DERRICK_EGRESS_PROXY_HOST") - portStr := os.Getenv("DERRICK_EGRESS_PROXY_PORT") - token := os.Getenv("DERRICK_EGRESS_PROXY_TOKEN") - if host == "" && portStr == "" && token == "" { - return nil - } - port, _ := strconv.Atoi(portStr) - return &newsreader.ProxyConfig{Host: host, Port: port, Token: token} -} - -func blockedResult(mode newsreader.Mode, message string) newsreader.Result { - if mode == "" { - mode = newsreader.ModeRSS - } - return newsreader.Result{ - OK: false, - Mode: mode, - Articles: []newsreader.Article{}, - Diagnostics: []string{message}, - } -} - -func writeResult(result newsreader.Result) { - payload, err := encodedResult(result) - if err != nil { - writeEncodedResult(blockedResult(result.Mode, "News reader output violated worker contract.")) - return - } - _, _ = os.Stdout.Write(payload) -} - -func encodedResult(result newsreader.Result) ([]byte, error) { - if result.Articles == nil { - result.Articles = []newsreader.Article{} - } - if result.Diagnostics == nil { - result.Diagnostics = []string{} - } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - if err := enc.Encode(result); err != nil { - return nil, err - } - data := bytes.TrimSpace(buf.Bytes()) - if err := contract.ValidateNewsReaderResultJSON(data); err != nil { - return nil, err - } - return append(data, '\n'), nil -} - -func writeEncodedResult(result newsreader.Result) { - payload, err := encodedResult(result) - if err != nil { - fallback := blockedResult(newsreader.ModeRSS, "News reader failed to produce schema-compliant output.") - payload, _ = encodedResult(fallback) - } - _, _ = os.Stdout.Write(payload) -} diff --git a/workers/go/internal/contract/contract.go b/workers/go/internal/contract/contract.go index e36ecc3..ba341c0 100644 --- a/workers/go/internal/contract/contract.go +++ b/workers/go/internal/contract/contract.go @@ -34,11 +34,6 @@ func ValidateFileExtractorResultJSON(data []byte) error { return validate("file-extractor-result.schema.json", data) } -// ValidateNewsReaderResultJSON checks news reader stdout against news-reader-result.schema.json. -func ValidateNewsReaderResultJSON(data []byte) error { - return validate("news-reader-result.schema.json", data) -} - func validate(schemaName string, data []byte) error { compiler := jsonschema.NewCompiler() if err := loadSchemas(compiler); err != nil { diff --git a/workers/go/internal/contract/schemas/news-reader-result.schema.json b/workers/go/internal/contract/schemas/news-reader-result.schema.json deleted file mode 100644 index 6632851..0000000 --- a/workers/go/internal/contract/schemas/news-reader-result.schema.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://derrick.local/schemas/news-reader-result.json", - "title": "News reader worker stdout", - "description": "JSON object written to stdout by derrick-news-reader.", - "$ref": "worker-product.schema.json#/$defs/news_reader_result" -} diff --git a/workers/go/internal/contract/schemas/worker-product.schema.json b/workers/go/internal/contract/schemas/worker-product.schema.json index 611ef5f..874f6bf 100644 --- a/workers/go/internal/contract/schemas/worker-product.schema.json +++ b/workers/go/internal/contract/schemas/worker-product.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://derrick.local/schemas/worker-product.json", "title": "Derrick trusted worker product contracts", - "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler, file extractor, and news reader). Swift host and Go workers must match these schemas.", + "description": "Shared stdout JSON contracts for prebuilt Go worker binaries (web crawler and file extractor). Swift host and Go workers must match these schemas.", "$defs": { "string_list": { "type": "array", @@ -92,35 +92,6 @@ }, "diagnostics": { "$ref": "#/$defs/string_list" } } - }, - "news_reader_mode": { - "type": "string", - "enum": ["rss", "list", "summary"] - }, - "news_reader_article": { - "type": "object", - "required": ["title", "url"], - "additionalProperties": false, - "properties": { - "title": { "type": "string" }, - "url": { "type": "string" }, - "detail": { "type": "string" }, - "published_at": { "type": "string" } - } - }, - "news_reader_result": { - "type": "object", - "required": ["ok", "mode", "articles", "diagnostics"], - "additionalProperties": false, - "properties": { - "ok": { "type": "boolean" }, - "mode": { "$ref": "#/$defs/news_reader_mode" }, - "articles": { - "type": "array", - "items": { "$ref": "#/$defs/news_reader_article" } - }, - "diagnostics": { "$ref": "#/$defs/string_list" } - } } } } diff --git a/workers/go/internal/newsreader/engine.go b/workers/go/internal/newsreader/engine.go deleted file mode 100644 index 74e6081..0000000 --- a/workers/go/internal/newsreader/engine.go +++ /dev/null @@ -1,34 +0,0 @@ -package newsreader - -import ( - "context" -) - -func Run(ctx context.Context, req Request, proxy *ProxyConfig) Result { - client := newHTTPClient(proxy) - collected := make([]Article, 0, req.MaxCount) - diagnostics := make([]string, 0) - - for _, source := range req.Sources { - articles, notes := fetchSource(ctx, client, source, req.Mode, req.ContextHint) - diagnostics = append(diagnostics, notes...) - collected = append(collected, articles...) - } - - filtered := filterTopics(collected, req.Topics) - unique := uniqueArticles(filtered) - if len(unique) > req.MaxCount { - unique = unique[:req.MaxCount] - } - - ok := len(unique) > 0 - if !ok && len(diagnostics) == 0 { - diagnostics = append(diagnostics, "No articles were found for the configured sources.") - } - return Result{ - OK: ok, - Mode: req.Mode, - Articles: unique, - Diagnostics: diagnostics, - } -} diff --git a/workers/go/internal/newsreader/fetch.go b/workers/go/internal/newsreader/fetch.go deleted file mode 100644 index 7c2b047..0000000 --- a/workers/go/internal/newsreader/fetch.go +++ /dev/null @@ -1,127 +0,0 @@ -package newsreader - -import ( - "context" - "fmt" - "io" - "net/http" - "strings" - "time" -) - -func fetchSource( - ctx context.Context, - client *http.Client, - source Source, - mode Mode, - contextHint string, -) ([]Article, []string) { - fetchURL := CanonicalFetchURL(source.URL, contextHint) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL, nil) - if err != nil { - return nil, []string{fmt.Sprintf("%s: %v", source.Label, err)} - } - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; DerrickNewsReader/1.0)") - req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.8") - - resp, err := client.Do(req) - if err != nil { - return nil, []string{fmt.Sprintf("Could not read %s: %v", source.URL, err)} - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return nil, []string{fmt.Sprintf("Could not read %s: HTTP %d", source.URL, resp.StatusCode)} - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) - if err != nil { - return nil, []string{fmt.Sprintf("Could not read %s: %v", source.URL, err)} - } - text := string(body) - if looksLikeFeed(text) { - articles := parseFeed(text, source.Label) - if len(articles) > 0 { - return articles, nil - } - } - if mode == ModeRSS { - articles := parseFeed(text, source.Label) - if len(articles) > 0 { - return articles, nil - } - return nil, []string{fmt.Sprintf("No articles were found in %s", source.URL)} - } - if mode == ModeList || mode == ModeSummary { - articles := parseHTMLArticles(text, fetchURL, source.Label) - if len(articles) > 0 { - return articles, nil - } - return nil, []string{fmt.Sprintf("No articles were found on %s", source.URL)} - } - return nil, []string{fmt.Sprintf("No articles were found in %s", source.URL)} -} - -func newHTTPClient(proxy *ProxyConfig) *http.Client { - transport := http.DefaultTransport.(*http.Transport).Clone() - return &http.Client{ - Timeout: time.Duration(DefaultTimeoutSeconds) * time.Second, - Transport: transport, - } -} - -func filterTopics(articles []Article, topics []string) []Article { - needles := make([]string, 0, len(topics)) - for _, topic := range topics { - trimmed := strings.ToLower(strings.TrimSpace(topic)) - if trimmed != "" { - needles = append(needles, trimmed) - } - } - if len(needles) == 0 { - return articles - } - matched := make([]Article, 0, len(articles)) - unmatched := make([]Article, 0, len(articles)) - for _, article := range articles { - hay := strings.ToLower(article.Title + " " + article.Detail) - if topicMatches(hay, needles) { - matched = append(matched, article) - } else { - unmatched = append(unmatched, article) - } - } - if len(matched) == 0 { - return articles - } - // Prefer topic matches, but keep other articles when filtering would drop too many - // (for example WSJ headlines that do not literally contain "tech"). - if len(matched) < 3 && len(unmatched) > 0 { - out := make([]Article, 0, len(articles)) - out = append(out, matched...) - out = append(out, unmatched...) - return out - } - return matched -} - -func topicMatches(hay string, needles []string) bool { - for _, needle := range needles { - if strings.Contains(hay, needle) { - return true - } - } - return false -} - -func uniqueArticles(articles []Article) []Article { - seen := map[string]bool{} - out := make([]Article, 0, len(articles)) - for _, article := range articles { - if seen[article.URL] { - continue - } - seen[article.URL] = true - out = append(out, article) - } - return out -} diff --git a/workers/go/internal/newsreader/list.go b/workers/go/internal/newsreader/list.go deleted file mode 100644 index 9d3c373..0000000 --- a/workers/go/internal/newsreader/list.go +++ /dev/null @@ -1,88 +0,0 @@ -package newsreader - -import ( - "net/url" - "strings" - - "github.com/PuerkitoBio/goquery" -) - -func parseHTMLArticles(body string, pageURL string, sourceLabel string) []Article { - doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) - if err != nil { - return nil - } - articles := make([]Article, 0, 32) - seen := map[string]bool{} - - add := func(title string, href string, detail string) { - title = SanitizeDetail(title, 240) - href = strings.TrimSpace(href) - if title == "" || href == "" || seen[href] { - return - } - seen[href] = true - articles = append(articles, Article{ - Title: title, - URL: href, - Detail: SanitizeDetail(detail, 280), - }) - } - - doc.Find("article a[href]").Each(func(_ int, sel *goquery.Selection) { - href := resolveHref(pageURL, sel.AttrOr("href", "")) - if href == "" { - return - } - title := strings.TrimSpace(sel.Text()) - if title == "" { - title = strings.TrimSpace(sel.Closest("article").Find("h1,h2,h3").First().Text()) - } - add(title, href, sel.Closest("article").Text()) - }) - - doc.Find("h1 a[href], h2 a[href], h3 a[href]").Each(func(_ int, sel *goquery.Selection) { - href := resolveHref(pageURL, sel.AttrOr("href", "")) - title := strings.TrimSpace(sel.Text()) - add(title, href, sel.Parent().Text()) - }) - - if len(articles) == 0 { - doc.Find("a[href]").Each(func(_ int, sel *goquery.Selection) { - href := resolveHref(pageURL, sel.AttrOr("href", "")) - title := strings.TrimSpace(sel.Text()) - if len(title) < 12 || len(title) > 200 { - return - } - add(title, href, "") - }) - } - - if len(articles) == 0 { - title := strings.TrimSpace(doc.Find("title").First().Text()) - if title != "" { - add(title, pageURL, doc.Find("meta[name=description]").AttrOr("content", "")) - } - } - return articles -} - -func resolveHref(pageURL string, href string) string { - href = strings.TrimSpace(href) - if href == "" || strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") { - return "" - } - base, err := url.Parse(pageURL) - if err != nil { - return href - } - ref, err := url.Parse(href) - if err != nil { - return "" - } - resolved := base.ResolveReference(ref) - if resolved.Scheme != "http" && resolved.Scheme != "https" { - return "" - } - return resolved.String() -} diff --git a/workers/go/internal/newsreader/newsreader_test.go b/workers/go/internal/newsreader/newsreader_test.go deleted file mode 100644 index 3bee2d7..0000000 --- a/workers/go/internal/newsreader/newsreader_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package newsreader - -import "testing" - -func TestValidateRequiresSources(t *testing.T) { - _, err := Validate(Request{Mode: ModeRSS, Sources: []Source{}}) - if err == nil { - t.Fatal("expected validation error") - } -} - -func TestParseFeedReadsRSSItem(t *testing.T) { - rss := ` - Hellohttps://example.com/aBody - ` - articles := parseFeed(rss, "Example") - if len(articles) != 1 { - t.Fatalf("expected 1 article, got %d", len(articles)) - } - if articles[0].Title != "Hello" { - t.Fatalf("unexpected title %q", articles[0].Title) - } -} - -func TestCanonicalFetchURLRewritesGoogleNewsHomepage(t *testing.T) { - got := CanonicalFetchURL("https://news.google.com/", "tech news") - if got == "https://news.google.com/" { - t.Fatalf("expected RSS rewrite, got %q", got) - } -} - -func TestCanonicalFetchURLUpgradesGeneralGoogleNewsRSSForTechHint(t *testing.T) { - got := CanonicalFetchURL( - "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en", - "tech-news Tech", - ) - want := "https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en" - if got != want { - t.Fatalf("expected %q, got %q", want, got) - } -} - -func TestFilterTopicsKeepsUnmatchedArticlesWhenFewMatches(t *testing.T) { - articles := []Article{ - {Title: "TechCrunch story", URL: "https://example.com/a"}, - {Title: "Wall Street earnings", URL: "https://example.com/b"}, - {Title: "Another market report", URL: "https://example.com/c"}, - } - got := filterTopics(articles, []string{"Tech"}) - if len(got) != 3 { - t.Fatalf("expected all articles when only one matches, got %d", len(got)) - } - if got[0].Title != "TechCrunch story" { - t.Fatalf("expected matched article first, got %q", got[0].Title) - } -} diff --git a/workers/go/internal/newsreader/rss.go b/workers/go/internal/newsreader/rss.go deleted file mode 100644 index 3d8372e..0000000 --- a/workers/go/internal/newsreader/rss.go +++ /dev/null @@ -1,207 +0,0 @@ -package newsreader - -import ( - "encoding/xml" - "regexp" - "strings" -) - -type rssFeed struct { - Channel rssChannel `xml:"channel"` -} - -type rssChannel struct { - Items []rssItem `xml:"item"` -} - -type rssItem struct { - Title string `xml:"title"` - Link string `xml:"link"` - Description string `xml:"description"` - PubDate string `xml:"pubDate"` -} - -type atomFeed struct { - Entries []atomEntry `xml:"entry"` -} - -type atomEntry struct { - Title string `xml:"title"` - Link atomLink `xml:"link"` - Summary string `xml:"summary"` - Updated string `xml:"updated"` - Content atomContent `xml:"content"` -} - -type atomLink struct { - Href string `xml:"href,attr"` -} - -type atomContent struct { - Value string `xml:",chardata"` -} - -func looksLikeFeed(body string) bool { - lower := strings.ToLower(body) - return strings.Contains(lower, "") - blocks = append(blocks, splitBlocks(body, "")...) - articles := make([]Article, 0, len(blocks)) - for _, block := range blocks { - title := firstTag(block, "title") - link := firstTag(block, "link") - if link == "" { - link = firstAttr(block, "link", "href") - } - if link == "" { - link = firstTag(block, "guid") - } - description := firstTag(block, "description") - if description == "" { - description = firstTag(block, "summary") - } - if description == "" { - description = firstTag(block, "content") - } - cleanTitle := SanitizeDetail(title, 240) - cleanLink := strings.TrimSpace(StripTags(link)) - if cleanTitle == "" || cleanLink == "" { - continue - } - resolved := PreferredArticleURL(cleanLink, description) - detail := SanitizeDetail(description, 280) - articles = append(articles, Article{ - Title: cleanTitle, - URL: resolved, - Detail: detail, - PublishedAt: strings.TrimSpace(firstTag(block, "pubDate")), - }) - } - return articles -} - -func splitBlocks(body string, startToken string, endToken string) []string { - lower := strings.ToLower(body) - startLower := strings.ToLower(startToken) - endLower := strings.ToLower(endToken) - var blocks []string - idx := 0 - for { - start := strings.Index(lower[idx:], startLower) - if start < 0 { - break - } - start += idx - end := strings.Index(lower[start:], endLower) - if end < 0 { - break - } - end += start + len(endToken) - blocks = append(blocks, body[start:end]) - idx = end - } - return blocks -} - -func firstTag(block string, name string) string { - open := "<" + strings.ToLower(name) - lower := strings.ToLower(block) - start := strings.Index(lower, open) - if start < 0 { - return "" - } - closeTag := strings.Index(block[start:], ">") - if closeTag < 0 { - return "" - } - contentStart := start + closeTag + 1 - endToken := "" - end := strings.Index(strings.ToLower(block[contentStart:]), endToken) - if end < 0 { - return "" - } - return block[contentStart : contentStart+end] -} - -func firstAttr(block string, tagName string, attr string) string { - open := "<" + strings.ToLower(tagName) - lower := strings.ToLower(block) - start := strings.Index(lower, open) - if start < 0 { - return "" - } - closeTag := strings.Index(block[start:], ">") - if closeTag < 0 { - return "" - } - tag := block[start : start+closeTag+1] - pattern := regexp.MustCompile(`(?i)` + attr + `\s*=\s*"([^"]+)"`) - match := pattern.FindStringSubmatch(tag) - if len(match) < 2 { - return "" - } - return match[1] -} diff --git a/workers/go/internal/newsreader/text.go b/workers/go/internal/newsreader/text.go deleted file mode 100644 index 9f8e952..0000000 --- a/workers/go/internal/newsreader/text.go +++ /dev/null @@ -1,76 +0,0 @@ -package newsreader - -import ( - "net/url" - "regexp" - "strings" -) - -var tagPattern = regexp.MustCompile(`<[^>]+>`) -var hrefPattern = regexp.MustCompile(`(?i)href\s*=\s*"([^"]+)"`) -var whitespacePattern = regexp.MustCompile(`\s+`) - -func StripTags(raw string) string { - value := tagPattern.ReplaceAllString(raw, " ") - value = strings.ReplaceAll(value, "", "") - return DecodeEntities(value) -} - -func DecodeEntities(raw string) string { - replacements := []struct { - from string - to string - }{ - {"&", "&"}, - {"<", "<"}, - {">", ">"}, - {""", "\""}, - {"'", "'"}, - {"'", "'"}, - {" ", " "}, - } - value := raw - for _, pair := range replacements { - value = strings.ReplaceAll(value, pair.from, pair.to) - } - return value -} - -func CollapseWhitespace(raw string) string { - return strings.TrimSpace(whitespacePattern.ReplaceAllString(raw, " ")) -} - -func SanitizeDetail(raw string, maxLen int) string { - cleaned := CollapseWhitespace(StripTags(raw)) - if cleaned == "" { - return "" - } - if len(cleaned) <= maxLen { - return cleaned - } - return strings.TrimSpace(cleaned[:maxLen]) + "…" -} - -func PreferredArticleURL(link string, htmlSnippet string) string { - link = strings.TrimSpace(link) - for _, match := range hrefPattern.FindAllStringSubmatch(htmlSnippet, -1) { - if len(match) < 2 { - continue - } - href := strings.TrimSpace(match[1]) - if !strings.HasPrefix(href, "http") { - continue - } - parsed, err := url.Parse(href) - if err != nil { - continue - } - host := strings.ToLower(parsed.Hostname()) - if strings.Contains(host, "google.com") || strings.Contains(host, "googleusercontent.com") { - continue - } - return href - } - return link -} diff --git a/workers/go/internal/newsreader/types.go b/workers/go/internal/newsreader/types.go deleted file mode 100644 index 3738664..0000000 --- a/workers/go/internal/newsreader/types.go +++ /dev/null @@ -1,49 +0,0 @@ -package newsreader - -const ( - DefaultMaxCount = 20 - MaximumMaxCount = 50 - DefaultTimeoutSeconds = 120 - MaximumTimeoutSeconds = 300 -) - -type Mode string - -const ( - ModeRSS Mode = "rss" - ModeList Mode = "list" - ModeSummary Mode = "summary" -) - -type Source struct { - Label string `json:"label"` - URL string `json:"url"` -} - -type Request struct { - Mode Mode `json:"mode"` - Sources []Source `json:"sources"` - Topics []string `json:"topics"` - MaxCount int `json:"maxCount"` - ContextHint string `json:"contextHint,omitempty"` -} - -type Article struct { - Title string `json:"title"` - URL string `json:"url"` - Detail string `json:"detail,omitempty"` - PublishedAt string `json:"published_at,omitempty"` -} - -type Result struct { - OK bool `json:"ok"` - Mode Mode `json:"mode"` - Articles []Article `json:"articles"` - Diagnostics []string `json:"diagnostics"` -} - -type ProxyConfig struct { - Host string - Port int - Token string -} diff --git a/workers/go/internal/newsreader/urls.go b/workers/go/internal/newsreader/urls.go deleted file mode 100644 index 599fe7d..0000000 --- a/workers/go/internal/newsreader/urls.go +++ /dev/null @@ -1,62 +0,0 @@ -package newsreader - -import ( - "net/url" - "strings" -) - -func CanonicalFetchURL(raw string, contextHint string) string { - parsed, err := url.Parse(raw) - if err != nil { - return raw - } - host := strings.ToLower(parsed.Hostname()) - if !strings.Contains(host, "news.google.com") { - return raw - } - path := strings.ToLower(parsed.Path) - if strings.Contains(path, "/rss") || strings.HasSuffix(path, ".xml") { - if !strings.Contains(path, "/headlines/section/topic/") { - if section := googleNewsSection(contextHint); section != "" { - return "https://news.google.com/rss/headlines/section/topic/" + section + "?hl=en-US&gl=US&ceid=US:en" - } - } - return raw - } - if strings.Contains(path, "/topics/") { - if section := googleNewsSection(contextHint); section != "" { - return "https://news.google.com/rss/headlines/section/topic/" + section + "?hl=en-US&gl=US&ceid=US:en" - } - return "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" - } - if parsed.RawQuery == "" { - return "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en" - } - return raw -} - -func googleNewsSection(hint string) string { - lower := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(hint, "-", " "), "_", " ")) - if strings.Contains(lower, "tech") { - return "TECHNOLOGY" - } - if strings.Contains(lower, "business") || strings.Contains(lower, "finance") || strings.Contains(lower, "market") { - return "BUSINESS" - } - if strings.Contains(lower, "science") { - return "SCIENCE" - } - if strings.Contains(lower, "sport") { - return "SPORTS" - } - if strings.Contains(lower, "health") { - return "HEALTH" - } - if strings.Contains(lower, "entertainment") { - return "ENTERTAINMENT" - } - if strings.Contains(lower, "world") { - return "WORLD" - } - return "" -} diff --git a/workers/go/internal/newsreader/validate.go b/workers/go/internal/newsreader/validate.go deleted file mode 100644 index a9ec819..0000000 --- a/workers/go/internal/newsreader/validate.go +++ /dev/null @@ -1,38 +0,0 @@ -package newsreader - -import ( - "fmt" - "net/url" - "strings" -) - -func Validate(req Request) (Request, error) { - out := req - if out.Mode != ModeRSS && out.Mode != ModeList && out.Mode != ModeSummary { - return out, fmt.Errorf("mode must be rss, list, or summary") - } - if len(out.Sources) == 0 { - return out, fmt.Errorf("add at least one source URL") - } - if out.MaxCount <= 0 { - out.MaxCount = DefaultMaxCount - } - if out.MaxCount > MaximumMaxCount { - out.MaxCount = MaximumMaxCount - } - for i, source := range out.Sources { - trimmed := strings.TrimSpace(source.URL) - if trimmed == "" { - return out, fmt.Errorf("source %d is missing a URL", i+1) - } - parsed, err := url.Parse(trimmed) - if err != nil || parsed.Scheme != "http" && parsed.Scheme != "https" { - return out, fmt.Errorf("source %d is not a usable web address", i+1) - } - out.Sources[i].URL = trimmed - if strings.TrimSpace(out.Sources[i].Label) == "" { - out.Sources[i].Label = parsed.Host - } - } - return out, nil -} From d32838e1658bbabb899f231070a90daf547107b3 Mon Sep 17 00:00:00 2001 From: David Choi Date: Fri, 11 Sep 2026 22:23:47 -0400 Subject: [PATCH 10/12] Fix plugin factory test fixtures so content hashes match packaged files. Co-authored-by: Cursor --- .../DBRepositoryTests/DBRepositoryTests.swift | 92 +++++++++---------- 1 file changed, 42 insertions(+), 50 deletions(-) diff --git a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift index 4d2ed20..343da15 100644 --- a/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift +++ b/packages/DBRepository/Tests/DBRepositoryTests/DBRepositoryTests.swift @@ -81,23 +81,9 @@ final class DBRepositoryTests: XCTestCase { func testSchemaUpgradeDoesNotWipeExistingRows() async throws { let repository = try makeRepository() _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"keep-me"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.go": Data("package main".utf8), - "app.derrick/plugin": artifact, - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "keep-me", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: [:], - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "keep-me" ) try await repository.savePluginFactoryRelease(release) let url = await repository.databaseURL @@ -119,25 +105,11 @@ final class DBRepositoryTests: XCTestCase { func testApprovedPluginFactoryReleasePersistsAndVerifies() async throws { let repository = try makeRepository() _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) let skillFiles = ["skills/weather/SKILL.md": "# Weather"] - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.go": Data("package main".utf8), - "app.derrick/plugin": artifact, - "skills/weather/SKILL.md": Data("# Weather".utf8), - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "weather-tool", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: skillFiles, - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "weather-tool", + skillFiles: skillFiles ) try await repository.savePluginFactoryRelease(release) @@ -164,25 +136,11 @@ final class DBRepositoryTests: XCTestCase { ) ) _ = try await repository.createEmptyDatabaseIfNeeded(username: "app-user", password: "app-secret") - let artifact = Data("compiled".utf8) let skillFiles = ["skills/weather/SKILL.md": "# Weather"] - let files: [String: Data] = [ - "plugin.json": Data(#"{"name":"weather-tool"}"#.utf8), - "app.derrick/runtime.json": Data(#"{"language":"swift"}"#.utf8), - "app.derrick/plugin.go": Data("package main".utf8), - "app.derrick/plugin": artifact, - "skills/weather/SKILL.md": Data("# Weather".utf8), - ] - let release = PluginFactoryRelease( + let release = makeGoFactoryRelease( pluginID: "weather-tool", - version: "1.0.0", - manifestJSON: String(decoding: files["plugin.json"] ?? Data(), as: UTF8.self), - runtimeJSON: String(decoding: files["app.derrick/runtime.json"] ?? Data(), as: UTF8.self), - guestSource: "print(\"[]\")", - compiledArtifact: artifact, - skillFiles: skillFiles, - contentHash: PluginContentHash.hash(files: files), - reviewSummary: "approved" + manifestName: "weather-tool", + skillFiles: skillFiles ) try await repository.savePluginFactoryRelease(release) @@ -633,4 +591,38 @@ final class DBRepositoryTests: XCTestCase { } return String(cString: c) } + + /// Hash must match `packageFiles()`, which uses `guestSource` at the Go guest path. + private func makeGoFactoryRelease( + pluginID: String, + manifestName: String, + skillFiles: [String: String] = [:] + ) -> PluginFactoryRelease { + let artifact = Data("compiled".utf8) + let guestSource = "package main" + let manifestJSON = "{\"name\":\"\(manifestName)\"}" + let runtimeJSON = #"{"language":"go"}"# + var files: [String: Data] = [ + "plugin.json": Data(manifestJSON.utf8), + "app.derrick/runtime.json": Data(runtimeJSON.utf8), + "app.derrick/plugin.go": Data(guestSource.utf8), + "app.derrick/plugin": artifact, + ] + for (path, body) in skillFiles { + files[path] = Data(body.utf8) + } + let release = PluginFactoryRelease( + pluginID: pluginID, + version: "1.0.0", + manifestJSON: manifestJSON, + runtimeJSON: runtimeJSON, + guestSource: guestSource, + compiledArtifact: artifact, + skillFiles: skillFiles, + contentHash: PluginContentHash.hash(files: files), + reviewSummary: "approved" + ) + XCTAssertTrue(release.verifyIntegrity(), "test fixture hash must match packageFiles()") + return release + } } From 1eeac37cd91d7f7ea629a4d1fc5b53139bdbd2a4 Mon Sep 17 00:00:00 2001 From: David Choi Date: Fri, 11 Sep 2026 22:44:19 -0400 Subject: [PATCH 11/12] Fix MCPServer Docker tests for the unified worker image. Mocks now return the binaries label and pin digest, so prewarm skips rebuild when the image is already current and CI no longer treats an empty inspect as a missing image. Co-authored-by: Cursor --- .../MCPServer/DockerImageInspector.swift | 2 +- .../DockerProductImagePrewarmer.swift | 2 +- .../Tests/MCPServerTests/MCPServerTests.swift | 128 +++++++++++------- 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift index 5787393..d242193 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerImageInspector.swift @@ -34,7 +34,7 @@ public enum DockerImageInspector: Sendable { } } - /// Returns false when the image exists but predates required worker binaries (e.g. news reader). + /// Returns false when the image exists but predates required worker binaries. public static func workerImageHasCurrentBinaries( tag: String = DockerWorkerRuntime.image, executor: @escaping DockerCLIExecutor diff --git a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift index 82f9d7b..1c1b340 100644 --- a/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift +++ b/packages/MCPServer/Sources/MCPServer/DockerProductImagePrewarmer.swift @@ -48,7 +48,7 @@ public enum DockerProductImagePrewarmer: Sendable { ) return } - // Stale worker image (missing news reader, etc.). Rebuild overwrites the tag. + // Stale worker image (missing required binaries). Rebuild overwrites the tag. } guard let repoRoot = DerrickRepositoryRoot.locate() else { diff --git a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift index c9dd3a5..b397091 100644 --- a/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift +++ b/packages/MCPServer/Tests/MCPServerTests/MCPServerTests.swift @@ -7,7 +7,7 @@ import Plugin import WebCrawler @testable import MCPServer -@Suite struct MCPServerTests { +@Suite(.serialized) struct MCPServerTests { private static let dummyGoScript = """ package main @@ -38,9 +38,49 @@ import WebCrawler let digest = DockerWorkerRuntime.pinnedDigest.rawValue + "\n" return DockerCLIResult(exitCode: 0, stdout: Data(digest.utf8), stderr: Data()) } + if arguments.contains(where: { $0.contains(DockerWorkerRuntime.binariesLabelKey) }) { + let label = DockerWorkerRuntime.binariesLabelValue + "\n" + return DockerCLIResult(exitCode: 0, stdout: Data(label.utf8), stderr: Data()) + } return DockerCLIResult(exitCode: 0, stdout: Data("[]".utf8), stderr: Data()) } + private actor ImageBuildLatch { + private(set) var succeeded = false + func markSucceeded() { succeeded = true } + } + + private static func missingUntilBuiltExecutor( + recorder: DockerCallRecorder, + latch: ImageBuildLatch, + failFirstBuild: Bool = false, + buildDelay: Duration? = nil + ) -> DockerCLIExecutor { + { args, _, _ in + await recorder.append(args) + if args.first == "build" { + if let buildDelay { + try await Task.sleep(for: buildDelay) + } + if failFirstBuild { + let builds = await recorder.calls.filter { $0.first == "build" }.count + if builds == 1 { + return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data("boom".utf8)) + } + } + await latch.markSucceeded() + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + if args.first == "image" { + if await latch.succeeded, let mocked = mockWorkerImageInspect(args) { + return mocked + } + return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) + } + return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) + } + } + private static let dummyCompiledGuest = Data([0x7f, 0x45, 0x4c, 0x46, 0x02]) private static func mockGuestDocker(_ arguments: [String]) -> DockerCLIResult? { @@ -299,42 +339,40 @@ import WebCrawler let recorder = DockerCallRecorder() let executor: DockerCLIExecutor = { args, _, _ in await recorder.append(args) + if let mocked = Self.mockWorkerImageInspect(args) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) } try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) - #expect(await recorder.calls == [["image", "inspect", DockerProductImagePolicy.webCrawlerImage]]) + let calls = await recorder.calls + #expect(calls.first == ["image", "inspect", DockerProductImagePolicy.workerImage]) + #expect(calls.contains { $0.contains("--format") && $0.contains(where: { $0.contains(DockerWorkerRuntime.binariesLabelKey) }) }) + #expect(calls.contains { $0.contains("{{.Id}}") }) + #expect(!calls.contains { $0.first == "build" }) } @Test func dockerProductImagePrewarmerBuildsWhenImageMissing() async throws { guard DerrickRepositoryRoot.locate() != nil else { return } let recorder = DockerCallRecorder() - let executor: DockerCLIExecutor = { args, _, _ in - await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor(recorder: recorder, latch: latch) try await DockerProductImagePrewarmer.ensureWebCrawlerImage(executor: executor) let calls = await recorder.calls - #expect(calls.count == 2) - #expect(calls[0] == ["image", "inspect", DockerProductImagePolicy.webCrawlerImage]) - #expect(calls[1].first == "build") - #expect(calls[1].contains(DockerProductImagePolicy.webCrawlerImage)) - #expect(calls[1].last?.hasSuffix("/\(DockerProductImagePolicy.webCrawlerBuildContextRelativePath)") == true) + #expect(calls[0] == ["image", "inspect", DockerProductImagePolicy.workerImage]) + #expect(calls.contains { $0.first == "build" && $0.contains(DockerProductImagePolicy.workerImage) }) + #expect(calls.contains { $0.contains("{{.Id}}") }) + #expect(calls.filter { $0.first == "build" }.count == 1) } @Test func crawlerImageBuildFailureMessageOmitsBuildkitDump() { let error = DockerProductImagePrewarmerError.buildFailed( - DockerProductImagePolicy.webCrawlerImage, + DockerProductImagePolicy.workerImage, "#0 building with \"default\" instance using docker driver" ) let text = error.localizedDescription #expect(!text.contains("#0 building")) - #expect(text.lowercased().contains("web crawler")) + #expect(text.lowercased().contains("worker image")) #expect(text.lowercased().contains("disk")) #expect(error.compilerDiagnostic == nil) } @@ -346,7 +384,7 @@ import WebCrawler error: Build failed """ let error = DockerProductImagePrewarmerError.buildFailed( - DockerProductImagePolicy.webCrawlerImage, + DockerProductImagePolicy.workerImage, detail ) #expect(error.compilerDiagnostic?.contains("CryptoKit") == true) @@ -356,17 +394,12 @@ import WebCrawler @Test func crawlerImageBuildIsSingleFlight() async throws { guard DerrickRepositoryRoot.locate() != nil else { return } let recorder = DockerCallRecorder() - let executor: DockerCLIExecutor = { args, _, _ in - await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - try await Task.sleep(for: .milliseconds(80)) - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor( + recorder: recorder, + latch: latch, + buildDelay: .milliseconds(80) + ) let gate = WebCrawlerImageGate() try await withThrowingTaskGroup(of: Void.self) { group in group.addTask { try await gate.ensureReady(executor: executor) } @@ -376,26 +409,18 @@ import WebCrawler } let calls = await recorder.calls #expect(calls.filter { $0.first == "build" }.count == 1) - #expect(calls.filter { $0.first == "image" }.count == 1) + #expect(calls.filter { $0 == ["image", "inspect", DockerProductImagePolicy.workerImage] }.count == 1) } @Test func crawlerImageBuildFailureAllowsRetry() async throws { guard DerrickRepositoryRoot.locate() != nil else { return } let recorder = DockerCallRecorder() - let executor: DockerCLIExecutor = { args, _, _ in - await recorder.append(args) - if args.first == "image" { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data()) - } - if args.first == "build" { - let builds = await recorder.calls.filter { $0.first == "build" }.count - if builds == 1 { - return DockerCLIResult(exitCode: 1, stdout: Data(), stderr: Data("boom".utf8)) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } - return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) - } + let latch = ImageBuildLatch() + let executor = Self.missingUntilBuiltExecutor( + recorder: recorder, + latch: latch, + failFirstBuild: true + ) let gate = WebCrawlerImageGate() do { try await gate.ensureReady(executor: executor) @@ -444,6 +469,9 @@ import WebCrawler let runner = FileExtractorDockerExecutor( executor: { arguments, _, _ in await recorder.append(arguments) + if let mocked = Self.mockWorkerImageInspect(arguments) { + return mocked + } return DockerCLIResult(exitCode: 0, stdout: Data(), stderr: Data()) }, queue: DerrickDockerRunQueue(maxConcurrentContainers: 1) @@ -480,11 +508,17 @@ import WebCrawler timeoutSeconds: 5 ) Issue.record("expected missing extractor image") + } catch is DockerProductImagePrewarmerError { + // Image is missing; prewarmer tries a rebuild and that mock also fails. + } catch is DockerImageDigestError { + // Pin check after a failed inspect. } catch let error as FileExtractorDockerExecutorError { #expect(error == .imageUnavailable(FileExtractorDockerExecutor.image)) } let calls = await recorder.calls - #expect(calls == [["image", "inspect", FileExtractorDockerExecutor.image]]) + #expect(calls.contains { $0.first == "image" && $0.contains("inspect") }) + #expect(!calls.contains { $0.first == "create" }) + #expect(!calls.contains { $0.first == "start" }) } @Test func orphanSweeperRemovesLabeledAndPrefixedContainers() async throws { From ae20ed2401d49278091f073002699502815c9865 Mon Sep 17 00:00:00 2001 From: David Choi Date: Fri, 11 Sep 2026 23:22:40 -0400 Subject: [PATCH 12/12] Stop requiring host Go in StructureTests on CI. Guest compile runs in Docker; ensureInstalled is a local diagnostic. Tests now cover version parsing instead of calling `go version` on the runner. Co-authored-by: Cursor --- .../DockerRunnerXPC/DerrickGoToolchain.swift | 4 ++-- .../DerrickGoToolchainTests.swift | 23 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift index dc08137..7253423 100644 --- a/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift +++ b/packages/Structure/Sources/DockerRunnerXPC/DerrickGoToolchain.swift @@ -27,7 +27,7 @@ public enum DerrickGoToolchain: Sendable { } } - private static func parseVersion(_ text: String) -> String? { + static func parseVersion(_ text: String) -> String? { // go version go1.27.1 darwin/arm64 for part in text.split(separator: " ") { let token = String(part) @@ -38,7 +38,7 @@ public enum DerrickGoToolchain: Sendable { return nil } - private static func versionSatisfies(_ found: String, minimum: String) -> Bool { + static func versionSatisfies(_ found: String, minimum: String) -> Bool { compareVersions(found, minimum) != .orderedAscending } diff --git a/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift index 6587b5d..b618f0f 100644 --- a/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift +++ b/packages/Structure/Tests/StructureTests/DerrickGoToolchainTests.swift @@ -1,12 +1,29 @@ -import Structure import Testing +@testable import Structure @Suite struct DerrickGoToolchainTests { @Test func minimumVersionIsPinned() { #expect(DerrickGoToolchain.minimumVersion == "1.27.1") } - @Test func ensureInstalledAcceptsCurrentGo() throws { - try DerrickGoToolchain.ensureInstalled() + @Test func parseVersionReadsGoVersionLine() { + #expect( + DerrickGoToolchain.parseVersion("go version go1.27.1 darwin/arm64") == "1.27.1" + ) + } + + @Test func parseVersionRejectsEmptyOutput() { + #expect(DerrickGoToolchain.parseVersion("") == nil) + #expect(DerrickGoToolchain.parseVersion("go version") == nil) + } + + @Test func versionSatisfiesAcceptsCurrentAndNewer() { + #expect(DerrickGoToolchain.versionSatisfies("1.27.1", minimum: "1.27.1")) + #expect(DerrickGoToolchain.versionSatisfies("1.28.0", minimum: "1.27.1")) + } + + @Test func versionSatisfiesRejectsOlder() { + #expect(!DerrickGoToolchain.versionSatisfies("1.27.0", minimum: "1.27.1")) + #expect(!DerrickGoToolchain.versionSatisfies("1.26.9", minimum: "1.27.1")) } }