From db321b8f61cd29788ef984840c7623489c5251bb Mon Sep 17 00:00:00 2001 From: Matteo Date: Fri, 25 Sep 2026 18:02:12 +0200 Subject: [PATCH] Ops scripts: backend container name, safe re-runs, probe Vinted again - scripts/ops: the cloud backend runs as amcp-cloud-backend since the container split; every docker cp/exec/restart line now names it. Each restart step now silences the host uptime probe first with an expiring /var/lib/anythingmcp-probe/maintenance marker (as deploy-cloud.yml does), since the restart takes the API down for ~30-60 s. - migrate-vinted-cloud.mjs: skip connectors already at the target (base URL, LOGIN_TOKEN with the adapter's authConfig, adapterVersion, tool set) and list them as "already current"; a second --apply writes nothing. - migrate-deutsche-bahn-cloud.mjs: the generated SQL leaves out connectors already on this adapterVersion and MOTIS URL and reports them. - Remove migrate-amadeus-client-credentials.mjs: the Amadeus adapter is gone (#713). - probe-keyless.mjs: "keyless" now means no user-supplied credentials, not authType NONE. Adapters whose authConfig has no {{VAR}} are probed too, with LOGIN_TOKEN reproduced from the adapter JSON (cookie or body token), which brings Vinted's anonymous session back into the weekly run. Auth types the probe cannot reproduce are listed as unsupported-auth. --- scripts/ops/fix-etsy-api-key.mjs | 20 ++- scripts/ops/fix-reddit-oauth.mjs | 20 ++- .../migrate-amadeus-client-credentials.mjs | 169 ----------------- scripts/ops/migrate-deutsche-bahn-cloud.mjs | 16 +- scripts/ops/migrate-vinted-cloud.mjs | 79 +++++++- scripts/ops/resync-catalog-connectors.mjs | 20 ++- scripts/probe-keyless.mjs | 170 ++++++++++++++++-- 7 files changed, 281 insertions(+), 213 deletions(-) delete mode 100644 scripts/ops/migrate-amadeus-client-credentials.mjs diff --git a/scripts/ops/fix-etsy-api-key.mjs b/scripts/ops/fix-etsy-api-key.mjs index 50e62b3f..d5cccd42 100644 --- a/scripts/ops/fix-etsy-api-key.mjs +++ b/scripts/ops/fix-etsy-api-key.mjs @@ -24,12 +24,18 @@ * Anything else (a stray email address someone typed into the field) is left * for its owner: we cannot invent the missing half. * - * Run it inside the app container, which holds ENCRYPTION_KEY and DATABASE_URL: + * Run it inside the backend container, which holds ENCRYPTION_KEY and DATABASE_URL: * - * docker cp scripts/ops/fix-etsy-api-key.mjs amcp-cloud-app:/app/backend/fix-etsy.mjs - * docker exec -w /app/backend amcp-cloud-app node fix-etsy.mjs # dry run - * docker exec -w /app/backend amcp-cloud-app node fix-etsy.mjs --apply - * docker restart amcp-cloud-app # the tool registry caches connector config + * docker cp scripts/ops/fix-etsy-api-key.mjs amcp-cloud-backend:/app/backend/fix-etsy.mjs + * docker exec -w /app/backend amcp-cloud-backend node fix-etsy.mjs # dry run + * docker exec -w /app/backend amcp-cloud-backend node fix-etsy.mjs --apply + * # These three on the droplet host, not in the container. The restart takes the API + * # down for ~30-60 s; silence the uptime probe first. It honours an expiry + * # epoch in this file (deploy/cloud/uptime-probe.sh, as deploy-cloud.yml + * # does), so a forgotten marker lapses by itself after 10 minutes: + * mkdir -p /var/lib/anythingmcp-probe && echo $(( $(date -u +%s) + 600 )) > /var/lib/anythingmcp-probe/maintenance + * docker restart amcp-cloud-backend # the tool registry caches connector config + * rm -f /var/lib/anythingmcp-probe/maintenance * * Secrets never reach stdout: it prints connector ids and which case applied. */ @@ -43,7 +49,7 @@ const SECRET_VAR = '{{ETSY_CLIENT_SECRET}}'; const KEY = process.env.ENCRYPTION_KEY; if (!KEY) { - console.error('ENCRYPTION_KEY is not set — run this inside the app container.'); + console.error('ENCRYPTION_KEY is not set — run this inside the backend container.'); process.exit(1); } @@ -155,6 +161,6 @@ console.log( `\n${APPLY ? 'Patched' : 'Would patch'} ${patched} of ${rows.length} Etsy connectors ` + `(${skipped} left alone).`, ); -if (!APPLY && patched > 0) console.log('Re-run with --apply, then restart the app.'); +if (!APPLY && patched > 0) console.log('Re-run with --apply, then restart the backend (see the header).'); await prisma.$disconnect(); diff --git a/scripts/ops/fix-reddit-oauth.mjs b/scripts/ops/fix-reddit-oauth.mjs index 3cb4dd7f..c93af440 100644 --- a/scripts/ops/fix-reddit-oauth.mjs +++ b/scripts/ops/fix-reddit-oauth.mjs @@ -31,12 +31,18 @@ * catalog no longer ships are soft-deprecated exactly as a full catalog * re-sync would (deprecatedAt set, disabled; nothing is deleted). * - * Run it inside the app container, which holds ENCRYPTION_KEY and DATABASE_URL: + * Run it inside the backend container, which holds ENCRYPTION_KEY and DATABASE_URL: * - * docker cp scripts/ops/fix-reddit-oauth.mjs amcp-cloud-app:/app/backend/fix-reddit.mjs - * docker exec -w /app/backend amcp-cloud-app node fix-reddit.mjs # dry run - * docker exec -w /app/backend amcp-cloud-app node fix-reddit.mjs --apply - * docker restart amcp-cloud-app # the tool registry caches connector config + * docker cp scripts/ops/fix-reddit-oauth.mjs amcp-cloud-backend:/app/backend/fix-reddit.mjs + * docker exec -w /app/backend amcp-cloud-backend node fix-reddit.mjs # dry run + * docker exec -w /app/backend amcp-cloud-backend node fix-reddit.mjs --apply + * # These three on the droplet host, not in the container. The restart takes the API + * # down for ~30-60 s; silence the uptime probe first. It honours an expiry + * # epoch in this file (deploy/cloud/uptime-probe.sh, as deploy-cloud.yml + * # does), so a forgotten marker lapses by itself after 10 minutes: + * mkdir -p /var/lib/anythingmcp-probe && echo $(( $(date -u +%s) + 600 )) > /var/lib/anythingmcp-probe/maintenance + * docker restart amcp-cloud-backend # the tool registry caches connector config + * rm -f /var/lib/anythingmcp-probe/maintenance * * Secrets never reach stdout: it prints connector ids, which case applied, and * whether the client ID/secret are present, never their values. @@ -61,7 +67,7 @@ const CATALOG_TOOLS = new Set([ const KEY = process.env.ENCRYPTION_KEY; if (!KEY) { - console.error('ENCRYPTION_KEY is not set — run this inside the app container.'); + console.error('ENCRYPTION_KEY is not set — run this inside the backend container.'); process.exit(1); } @@ -207,6 +213,6 @@ console.log( `\n${APPLY ? 'Patched' : 'Would patch'} ${patched} of ${rows.length} Reddit connectors ` + `(${skipped} unchanged or left alone). ${needCustomer} still need the customer's client ID and secret.`, ); -if (!APPLY && patched > 0) console.log('Re-run with --apply, then restart the app.'); +if (!APPLY && patched > 0) console.log('Re-run with --apply, then restart the backend (see the header).'); await prisma.$disconnect(); diff --git a/scripts/ops/migrate-amadeus-client-credentials.mjs b/scripts/ops/migrate-amadeus-client-credentials.mjs deleted file mode 100644 index 9036eda3..00000000 --- a/scripts/ops/migrate-amadeus-client-credentials.mjs +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env node -/** - * One-off: move installed `amadeus` connectors from a pasted Bearer token to - * the OAuth2 client-credentials grant the adapter now declares. - * - * Amadeus access tokens expire after ~30 minutes, so the old adapter's - * `BEARER_TOKEN` + `{{AMADEUS_ACCESS_TOKEN}}` could never work for longer than - * that. The adapter now asks for the API key and secret and lets the engine - * mint tokens. The catalog re-sync never touches authType/authConfig, and - * authConfig is encrypted, so existing rows need this script: without it they - * would show the new instructions (set AMADEUS_CLIENT_ID / SECRET) while still - * demanding AMADEUS_ACCESS_TOKEN. - * - * Per BEARER_TOKEN row it writes: - * authType OAUTH2 - * authConfig { grant: client_credentials, tokenAuthMethod: client_secret_post, - * tokenUrl: /v1/security/oauth2/token, - * clientId: {{AMADEUS_CLIENT_ID}}, clientSecret: {{AMADEUS_CLIENT_SECRET}} } - * Placeholders are resolved from the row's own env vars where it already has - * them — the token service re-reads authConfig from the database before each - * token request, so a placeholder at rest would be sent as the literal client - * id. Where it has none they stay placeholders: the call then fails with the - * "missing a value for AMADEUS_CLIENT_ID" message, and setting the variables in - * the UI re-resolves authConfig from the adapter template. The old token is - * dropped: whatever it was, it expired half an hour after it was pasted. - * - * Host: `test.api.amadeus.com` stopped resolving when Amadeus retired the - * Self-Service portal (2026-07-17), so rows on it move to `api.amadeus.com` - * (baseUrl and baseUrlBaseline). Any other host is kept, and the token URL - * follows it. Rows already on OAUTH2, or on any other auth type, are left alone. - * - * Run it inside the backend container, which holds ENCRYPTION_KEY and DATABASE_URL: - * - * docker cp scripts/ops/migrate-amadeus-client-credentials.mjs amcp-cloud-backend:/app/backend/migrate-amadeus.mjs - * docker exec -w /app/backend amcp-cloud-backend node migrate-amadeus.mjs # dry run - * docker exec -w /app/backend amcp-cloud-backend node migrate-amadeus.mjs --apply - * docker restart amcp-cloud-backend # the tool registry caches connector config - * - * Secrets never reach stdout: it prints connector ids and what changed. - */ -import { createDecipheriv, createCipheriv, randomBytes } from 'node:crypto'; -import { PrismaClient } from './dist/src/generated/prisma/client.js'; -import { PrismaPg } from '@prisma/adapter-pg'; - -const APPLY = process.argv.includes('--apply'); -const RETIRED_HOST = 'test.api.amadeus.com'; -const LIVE_ORIGIN = 'https://api.amadeus.com'; - -const KEY = process.env.ENCRYPTION_KEY; -if (!KEY) { - console.error('ENCRYPTION_KEY is not set — run this inside the app container.'); - process.exit(1); -} - -// Byte-for-byte the scheme in packages/backend/src/common/crypto/encryption.util.ts: -// base64(iv[16] + ciphertext + tag[16]), key = the raw env value truncated to 32 -// bytes (NOT hashed), and no AAD for connector.authConfig. -const IV_LENGTH = 16; -const TAG_LENGTH = 16; -const keyBuf = Buffer.from(KEY, 'utf-8').subarray(0, 32); - -function decrypt(ciphertext) { - const data = Buffer.from(ciphertext, 'base64'); - const iv = data.subarray(0, IV_LENGTH); - const tag = data.subarray(data.length - TAG_LENGTH); - const body = data.subarray(IV_LENGTH, data.length - TAG_LENGTH); - const d = createDecipheriv('aes-256-gcm', keyBuf, iv); - d.setAuthTag(tag); - return d.update(body) + d.final('utf8'); -} - -function encrypt(plaintext) { - const iv = randomBytes(IV_LENGTH); - const c = createCipheriv('aes-256-gcm', keyBuf, iv); - const body = Buffer.concat([c.update(plaintext, 'utf8'), c.final()]); - return Buffer.concat([iv, body, c.getAuthTag()]).toString('base64'); -} - -// Round-trip first: a wrong key or a drifted scheme stops here instead of -// writing rows nobody can read again. -const probe = JSON.stringify({ probe: 'amadeus', n: Date.now() }); -if (decrypt(encrypt(probe)) !== probe) { - console.error('Encryption self-test failed — refusing to touch any row.'); - process.exit(1); -} - -const prisma = new PrismaClient({ - adapter: new PrismaPg({ connectionString: process.env.DATABASE_URL }), -}); - -const rows = await prisma.connector.findMany({ - where: { config: { path: ['adapterSlug'], equals: 'amadeus' } }, - select: { - id: true, - organizationId: true, - authType: true, - baseUrl: true, - config: true, - envVars: true, - }, -}); - -let patched = 0; -let skipped = 0; - -for (const row of rows) { - if (row.authType !== 'BEARER_TOKEN') { - console.log(`- ${row.id}: authType ${row.authType}, left alone`); - skipped++; - continue; - } - - let baseUrl; - let origin; - try { - const url = new URL(row.baseUrl); - const retired = url.hostname === RETIRED_HOST; - baseUrl = retired ? LIVE_ORIGIN + row.baseUrl.slice(url.origin.length) : row.baseUrl; - origin = retired ? LIVE_ORIGIN : url.origin; - } catch { - console.log(`! ${row.id}: baseUrl is not a URL, left alone`); - skipped++; - continue; - } - - const vars = row.envVars ?? {}; - const valueOr = (name) => - typeof vars[name] === 'string' && vars[name].trim() ? vars[name].trim() : `{{${name}}}`; - const authConfig = { - grant: 'client_credentials', - tokenAuthMethod: 'client_secret_post', - tokenUrl: `${origin}/v1/security/oauth2/token`, - clientId: valueOr('AMADEUS_CLIENT_ID'), - clientSecret: valueOr('AMADEUS_CLIENT_SECRET'), - }; - const credentials = authConfig.clientId.includes('{{') || authConfig.clientSecret.includes('{{') - ? 'awaiting AMADEUS_CLIENT_ID/SECRET' - : 'credentials from env vars'; - const movedHost = baseUrl !== row.baseUrl; - - if (APPLY) { - await prisma.connector.update({ - where: { id: row.id }, - data: { - authType: 'OAUTH2', - authConfig: encrypt(JSON.stringify(authConfig)), - ...(movedHost - ? { - baseUrl, - config: { ...(row.config ?? {}), baseUrlBaseline: baseUrl }, - } - : {}), - }, - }); - } - console.log( - `${APPLY ? '✓' : '→'} ${row.id} (org ${row.organizationId}) — OAUTH2 client_credentials, ${credentials}` + - (movedHost ? `, baseUrl ${row.baseUrl} → ${baseUrl}` : ''), - ); - patched++; -} - -console.log( - `\n${APPLY ? 'Migrated' : 'Would migrate'} ${patched} of ${rows.length} Amadeus connectors ` + - `(${skipped} left alone).`, -); -if (!APPLY && patched > 0) console.log('Re-run with --apply, then restart the app.'); - -await prisma.$disconnect(); diff --git a/scripts/ops/migrate-deutsche-bahn-cloud.mjs b/scripts/ops/migrate-deutsche-bahn-cloud.mjs index 3ee7f76e..664e185a 100644 --- a/scripts/ops/migrate-deutsche-bahn-cloud.mjs +++ b/scripts/ops/migrate-deutsche-bahn-cloud.mjs @@ -9,7 +9,13 @@ * node scripts/ops/migrate-deutsche-bahn-cloud.mjs http://motis:8080 > /tmp/db.sql * scp /tmp/db.sql root@:/tmp/db.sql * ssh root@ 'docker exec -i amcp-cloud-postgres psql -U amcp -d anythingmcp -v ON_ERROR_STOP=1 -1 -f - < /tmp/db.sql' - * ssh root@ 'docker restart amcp-cloud-app' # the tool registry is in memory + * # The restart takes the API down for ~30-60 s; silence the uptime probe + * # first. It honours an expiry + * # epoch in this file (deploy/cloud/uptime-probe.sh, as deploy-cloud.yml + * # does), so a forgotten marker lapses by itself after 10 minutes: + * ssh root@ 'mkdir -p /var/lib/anythingmcp-probe && echo $(( $(date -u +%s) + 600 )) > /var/lib/anythingmcp-probe/maintenance' + * ssh root@ 'docker restart amcp-cloud-backend' # the tool registry is in memory + * ssh root@ 'rm -f /var/lib/anythingmcp-probe/maintenance' * * Why not the catalog re-sync: the change is structural (every endpoint * moved) so the boot-time reconciler will not apply it, the per-connector @@ -75,9 +81,15 @@ const json = (v) => `${lit(JSON.stringify(v))}::jsonb`; const out = []; out.push('-- deutsche-bahn: db-rest → MOTIS. Generated by scripts/ops/migrate-deutsche-bahn-cloud.mjs'); out.push(`-- adapterVersion ${adapterVersion}, MOTIS at ${motisUrl}`); +// Safe to re-run: a connector already on this adapterVersion and this MOTIS +// URL is left out of db_targets, so a second run reports it as already current +// and writes nothing (updated_at included). +const matches = `(config->>'adapterSlug' = 'deutsche-bahn' OR base_url LIKE ${lit(OLD_BASE + '%')})`; +const isCurrent = `(COALESCE(config->>'adapterVersion', '') = ${lit(adapterVersion)} AND base_url = ${lit(motisUrl)})`; out.push('CREATE TEMP TABLE db_targets AS'); out.push(' SELECT id FROM connectors'); -out.push(` WHERE config->>'adapterSlug' = 'deutsche-bahn' OR base_url LIKE ${lit(OLD_BASE + '%')};`); +out.push(` WHERE ${matches} AND NOT ${isCurrent};`); +out.push(`SELECT count(*) AS already_current FROM connectors WHERE ${matches} AND ${isCurrent};`); out.push("SELECT count(*) AS connectors_to_migrate FROM db_targets;"); out.push('UPDATE connectors SET'); diff --git a/scripts/ops/migrate-vinted-cloud.mjs b/scripts/ops/migrate-vinted-cloud.mjs index 0589ffd3..3c9c7aa8 100644 --- a/scripts/ops/migrate-vinted-cloud.mjs +++ b/scripts/ops/migrate-vinted-cloud.mjs @@ -27,14 +27,27 @@ * A connector is recognised by `config.adapterSlug`, or — for installs that * predate the baseline — by a www.vinted.* base URL. * - * Run it inside the app container, which holds ENCRYPTION_KEY and DATABASE_URL, + * Safe to re-run. A connector already at the target — base URL, LOGIN_TOKEN + * with this adapter's authConfig, `config.adapterVersion` equal to this + * adapter's, every catalog tool present, live and matching the catalog + * (description, parameters, endpoint, useProxy off), and no live tool the + * catalog dropped — is listed as "already current" and not written. Without + * this check a dry run after --apply listed every connector as still to do. + * + * Run it inside the backend container, which holds ENCRYPTION_KEY and DATABASE_URL, * after the release carrying the new adapter is deployed: * - * docker cp scripts/ops/migrate-vinted-cloud.mjs amcp-cloud-app:/app/backend/migrate-vinted.mjs - * docker cp packages/backend/src/adapters/intl/vinted.json amcp-cloud-app:/app/backend/vinted.json - * docker exec -w /app/backend amcp-cloud-app node migrate-vinted.mjs vinted.json # dry run - * docker exec -w /app/backend amcp-cloud-app node migrate-vinted.mjs vinted.json --apply - * docker restart amcp-cloud-app # the tool registry caches connector config + * docker cp scripts/ops/migrate-vinted-cloud.mjs amcp-cloud-backend:/app/backend/migrate-vinted.mjs + * docker cp packages/backend/src/adapters/intl/vinted.json amcp-cloud-backend:/app/backend/vinted.json + * docker exec -w /app/backend amcp-cloud-backend node migrate-vinted.mjs vinted.json # dry run + * docker exec -w /app/backend amcp-cloud-backend node migrate-vinted.mjs vinted.json --apply + * # These three on the droplet host, not in the container. The restart takes the API + * # down for ~30-60 s; silence the uptime probe first. It honours an expiry + * # epoch in this file (deploy/cloud/uptime-probe.sh, as deploy-cloud.yml + * # does), so a forgotten marker lapses by itself after 10 minutes: + * mkdir -p /var/lib/anythingmcp-probe && echo $(( $(date -u +%s) + 600 )) > /var/lib/anythingmcp-probe/maintenance + * docker restart amcp-cloud-backend # the tool registry caches connector config + * rm -f /var/lib/anythingmcp-probe/maintenance * * Prints connector ids and what changed; never secrets. */ @@ -57,7 +70,7 @@ if (adapter.slug !== 'vinted' || adapter.connector.authType !== 'LOGIN_TOKEN') { const KEY = process.env.ENCRYPTION_KEY; if (!KEY) { - console.error('ENCRYPTION_KEY is not set — run this inside the app container.'); + console.error('ENCRYPTION_KEY is not set — run this inside the backend container.'); process.exit(1); } @@ -137,9 +150,53 @@ const rows = await prisma.connector.findMany({ const catalogByName = new Map(adapter.tools.map((t) => [t.name, t])); const authConfig = encrypt(JSON.stringify(adapter.connector.authConfig)); const hasMapping = (m) => m !== null && m !== undefined; +const same = (a, b) => JSON.stringify(canonicalize(a ?? null)) === JSON.stringify(canonicalize(b ?? null)); +const targetAuth = canonicalize(adapter.connector.authConfig); + +/** Stored authConfig equals the adapter's. Undecryptable → not current. */ +function authIsCurrent(stored) { + if (!stored) return false; + try { + return same(JSON.parse(decrypt(stored)), targetAuth); + } catch { + return false; + } +} + +/** Every reason the row is not yet at the target state; empty when it is. */ +function pendingChanges(c, cfg) { + const why = []; + if (c.baseUrl !== adapter.connector.baseUrl) why.push('baseUrl'); + if (c.authType !== 'LOGIN_TOKEN' || !authIsCurrent(c.authConfig)) why.push('auth'); + if (cfg.adapterSlug !== 'vinted' || cfg.adapterVersion !== adapterVersion) why.push('adapterVersion'); + const byName = new Map(c.tools.map((t) => [t.name, t])); + for (const ct of adapter.tools) { + const et = byName.get(ct.name); + if ( + !et || + et.deprecatedAt || + et.useProxy || + et.description !== ct.description || + !same(et.parameters, ct.parameters) || + !same(et.endpointMapping, ct.endpointMapping) + ) { + why.push(`tool ${ct.name}`); + } + } + for (const t of c.tools) if (!t.deprecatedAt && !catalogByName.has(t.name)) why.push(`tool -${t.name}`); + return why; +} + +let migrated = 0; +let current = 0; for (const c of rows) { const cfg = c.config && typeof c.config === 'object' ? c.config : {}; + if (pendingChanges(c, cfg).length === 0) { + console.log(`= ${c.id} — already current`); + current++; + continue; + } const baseline = typeof cfg.instructionsBaseline === 'string' ? cfg.instructionsBaseline : null; const userEdited = baseline !== null && baseline !== hashInstructions(c.instructions); const notes = [`baseUrl ${c.baseUrl} → ${adapter.connector.baseUrl}`, `auth ${c.authType} → LOGIN_TOKEN`]; @@ -213,9 +270,13 @@ for (const c of rows) { await prisma.$transaction([connectorUpdate, ...updates, ...creates, ...deprecates]); } console.log(`${APPLY ? '✓' : '→'} ${c.id} — ${notes.join(', ')}`); + migrated++; } -console.log(`\n${APPLY ? 'Migrated' : 'Would migrate'} ${rows.length} Vinted connectors (adapterVersion ${adapterVersion}).`); -if (!APPLY && rows.length > 0) console.log('Re-run with --apply, then restart the app.'); +console.log( + `\n${APPLY ? 'Migrated' : 'Would migrate'} ${migrated} of ${rows.length} Vinted connectors ` + + `(${current} already current; adapterVersion ${adapterVersion}).`, +); +if (!APPLY && migrated > 0) console.log('Re-run with --apply, then restart the backend (see the header).'); await prisma.$disconnect(); diff --git a/scripts/ops/resync-catalog-connectors.mjs b/scripts/ops/resync-catalog-connectors.mjs index 9df5b204..d40a3bf6 100644 --- a/scripts/ops/resync-catalog-connectors.mjs +++ b/scripts/ops/resync-catalog-connectors.mjs @@ -36,16 +36,22 @@ * pointed somewhere else on purpose (a proxy, a sandbox) is left alone and * listed. * - * Run it inside the app container AFTER the release carrying the adapter + * Run it inside the backend container AFTER the release carrying the adapter * change is deployed — the diff is computed against the catalog compiled into * that container: * - * docker cp scripts/ops/resync-catalog-connectors.mjs amcp-cloud-app:/app/backend/resync.mjs - * docker exec -w /app/backend amcp-cloud-app node resync.mjs buffer \ + * docker cp scripts/ops/resync-catalog-connectors.mjs amcp-cloud-backend:/app/backend/resync.mjs + * docker exec -w /app/backend amcp-cloud-backend node resync.mjs buffer \ * --base-url-from=https://graphql.buffer.com,https://api.buffer.com/graphql # dry run - * docker exec -w /app/backend amcp-cloud-app node resync.mjs buffer \ + * docker exec -w /app/backend amcp-cloud-backend node resync.mjs buffer \ * --base-url-from=https://graphql.buffer.com,https://api.buffer.com/graphql --apply - * docker restart amcp-cloud-app # the tool registry caches connector tools + * # These three on the droplet host, not in the container. The restart takes the API + * # down for ~30-60 s; silence the uptime probe first. It honours an expiry + * # epoch in this file (deploy/cloud/uptime-probe.sh, as deploy-cloud.yml + * # does), so a forgotten marker lapses by itself after 10 minutes: + * mkdir -p /var/lib/anythingmcp-probe && echo $(( $(date -u +%s) + 600 )) > /var/lib/anythingmcp-probe/maintenance + * docker restart amcp-cloud-backend # the tool registry caches connector tools + * rm -f /var/lib/anythingmcp-probe/maintenance * * Prints connector ids and what changed; no credentials, no customer data. */ @@ -76,7 +82,7 @@ if (!slug || !getAdapter(slug)) { process.exit(1); } if (!process.env.DATABASE_URL) { - console.error('DATABASE_URL is not set — run this inside the app container.'); + console.error('DATABASE_URL is not set — run this inside the backend container.'); process.exit(1); } @@ -151,6 +157,6 @@ console.log( for (const { id } of unmanaged) { console.log(`! ${id}: on an old ${slug} base URL but not catalog-managed (no adapterSlug) — not touched`); } -if (!APPLY && counts.synced > 0) console.log('Re-run with --apply, then restart the app.'); +if (!APPLY && counts.synced > 0) console.log('Re-run with --apply, then restart the backend (see the header).'); await prisma.$disconnect(); diff --git a/scripts/probe-keyless.mjs b/scripts/probe-keyless.mjs index 073ebab2..c0944e5f 100644 --- a/scripts/probe-keyless.mjs +++ b/scripts/probe-keyless.mjs @@ -13,7 +13,7 @@ * * Run it from a datacenter address (CI runner, the cloud droplet) — a * residential IP proves nothing. Dependency-free on purpose so it can be - * copied into the app container and run there. + * copied into the backend container and run there. * * node scripts/probe-keyless.mjs # table for every keyless adapter * node scripts/probe-keyless.mjs --check # exit 1 if a non-selfHostOnly one fails @@ -30,6 +30,21 @@ * `{{VAR}}` placeholders in the base URL are filled from PROBE_ in the * environment (e.g. PROBE_MOTIS_URL for deutsche-bahn); an unfilled one is * reported as `skipped`. + * + * "Keyless" means the user supplies nothing, not `authType: NONE`. An adapter + * whose auth is fully described by its own JSON — no `{{VAR}}` in authConfig, + * no requiredEnvVars beyond operator-provided ones — is probed too, with its + * auth reproduced here. Vinted is the case that made this necessary: it moved + * to LOGIN_TOKEN with an anonymous session (HEAD the catalog page, read the + * `access_token_web` cookie, send it as a Bearer token), and the NONE-only + * filter silently dropped it from the weekly run. Supported: + * - NONE + * - LOGIN_TOKEN, mirroring login-token.service.ts: tokenSource `cookie` + * (last non-empty Set-Cookie value, as a browser keeps it) or `body` + * (tokenJsonPath), then headerName/headerTemplate/extraHeaders as in + * injectLoginTokenHeaders(). Password hashing (bcrypt) is not. + * Anything else with static credentials is listed as `unsupported-auth`, a + * warning like `no-probe`, so it shows up instead of vanishing. */ import { readdirSync, readFileSync, statSync } from 'node:fs'; import { join, dirname } from 'node:path'; @@ -65,11 +80,21 @@ function loadAdapters() { return out; } +/** + * No user-supplied credentials: every required env var is operator-provided, + * and — for any auth type other than NONE — the authConfig carries no `{{VAR}}` + * placeholder, i.e. the adapter JSON alone is enough to authenticate. + */ function isKeyless(a) { - if (a.connector?.authType !== 'NONE') return false; - return (a.requiredEnvVars || []).every((v) => OPERATOR_PROVIDED.has(v)); + if (!(a.requiredEnvVars || []).every((v) => OPERATOR_PROVIDED.has(v))) return false; + const authType = a.connector?.authType; + if (!authType || authType === 'NONE') return true; + return !/\{\{\w+\}\}/.test(JSON.stringify(a.connector.authConfig ?? {})); } +const PROBED_AUTH = new Set(['NONE', 'LOGIN_TOKEN']); +const USER_AGENT = 'anythingmcp/1.0 (+https://anythingmcp.com)'; + /** Mirror of RestEngine.resolveValue for the subset a probe needs. */ function resolveValue(value, params) { if (typeof value === 'string') { @@ -130,6 +155,117 @@ function pickProbe(a) { return { tool, params }; } +/** Mirror of jsonPath() in login-token.service.ts. */ +function jsonPath(value, path) { + let cur = value; + for (const p of path.replace(/\[(\d+)\]/g, '.$1').split('.').filter(Boolean)) { + if (cur === undefined || cur === null) return undefined; + cur = Array.isArray(cur) ? cur[Number(p)] : typeof cur === 'object' ? cur[p] : undefined; + } + return cur; +} + +/** Mirror of interpolateDeep() in login-token.service.ts. */ +function interpolateDeep(value, params) { + if (typeof value === 'string') { + const full = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/.exec(value); + if (full) return params[full[1]] ?? ''; + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, n) => params[n] ?? ''); + } + if (Array.isArray(value)) return value.map((v) => interpolateDeep(v, params)); + if (value && typeof value === 'object') { + const o = {}; + for (const [k, v] of Object.entries(value)) o[k] = interpolateDeep(v, params); + return o; + } + return value; +} + +/** + * Last non-empty value of a cookie across every Set-Cookie header — mirror of + * extractSetCookieValue(). Vinted clears `access_token_web` on one domain and + * sets the real one on another in the same response; the first match is empty. + */ +function lastSetCookie(headers, name) { + const all = typeof headers.getSetCookie === 'function' ? headers.getSetCookie() : [headers.get('set-cookie') || '']; + let found = null; + for (const entry of all) { + const t = entry.trimStart(); + if (!t.startsWith(`${name}=`)) continue; + const end = t.indexOf(';'); + const v = t.slice(name.length + 1, end === -1 ? undefined : end); + if (v) found = v; + } + return found; +} + +async function timedFetch(url, init) { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS); + try { + return await fetch(url, { ...init, signal: ctrl.signal, redirect: 'follow' }); + } finally { + clearTimeout(timer); + } +} + +/** + * Obtain a LOGIN_TOKEN the way login-token.service.ts performLogin() does and + * return the headers injectLoginTokenHeaders() would add. Throws an Error with + * a `verdict` on failure; never prints the token. + */ +async function loginTokenHeaders(cfg) { + const fail = (verdict, note, status) => Object.assign(new Error(note), { verdict, status }); + if (cfg.passwordHashing && cfg.passwordHashing.scheme !== 'none') { + throw fail('unsupported-auth', `LOGIN_TOKEN passwordHashing ${cfg.passwordHashing.scheme}`); + } + if (!cfg.loginUrl) throw fail('bad-probe', 'LOGIN_TOKEN without loginUrl'); + const params = { username: cfg.username ?? '', password: cfg.password ?? '', passwordHashed: cfg.password ?? '', aud: cfg.aud ?? '', otp: cfg.otp ?? '' }; + let data; + if (cfg.loginBody !== undefined) data = interpolateDeep(cfg.loginBody, params); + else if (cfg.loginBodyTemplate) data = JSON.parse(cfg.loginBodyTemplate.replace(/\$\{(\w+)\}/g, (_, n) => params[n] ?? '')); + else data = params; + + const method = String(cfg.loginMethod || 'POST').toUpperCase(); + const url = new URL(cfg.loginUrl); + let body; + // axios sends `data` as the query string for GET and drops it for HEAD. + if (method === 'GET' && data && typeof data === 'object') { + for (const [k, v] of Object.entries(data)) url.searchParams.set(k, String(v)); + } else if (method !== 'GET' && method !== 'HEAD' && data !== null && data !== undefined) { + body = JSON.stringify(data); + } + const headers = { 'User-Agent': USER_AGENT, 'Content-Type': 'application/json', ...(cfg.loginHeaders || {}) }; + + let res; + try { + res = await timedFetch(url, { method, headers, body }); + } catch (err) { + throw fail('login-failed', `login ${method} ${url.origin}${url.pathname}: ${err?.name === 'AbortError' ? 'timeout' : err?.cause?.code || err?.message}`); + } + const text = method === 'HEAD' ? '' : await res.text().catch(() => ''); + if (res.status >= 400) { + const v = classify(res.status, text, res.headers); + throw fail(v === 'bot-blocked' ? 'bot-blocked' : 'login-failed', `login ${method} ${url.origin}${url.pathname} answered HTTP ${res.status}`, res.status); + } + + let token; + if (cfg.tokenSource === 'cookie') { + token = lastSetCookie(res.headers, cfg.cookieName); + if (!token) throw fail('login-failed', `login set no non-empty "${cfg.cookieName}" cookie`, res.status); + } else { + let json; + try { json = JSON.parse(text); } catch { /* not JSON */ } + token = jsonPath(json, cfg.tokenJsonPath || ''); + if (!token || typeof token !== 'string') throw fail('login-failed', `no token at "${cfg.tokenJsonPath}" in login response`, res.status); + } + const aud = cfg.audJsonPath ? undefined : cfg.aud; + const fill = (s) => s.replace(/\$\{token\}/g, token).replace(/\$\{aud\}/g, aud || ''); + const out = { [cfg.headerName || 'Authorization']: fill(cfg.headerTemplate || 'Bearer ${token}') }; + for (const [k, v] of Object.entries(cfg.extraHeaders || {})) out[k] = fill(String(v)); + return out; +} + function fillTemplate(str, env) { const missing = []; const out = str.replace(/\{\{(\w+)\}\}/g, (_, name) => { @@ -161,6 +297,8 @@ function classify(status, bodyText, headers) { } async function probe(a) { + const authType = a.connector.authType || 'NONE'; + if (!PROBED_AUTH.has(authType)) return { slug: a.slug, verdict: 'unsupported-auth', note: `authType ${authType} is not reproduced by this probe` }; const picked = pickProbe(a); if (picked.error) return { slug: a.slug, verdict: picked.error === 'no-probe' ? 'no-probe' : 'bad-probe', note: picked.error }; const { tool, params } = picked; @@ -177,7 +315,7 @@ async function probe(a) { const q = resolveValue(em.queryParams || {}, params); for (const [k, v] of Object.entries(q)) url.searchParams.set(k, String(v)); - const headers = { 'User-Agent': 'anythingmcp/1.0 (+https://anythingmcp.com)' }; + const headers = { 'User-Agent': USER_AGENT }; for (const [k, v] of Object.entries(a.connector.headers || {})) headers[k] = fillTemplate(String(v), process.env).out; for (const [k, v] of Object.entries(em.headers || {})) headers[k] = String(resolveValue(v, params) ?? ''); @@ -192,19 +330,25 @@ async function probe(a) { if (!Object.keys(headers).some((h) => h.toLowerCase() === 'content-type')) headers['Content-Type'] = 'application/json'; } - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS); const started = Date.now(); + let authNote; + if (authType === 'LOGIN_TOKEN') { + try { + Object.assign(headers, await loginTokenHeaders(a.connector.authConfig || {})); + authNote = 'anonymous LOGIN_TOKEN'; + } catch (err) { + return { slug: a.slug, tool: tool.name, status: err.status ?? 0, ms: Date.now() - started, verdict: err.verdict || 'login-failed', note: err.message }; + } + } + try { - const res = await fetch(url, { method, headers, body, signal: ctrl.signal, redirect: 'follow' }); + const res = await timedFetch(url, { method, headers, body }); const text = await res.text().catch(() => ''); const verdict = classify(res.status, text, res.headers); - return { slug: a.slug, tool: tool.name, status: res.status, ms: Date.now() - started, verdict, url: url.origin + url.pathname }; + return { slug: a.slug, tool: tool.name, status: res.status, ms: Date.now() - started, verdict, note: authNote, url: url.origin + url.pathname }; } catch (err) { const note = err?.name === 'AbortError' ? `timeout after ${TIMEOUT_MS / 1000}s` : String(err?.cause?.code || err?.message || err); return { slug: a.slug, tool: tool.name, status: 0, ms: Date.now() - started, verdict: 'unreachable', note, url: url.origin + url.pathname }; - } finally { - clearTimeout(timer); } } @@ -224,17 +368,19 @@ for (const a of adapters) { ); } -const FAIL = new Set(['bot-blocked', 'auth-required', 'upstream-error', 'unreachable', 'bad-probe']); +const FAIL = new Set(['bot-blocked', 'auth-required', 'login-failed', 'upstream-error', 'unreachable', 'bad-probe']); const failing = results.filter((r) => FAIL.has(r.verdict) && !r.selfHostOnly); const noProbe = results.filter((r) => r.verdict === 'no-probe'); +const unsupported = results.filter((r) => r.verdict === 'unsupported-auth'); console.log(''); console.log( `${results.length} keyless adapters probed: ${results.filter((r) => r.verdict === 'ok').length} ok, ` + - `${failing.length} failing, ${noProbe.length} without a probe, ` + + `${failing.length} failing, ${noProbe.length} without a probe, ${unsupported.length} with unsupported auth, ` + `${results.filter((r) => r.verdict === 'skipped').length} skipped.`, ); if (noProbe.length) console.log(`No probe (add a "probe" field): ${noProbe.map((r) => r.slug).join(', ')}`); +if (unsupported.length) console.log(`Static credentials this probe cannot reproduce yet: ${unsupported.map((r) => r.slug).join(', ')}`); if (CHECK && failing.length) { for (const r of failing) {