fix(service-datasource): an adopted plugin driver's failed-credentials verdict clears on the next granted borrow, and the connector path withholds the form-encoded credential spelling

.changeset/27919-borrowed-credential-verdict-retracts.md

@@ -0,0 +1,14 @@
+---
+'@objectstack/types': patch
+'@objectstack/service-automation': patch
+'@objectstack/service-datasource': patch
+---
+
+fix: a plugin-driver datasource refused at boot for its connector instance's credential recovers on the next granted borrow, and the connector call path withholds the form-encoded spelling of a credential
+
+Clause-②: yes (widening)
+Metadata-protocol: none
+
+- **`@objectstack/service-datasource`**: a datasource served by a plugin driver (driver-odata) that names `external.connector`, whose instance could not lend at boot (undeclared, disabled, no `auth`), was recorded `failed-credentials` and marked unavailable until a restart, although the driver borrows again on every request and served as soon as the instance was repaired. The first borrow the lender grants for that instance now retracts the verdict and the engine's unavailable mark, so Setup → Datasources shows it available again with no restart. A grant for any other instance leaves it standing. The degraded-boot banner says which recovery applies; with no lender registered the verdict still lasts until a restart.
+- **`@objectstack/types`**: `IConnectorCredentialSource` gains an optional `onLent(listener)`, called with the instance's name on every granted borrow and returning the unsubscribe. A source without it keeps working; nothing then retracts such a verdict.
+- **`@objectstack/service-automation`**: the connector credential source implements `onLent`. The connector call path's `credentialForms` now takes its list from `borrowedCredentialForms`, so an upstream that echoes an API key in a form-encoded query string (a space written `+`) has that spelling withheld from the degraded reason, the stored status and the logs, as a datasource borrowing the same credential already had.

packages/services/service-automation/src/connector-credential-door.test.ts

@@ -682,4 +682,43 @@ describe('[#27237] contract review: an encoded spelling of the value is withheld
             }
         }
     });
+
+    // [ADR-0015 D7] The connector path takes its spellings from the shared
+    // `borrowedCredentialForms`, so the form encoding `URLSearchParams` writes
+    // (a space as `+`, which `encodeURIComponent` spells `%20`) is withheld here
+    // as it is for a datasource borrowing the same credential.
+    it('an api-key instance whose upstream echoes the form-encoded query string: no spelling reaches the reason, the row or a log', async () => {
+        const formEncoded = new URLSearchParams({ v: ENCODABLE }).toString().slice(2);
+        expect(formEncoded).not.toBe(encodeURIComponent(ENCODABLE));
+        expect(credentialForms(ENCODABLE)).toContain(formEncoded);
+
+        const out = captureOutput();
+        const provider = fakeProvider();
+        const factory: ConnectorProviderFactory = (ctx) => {
+            provider.state.materialized.push(ctx);
+            const key = ctx.auth?.type === 'api-key' ? ctx.auth.key : '';
+            const echo = `GET https://erp.example/orders?${new URLSearchParams({ api_key: key })} was refused`;
+            return {
+                def: { name: ctx.name, label: ctx.label, type: 'api', actions: [{ key: 'request', label: 'Request' }] } as unknown as Connector,
+                handlers: { request: async () => ({ status: 401, ok: false, body: { message: echo } }) },
+            };
+        };
+        const erp = { ...billing('erp'), auth: { type: 'api-key', paramName: 'api_key', credentialRef: REF } };
+        const { engine, call } = await boot([erp], { provider: { ...provider, factory } });
+
+        await engine.setConnectorCredential({ ref: REF, value: ENCODABLE, context: ADMIN });
+        expect(await call('erp')).toMatchObject({ status: 401 });
+        const reason = engine.getConnectorDegradedReason('erp') ?? '';
+        expect(reason).toContain('[credential withheld]');
+
+        for (const [where, text] of [
+            ['reason on a call', reason],
+            ['listing after the call', JSON.stringify(await engine.listConnectorCredentialRefs())],
+            ['logs', out.text()],
+        ] as const) {
+            for (const form of [formEncoded, encodeURIComponent(ENCODABLE), ENCODABLE]) {
+                expect(text.includes(form), `${where}: a spelling of the value`).toBe(false);
+            }
+        }
+    });
 });

packages/services/service-automation/src/connector-credential-source.test.ts

@@ -259,3 +259,42 @@ describe('[ADR-0015 D7] createConnectorCredentialSource — the refusals of a ha
         expect(reads).toBe(1);
     });
 });
+
+describe('[ADR-0015 D7] createConnectorCredentialSource — onLent announces each granted borrow', () => {
+    const entry = sap();
+    const source = (readDeclared: () => Promise<unknown[]>) => createConnectorCredentialSource({
+        readDeclared,
+        chain: async () => ({ env: () => DUMMY_VALUE }),
+        vaultWired: () => false,
+        generation: () => 0,
+    });
+
+    it('names the instance on a lend, never on a refusal, and stops after the unsubscribe', async () => {
+        let declared: unknown[] = [{ ...entry, enabled: false }];
+        const lender = source(async () => declared);
+        const lent: string[] = [];
+        const unsubscribe = lender.onLent!((connector) => lent.push(connector));
+
+        await expect(lender.borrow('sap_s4')).rejects.toMatchObject({ status: 422 });
+        await expect(lender.borrow('not_declared')).rejects.toMatchObject({ status: 422 });
+        expect(lent).toEqual([]);
+
+        declared = [entry];
+        await lender.borrow('sap_s4');
+        expect(lent).toEqual(['sap_s4']);
+
+        unsubscribe();
+        await lender.borrow('sap_s4');
+        expect(lent).toEqual(['sap_s4']);
+    });
+
+    it('grants the borrow when a listener throws', async () => {
+        const lender = source(async () => [entry]);
+        const later: string[] = [];
+        lender.onLent!(() => { throw new Error('listener fault'); });
+        lender.onLent!((connector) => later.push(connector));
+        const handle = await lender.borrow('sap_s4');
+        expect(handle.connector).toBe('sap_s4');
+        expect(later).toEqual(['sap_s4']);
+    });
+});

packages/services/service-automation/src/connector-credential-source.ts

@@ -29,6 +29,11 @@
  * path runs, or on a `refresh()`. The grant-bearing kinds ADR-0143 adds answer
  * through the same handle, per dispatch, when they land.
  *
+ * Each granted borrow is announced to the source's `onLent` subscribers by
+ * the instance's name alone, so a datasource the boot recorded as unable to
+ * borrow is re-judged the moment its instance lends again (the datasource
+ * connection service subscribes).
+ *
  * ⛔ A value leaves this module only as the return of a handle's `get` /
  * `refresh`. Every refusal names the instance and the reference, never a
  * value, and a host resolver's own message is not interpolated.
@@ -195,10 +200,27 @@ export function createConnectorCredentialSource(deps: ConnectorCredentialSourceD
         return item as Lendable;
     };
 
+    const lentListeners = new Set<(connector: string) => void>();
+    /** Tell every `onLent` subscriber; a listener's throw stays its own. */
+    const announceLent = (connector: string): void => {
+        for (const listener of [...lentListeners]) {
+            try {
+                listener(connector);
+            } catch {
+                /* the borrow was granted either way */
+            }
+        }
+    };
+
     return {
+        onLent(listener: (connector: string) => void): () => void {
+            lentListeners.add(listener);
+            return () => void lentListeners.delete(listener);
+        },
         async borrow(connector: string): Promise<ConnectorCredentialHandle> {
             const refusal = lendRefusal(connector, findDeclared(await readDeclared(connector), connector));
             if (refusal) throw connectorCredentialRefusal(refusal, 'VALIDATION_ERROR', 422);
+            announceLent(connector);
 
             let held: { generation: number; credential: BorrowedConnectorCredential } | undefined;
             let inflight: Promise<BorrowedConnectorCredential> | undefined;

packages/services/service-automation/src/connector-credentials.ts

@@ -27,6 +27,7 @@
  * a status and a revision, never the value.
  */
 
+import { borrowedCredentialForms } from '@objectstack/types';
 import { defaultEnvCredentialResolver, type CredentialResolver } from './credential-resolver.js';
 
 /** The object a connector credential lives in (`sys-connector-credential.object.ts`). */
@@ -390,27 +391,22 @@ export type WithheldCredential = string | readonly string[] | undefined;
 /** The shortest spelling a scrub withholds — a shorter one would blank ordinary words. */
 const MIN_WITHHELD_LENGTH = 4;
 
-function base64Of(text: string): string {
-    const bytes = new TextEncoder().encode(text);
-    let binary = '';
-    for (const byte of bytes) binary += String.fromCharCode(byte);
-    return btoa(binary);
-}
-
 /**
- * Every spelling of `value` an upstream or a client library may echo back:
- * the value itself; URL-encoded, as an api-key sent as a query parameter
- * travels in the request URL; base64; and, given the `basic` `username`,
- * base64(`username:value`), the `Authorization: Basic` header. A base64 form
- * is kept without its `=` padding, so an echo that drops the padding is
- * withheld too. Longest first, so a longer form is replaced before a shorter
- * one inside it.
+ * Every spelling of `value` an upstream or a client library may echo back —
+ * the one rule `borrowedCredentialForms` (`@objectstack/types`) holds for a
+ * datasource borrowing this credential, so the connector's own call path and
+ * the borrowing driver withhold the same list: the value itself; URL-encoded
+ * both ways (`encodeURIComponent`, and the form encoding `URLSearchParams`
+ * writes), as an api-key sent as a query parameter travels in the request
+ * URL; base64; and, given the `basic` `username`, base64(`username:value`),
+ * the `Authorization: Basic` header. Base64 forms are unpadded and the list
+ * is longest first.
  */
 export function credentialForms(value: string | undefined, username?: string): string[] {
     if (!value) return [];
-    const forms = new Set<string>([value, encodeURIComponent(value), base64Of(value).replace(/=+$/, '')]);
-    if (username !== undefined) forms.add(base64Of(`${username}:${value}`).replace(/=+$/, ''));
-    return [...forms].filter((f) => f.length >= MIN_WITHHELD_LENGTH).sort((a, b) => b.length - a.length);
+    return borrowedCredentialForms(
+        username !== undefined ? { type: 'basic', username, password: value } : { type: 'bearer', token: value },
+    );
 }
 
 function withheldForms(secret: WithheldCredential): readonly string[] {

packages/services/service-datasource/src/__tests__/datasource-connector-credential.test.ts

@@ -1,6 +1,6 @@
 // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
 
-import { describe, it, expect, vi } from 'vitest';
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 import { ObjectQL } from '@objectstack/objectql';
 import {
   CONNECTOR_CREDENTIAL_SOURCE_SERVICE,
@@ -393,6 +393,137 @@ describe('[ADR-0015 D7] a plugin driver adopted from its driver.<name> service i
   });
 });
 
+/**
+ * A lender double whose declaration can be repaired, with `onLent`, as the
+ * automation service's source has it: a borrow of a declared, enabled
+ * instance is granted and announced to the subscribers.
+ */
+function repairableLender(disabled: Set<string>) {
+  const listeners = new Set<(connector: string) => void>();
+  const source: IConnectorCredentialSource = {
+    async borrow(connector) {
+      if (disabled.has(connector)) {
+        throw connectorCredentialRefusal(
+          `connector instance '${connector}' (named by this datasource's external.connector) is disabled (enabled: false).`,
+          'VALIDATION_ERROR',
+          422,
+        );
+      }
+      for (const listener of listeners) listener(connector);
+      const fixed: BorrowedConnectorCredential = { auth: { type: 'bearer', token: 'static-key' } };
+      return { connector, get: async () => fixed, refresh: async () => fixed };
+    },
+    onLent(listener) {
+      listeners.add(listener);
+      return () => void listeners.delete(listener);
+    },
+  };
+  return { source, listeners };
+}
+
+describe('[ADR-0015 D7] an adopted plugin driver\'s failed-credentials verdict clears on the next granted borrow', () => {
+  const ENV = 'OS_ALLOW_DRIVER_CONNECT_FAILURE';
+  let saved: string | undefined;
+  beforeEach(() => { saved = process.env[ENV]; delete process.env[ENV]; });
+  afterEach(() => {
+    if (saved === undefined) delete process.env[ENV];
+    else process.env[ENV] = saved;
+  });
+
+  it('retracts the verdict and the unavailable mark once the repaired instance lends — and only for that instance', async () => {
+    const disabled = new Set(['sap_s4']);
+    const { source } = repairableLender(disabled);
+    const { ql, connection } = await adoptedPluginDriver(source);
+    const result = await connection.connect(sapOrders, { context: { origin: 'code', trigger: 'declared-auto' } });
+    expect(result.status).toBe('failed-credentials');
+    expect(ql.listUnavailableDatasources()).toEqual([expect.objectContaining({ name: 'sap_orders' })]);
+
+    // A grant for another instance proves nothing about this one.
+    await source.borrow('salesforce_crm');
+    expect(connection.getConnectionState('sap_orders')).toMatchObject({ status: 'failed-credentials', availability: 'failed' });
+    expect(ql.listUnavailableDatasources()).toHaveLength(1);
+
+    // The instance is repaired, and the driver's next request borrows it.
+    disabled.delete('sap_s4');
+    await source.borrow('sap_s4');
+    expect(connection.getConnectionState('sap_orders')).toMatchObject({ status: 'already-registered', availability: 'available' });
+    expect(connection.getConnectionState('sap_orders')?.reason).toBeUndefined();
+    expect(ql.listUnavailableDatasources()).toEqual([]);
+    expect(ql.getDriverByName('sap_orders')).toBeDefined();
+  });
+
+  it('subscribes once per lender, however many datasources wait on it', async () => {
+    const { source, listeners } = repairableLender(new Set(['sap_s4']));
+    const { svc, engine } = service({ supports: () => false, create: vi.fn() } as unknown as IDatasourceDriverFactory, source);
+    for (const name of ['sap_orders', 'sap_invoices']) {
+      engine.drivers.set(name, { name } as IDataDriver);
+      await svc.connect({ ...sapOrders, name }, { context: { origin: 'code', trigger: 'declared-auto' } });
+    }
+    expect(listeners.size).toBe(1);
+    await svc.disconnectAll();
+    expect(listeners.size).toBe(0);
+  });
+
+  it('leaves a verdict standing that a disconnect or a later connect replaced', async () => {
+    const disabled = new Set(['sap_s4']);
+    const { source } = repairableLender(disabled);
+    const { svc, engine } = service({ supports: () => false, create: vi.fn() } as unknown as IDatasourceDriverFactory, source);
+    engine.drivers.set('sap_orders', { name: 'sap_orders' } as IDataDriver);
+    await svc.connect(sapOrders, { context: { origin: 'code', trigger: 'declared-auto' } });
+    await svc.disconnect('sap_orders');
+    disabled.delete('sap_s4');
+    await source.borrow('sap_s4');
+    // Nothing resurrected for a datasource that was removed.
+    expect(svc.getConnectionState('sap_orders')).toBeUndefined();
+  });
+
+  it('leaves the verdict when the driver was evicted since, rather than reporting a driver that is gone', async () => {
+    const disabled = new Set(['sap_s4']);
+    const { source } = repairableLender(disabled);
+    const { svc, engine } = service({ supports: () => false, create: vi.fn() } as unknown as IDatasourceDriverFactory, source);
+    engine.drivers.set('sap_orders', { name: 'sap_orders' } as IDataDriver);
+    await svc.connect(sapOrders, { context: { origin: 'code', trigger: 'declared-auto' } });
+    engine.drivers.delete('sap_orders');
+    disabled.delete('sap_s4');
+    await source.borrow('sap_s4');
+    expect(svc.getConnectionState('sap_orders')?.status).toBe('failed-credentials');
+  });
+
+  it('says in the degraded-boot banner that the next granted borrow clears it, and a restart where nothing can', async () => {
+    process.env[ENV] = '1';
+    const banners = async (source: IConnectorCredentialSource) => {
+      const warnings: string[] = [];
+      const engine = fakeEngine();
+      engine.drivers.set('sap_orders', { name: 'sap_orders' } as IDataDriver);
+      const svc = new DatasourceConnectionService({
+        factory: () => ({ supports: () => false, create: vi.fn() }) as unknown as IDatasourceDriverFactory,
+        engine: () => engine,
+        connectorCredentials: () => source,
+        logger: { info() {}, debug() {}, error() {}, warn: (m: string) => void warnings.push(m) } as never,
+      });
+      const realWrite = process.stderr.write;
+      (process.stderr as { write: unknown }).write = () => true;
+      try {
+        const result = await svc.connect(sapOrders, { objects: ['sap_order'], context: { origin: 'code', trigger: 'declared-auto' } });
+        expect(result.status).toBe('failed-credentials');
+      } finally {
+        (process.stderr as { write: unknown }).write = realWrite;
+      }
+      return warnings.join('\n');
+    };
+
+    const retractable = await banners(repairableLender(new Set(['sap_s4'])).source);
+    expect(retractable).toContain('DEGRADED BOOT');
+    expect(retractable).toContain("the first borrow granted for 'sap_s4' clears this verdict");
+    expect(retractable).not.toContain('server is restarted');
+
+    // A lender without onLent: nothing can retract the verdict, and the banner keeps the restart.
+    const { onLent: _omitted, ...withoutOnLent } = repairableLender(new Set(['sap_s4'])).source;
+    const standing = await banners(withoutOnLent);
+    expect(standing).toContain('until it is reachable AND the server is restarted');
+  });
+});
+
 describe('[ADR-0015 D7] borrowConnectorCredential', () => {
   it('never throws: a source that throws a non-Error comes back as a reason', async () => {
     const source = { borrow: async () => { throw 'boom'; } } as unknown as IConnectorCredentialSource;