beyondnetcode · beyondnetPeru · Jul 31, 2026 · Jul 31, 2026 · Jul 31, 2026
@@ -144,6 +144,17 @@ jobs:
       - name: Self-test for the joined-path guard
         run: node --test .harness/scripts/ci/47-validate-joined-paths.test.mjs
 
+      # GT-587. The OpenTelemetry GenAI conventions are Development-status, and the
+      # Core emits their attribute names as plain LITERALS (it must not import an
+      # observability SDK — rule HXA-05). Literals do not move when upstream moves, so
+      # a rename there turns Evolith's "standard" telemetry back into a private
+      # vocabulary with nothing going red. This is the check that goes red.
+      - name: Validate the pinned OpenTelemetry semconv revision
+        run: node .harness/scripts/ci/51-validate-semconv-pin.mjs --verbose
+
+      - name: Self-test for the semconv pin guard
+        run: node --test .harness/scripts/ci/51-validate-semconv-pin.test.mjs
+
       # GT-597. The gate itself runs inside `openssf-scorecard.yml`, where the
       # Scorecard JSON exists; only its logic is tested here. That logic is the
       # difference between a weekly score that is published and a weekly score

@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+
+/**
+ * GT-587 criterion 3 — the pinned OpenTelemetry semconv revision is DECLARED, and a
+ * drift in it is a red build.
+ *
+ * ## Why a pin needs a guard at all
+ *
+ * The GenAI semantic conventions are Development-status: upstream may rename or drop an
+ * attribute between releases. Evolith emits `gen_ai.*` and `mcp.*` names from
+ * `src/packages/core-domain/src/evaluation/telemetry/semconv.ts`, as plain literals —
+ * the Core is pure and must not import an observability SDK (rule HXA-05). Literals do
+ * not move when upstream moves. So the failure this guard exists to catch is not loud:
+ * the build stays green, the tests stay green, and the emitted telemetry quietly stops
+ * being the standard it claims to be. That is a SECOND private vocabulary wearing the
+ * costume of a standard one, which is worse than the private names it replaced.
+ *
+ * ## What it checks (three rules, each able to go red on its own)
+ *
+ *   1. **Version pin.** `SEMCONV_VERSION` must equal the `@opentelemetry/semantic-conventions`
+ *      version resolved in `package-lock.json`. Reading the LOCKFILE, not `node_modules`,
+ *      is deliberate: the lockfile is committed, so the check answers the same way on a
+ *      machine that has never run `npm install`, and cannot be made green by local state.
+ *   2. **Literal agreement.** Every symbol declared `exported` in `PINNED_SEMCONV_ATTRIBUTES`
+ *      must exist in the installed package with EXACTLY our value. A rename upstream, or a
+ *      typo here, fails.
+ *   3. **Manifest completeness.** Every `ATTR_*` / `EVENT_*` / `MCP_*_VALUE_*` constant the
+ *      module exports must appear in the manifest. Without this rule a new literal could be
+ *      added and never checked — the manifest would still pass while covering less.
+ *
+ * `registry-only` symbols (in the upstream registry, not yet exported by the package at the
+ * pinned version) are the interesting third state: they are ALLOWED to be missing, and they
+ * FAIL the day the package starts exporting them with a different value. That is exactly the
+ * moment a locally-declared literal silently becomes wrong.
+ *
+ * ## Anti-vacuous pass
+ *
+ * Zero pinned symbols, or a semconv module that cannot be read, is a hard exit 1 through the
+ * repository's own `assertScanned` — never "nothing to check, all good". `resolve()` is
+ * fail-closed, so a moved file is a loud error rather than an empty corpus.
+ *
+ * Usage:
+ *   node .harness/scripts/ci/51-validate-semconv-pin.mjs
+ *   node .harness/scripts/ci/51-validate-semconv-pin.mjs --verbose
+ */
+
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { runInNewContext } from 'node:vm';
+import { assertScanned } from '../lib/coverage.mjs';
+import { resolve } from '../lib/paths.mjs';
+
+const GUARD = '51-validate-semconv-pin';
+const VERBOSE = process.argv.slice(2).includes('--verbose');
+
+/** Path KEY, not a literal — `lib/paths.mjs` owns the location and fails closed if it moves. */
+const SEMCONV_MODULE_KEY = 'semconvPin';
+const SEMCONV_PACKAGE = '@opentelemetry/semantic-conventions';
+const LOCKFILE_KEY = `node_modules/${SEMCONV_PACKAGE}`;
+
+const require_ = createRequire(import.meta.url);
+
+/**
+ * Load the pinned vocabulary from SOURCE.
+ *
+ * Not from `dist/`: that is gitignored build output, so a check reading it would be
+ * measuring whatever the last local build happened to leave behind. TypeScript is a
+ * declared dependency of this repository, so transpiling the source is both hermetic
+ * and exact — a hand-rolled regex over `export const` lines would drift the first time
+ * someone writes a multi-line declaration.
+ */
+function loadPinnedVocabulary(file) {
+  const ts = require_('typescript');
+  const source = readFileSync(file, 'utf8');
+  const { outputText } = ts.transpileModule(source, {
+    compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
+  });
+  const module_ = { exports: {} };
+  runInNewContext(outputText, { module: module_, exports: module_.exports, require: require_ });
+  return module_.exports;
+}
+
+/** Version `npm ci` would install — read from the committed lockfile, not node_modules. */
+function lockedSemconvVersion() {
+  const lock = JSON.parse(readFileSync(resolve('rootPackageLock'), 'utf8'));
+  return lock.packages?.[LOCKFILE_KEY]?.version;
+}
+
+function main() {
+  const modulePath = resolve(SEMCONV_MODULE_KEY);
+  const vocabulary = loadPinnedVocabulary(modulePath);
+
+  const pinned = vocabulary.PINNED_SEMCONV_ATTRIBUTES ?? [];
+  assertScanned(pinned.length, {
+    what: 'pinned semconv symbols',
+    where: `${SEMCONV_MODULE_KEY} → PINNED_SEMCONV_ATTRIBUTES`,
+  });
+
+  const violations = [];
+
+  // --- Rule 1: the declared revision is the revision the lockfile installs ---------
+  const declared = vocabulary.SEMCONV_VERSION;
+  const locked = lockedSemconvVersion();
+  if (!locked) {
+    violations.push(
+      `${SEMCONV_PACKAGE} is not present in package-lock.json under '${LOCKFILE_KEY}'.\n` +
+        `      The pin cannot be verified against anything, which is indistinguishable from\n` +
+        `      not being pinned. Declare the dependency or remove the pin.`,
+    );
+  } else if (declared !== locked) {
+    violations.push(
+      `SEMCONV_VERSION drift: the module declares '${declared}', the lockfile installs '${locked}'.\n` +
+        `      The GenAI conventions are Development-status, so an upgrade may have renamed or\n` +
+        `      dropped an attribute. Re-verify every literal in PINNED_SEMCONV_ATTRIBUTES against\n` +
+        `      ${SEMCONV_PACKAGE}@${locked}, THEN bump SEMCONV_VERSION — never the other way round.`,
+    );
+  }
+
+  // --- Rule 2: each pinned literal still matches upstream --------------------------
+  let upstream;
+  try {
+    upstream = require_(`${SEMCONV_PACKAGE}/incubating`);
+  } catch (err) {
+    violations.push(
+      `cannot load '${SEMCONV_PACKAGE}/incubating' (${err.message}).\n` +
+        `      Without it the literals are unverifiable. This is a failure, not a skip: a check\n` +
+        `      that quietly passes when it cannot look is the defect it was written to prevent.`,
+    );
+    upstream = undefined;
+  }
+
+  let compared = 0;
+  if (upstream) {
+    for (const symbol of pinned) {
+      const actual = upstream[symbol.exportName];
+      if (symbol.upstream === 'exported') {
+        compared += 1;
+        if (actual === undefined) {
+          violations.push(
+            `${symbol.exportName} is declared 'exported' but ${SEMCONV_PACKAGE}@${locked} no longer\n` +
+              `      exports it. Evolith is emitting '${symbol.value}' against a convention that moved.`,
+          );
+        } else if (actual !== symbol.value) {
+          violations.push(
+            `${symbol.exportName} drift: pinned as '${symbol.value}', upstream now '${actual}'.\n` +
+              `      Update the literal in the ${SEMCONV_MODULE_KEY} module and everything that asserts on it.`,
+          );
+        }
+      } else if (symbol.upstream === 'registry-only') {
+        if (actual !== undefined) {
+          compared += 1;
+          if (actual !== symbol.value) {
+            violations.push(
+              `${symbol.exportName} was declared 'registry-only' (locally spelled '${symbol.value}')\n` +
+                `      and upstream now exports it as '${actual}'. The local spelling is wrong from this\n` +
+                `      release onwards. Adopt the upstream value and change upstream: 'exported'.`,
+            );
+          }
+        }
+      } else {
+        violations.push(`${symbol.exportName} has an unknown upstream status '${symbol.upstream}'.`);
+      }
+    }
+  }
+
+  // --- Rule 3: the manifest covers every literal the module exports ----------------
+  const manifestNames = new Set(pinned.map((s) => s.exportName));
+  const declaredConstants = Object.keys(vocabulary).filter(
+    (k) => k.startsWith('ATTR_') || k.startsWith('EVENT_') || /^MCP_[A-Z_]+_VALUE_/.test(k),
+  );
+  assertScanned(declaredConstants.length, {
+    what: 'exported semconv constants',
+    where: SEMCONV_MODULE_KEY,
+  });
+  for (const name of declaredConstants) {
+    if (!manifestNames.has(name)) {
+      violations.push(
+        `${name} is exported but absent from PINNED_SEMCONV_ATTRIBUTES, so nothing checks it\n` +
+          `      against upstream. Add it to the manifest with its upstream status.`,
+      );
+    }
+  }
+
+  if (violations.length > 0) {
+    console.error(`✗ ${GUARD}: ${violations.length} finding(s)\n`);
+    for (const v of violations) console.error(`  • ${v}\n`);
+    process.exit(1);
+  }
+
+  console.log(`✅ ${GUARD}`);
+  console.log(`  pinned revision .... ${declared} (lockfile: ${locked})`);
+  console.log(`  symbols pinned ..... ${pinned.length}`);
+  console.log(`  compared upstream .. ${compared}`);
+  console.log(`  registry-only ...... ${pinned.filter((s) => s.upstream === 'registry-only').length} (allowed to be absent upstream)`);
+  if (VERBOSE) {
+    for (const s of pinned) console.log(`    • ${s.exportName} = ${s.value} [${s.upstream}]`);
+  }
+}
+
+main();
@@ -0,0 +1,144 @@
+#!/usr/bin/env node --test
+
+/**
+ * Negative fixtures for `51-validate-semconv-pin.mjs` (GT-587 criterion 3).
+ *
+ * A pin check that has never been observed failing is decoration. Each case below
+ * materialises a repo-SHAPED sandbox — the three ROOT_MARKERS so `lib/paths.mjs`
+ * ascends to the sandbox and not to this repository, a copy of `.harness/scripts`, a
+ * synthetic `package-lock.json`, and one synthetic `semconv.ts` — and asserts the guard's
+ * exit code. The green case is included deliberately: without it, a guard that failed
+ * unconditionally would pass every red test and look thorough.
+ *
+ * Run: node --test .harness/scripts/ci/51-validate-semconv-pin.test.mjs
+ */
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = join(__dirname, '../../..');
+const GUARD_REL = '.harness/scripts/ci/51-validate-semconv-pin.mjs';
+
+/** The version the fixture lockfile claims, matching the fixture module unless a case says otherwise. */
+const PINNED = '1.41.1';
+
+/**
+ * A minimal but REAL semconv module: same export shapes the guard parses, and values
+ * that genuinely match `@opentelemetry/semantic-conventions@1.41.1`. Using fake values
+ * would make the green case pass for the wrong reason.
+ */
+function semconvModule({ version = PINNED, nameValue = 'gen_ai.evaluation.name', extraConstant = '' } = {}) {
+  return [
+    `export const SEMCONV_VERSION = '${version}';`,
+    `export const EVENT_GEN_AI_EVALUATION_RESULT = 'gen_ai.evaluation.result';`,
+    `export const ATTR_GEN_AI_EVALUATION_NAME = '${nameValue}';`,
+    `export const ATTR_MCP_TOOL_NAME = 'mcp.tool.name';`,
+    extraConstant,
+    `export interface PinnedSemconvSymbol {`,
+    `  readonly exportName: string;`,
+    `  readonly value: string;`,
+    `  readonly upstream: 'exported' | 'registry-only';`,
+    `}`,
+    `export const PINNED_SEMCONV_ATTRIBUTES: readonly PinnedSemconvSymbol[] = [`,
+    `  { exportName: 'EVENT_GEN_AI_EVALUATION_RESULT', value: EVENT_GEN_AI_EVALUATION_RESULT, upstream: 'exported' },`,
+    `  { exportName: 'ATTR_GEN_AI_EVALUATION_NAME', value: ATTR_GEN_AI_EVALUATION_NAME, upstream: 'exported' },`,
+    `  { exportName: 'ATTR_MCP_TOOL_NAME', value: ATTR_MCP_TOOL_NAME, upstream: 'registry-only' },`,
+    `];`,
+  ].join('\n');
+}
+
+function sandbox(moduleSource, lockedVersion = PINNED) {
+  // realpath: on macOS os.tmpdir() is a symlink, and a guard comparing import.meta.url
+  // against process.argv[1] would then skip its own main() and exit 0 in silence.
+  const root = realpathSync(mkdtempSync(join(tmpdir(), 'gt587-semconv-')));
+
+  writeFileSync(join(root, 'package.json'), JSON.stringify({ name: 'fixture', private: true }));
+  writeFileSync(join(root, 'evolith.yaml'), 'name: fixture\n');
+  writeFileSync(
+    join(root, 'package-lock.json'),
+    JSON.stringify({
+      packages: lockedVersion
+        ? { 'node_modules/@opentelemetry/semantic-conventions': { version: lockedVersion } }
+        : {},
+    }),
+  );
+
+  mkdirSync(join(root, '.harness'), { recursive: true });
+  cpSync(join(REPO_ROOT, '.harness/scripts'), join(root, '.harness/scripts'), { recursive: true });
+
+  const modulePath = join(root, 'src/packages/core-domain/src/evaluation/telemetry/semconv.ts');
+  mkdirSync(dirname(modulePath), { recursive: true });
+  writeFileSync(modulePath, moduleSource);
+
+  const realNodeModules = join(REPO_ROOT, 'node_modules');
+  if (existsSync(realNodeModules)) symlinkSync(realNodeModules, join(root, 'node_modules'), 'dir');
+
+  return root;
+}
+
+function run(root) {
+  const res = spawnSync(process.execPath, [join(root, GUARD_REL)], { cwd: root, encoding: 'utf8' });
+  return { code: res.status, out: `${res.stdout}${res.stderr}` };
+}
+
+function withSandbox(moduleSource, lockedVersion, fn) {
+  const root = sandbox(moduleSource, lockedVersion);
+  try {
+    fn(run(root));
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+test('PASSES on a pin that agrees with the lockfile and with upstream', () => {
+  withSandbox(semconvModule(), PINNED, ({ code, out }) => {
+    assert.equal(code, 0, out);
+    assert.match(out, /pinned revision/);
+  });
+});
+
+test('FAILS when the declared revision drifts from the lockfile', () => {
+  withSandbox(semconvModule({ version: '1.99.0' }), PINNED, ({ code, out }) => {
+    assert.equal(code, 1, out);
+    assert.match(out, /SEMCONV_VERSION drift/);
+  });
+});
+
+test('FAILS when a pinned literal no longer matches upstream', () => {
+  withSandbox(semconvModule({ nameValue: 'gen_ai.eval.name' }), PINNED, ({ code, out }) => {
+    assert.equal(code, 1, out);
+    assert.match(out, /ATTR_GEN_AI_EVALUATION_NAME drift/);
+  });
+});
+
+test('FAILS when a constant is exported but left out of the manifest', () => {
+  const extra = `export const ATTR_GEN_AI_REQUEST_MODEL = 'gen_ai.request.model';`;
+  withSandbox(semconvModule({ extraConstant: extra }), PINNED, ({ code, out }) => {
+    assert.equal(code, 1, out);
+    assert.match(out, /absent from PINNED_SEMCONV_ATTRIBUTES/);
+  });
+});
+
+test('FAILS when the package is absent from the lockfile — an unverifiable pin is not a pin', () => {
+  withSandbox(semconvModule(), null, ({ code, out }) => {
+    assert.equal(code, 1, out);
+    assert.match(out, /not present in package-lock\.json/);
+  });
+});
+
+test('FAILS on an empty manifest rather than reporting a vacuous pass', () => {
+  const empty = [
+    `export const SEMCONV_VERSION = '${PINNED}';`,
+    `export const PINNED_SEMCONV_ATTRIBUTES: readonly { exportName: string; value: string; upstream: string }[] = [];`,
+  ].join('\n');
+  withSandbox(empty, PINNED, ({ code, out }) => {
+    assert.equal(code, 1, out);
+    assert.match(out, /pinned semconv symbols/);
+  });
+});
@@ -49,6 +49,7 @@ export const ROOT_MARKERS = ['package.json', '.harness', 'evolith.yaml'];
 export const PATH_KEYS = Object.freeze({
   // --- repo skeleton -------------------------------------------------------
   rootPackageJson: 'package.json',
+  rootPackageLock: 'package-lock.json',
   evolithManifest: 'evolith.yaml',
   masterIndex: 'MASTER_INDEX.md',
 
@@ -118,6 +119,8 @@ export const PATH_KEYS = Object.freeze({
   // Formerly referenced as `packages/agent-runtime/...` (missing `src/`) in script 33.
   agentRuntimeAdaptersBarrel: 'src/packages/agent-runtime/src/adapters/index.ts',
   agentRuntimeInteractionAdapters: 'src/packages/agent-runtime/src/adapters/interaction',
+  // GT-587: the single declaration of the pinned OpenTelemetry semconv vocabulary.
+  semconvPin: 'src/packages/core-domain/src/evaluation/telemetry/semconv.ts',
 });
 
 /**

@@ -18,6 +18,7 @@ import { validateEnv } from './infrastructure/config/env.validation';
 import { AuditThrottlerGuard } from './infrastructure/guards/audit-throttler.guard';
 import { ApiKeyGuard } from './infrastructure/guards/api-key.guard';
 import { MetricsService } from './infrastructure/metrics/metrics.service';
+import { EvaluationTelemetryService } from './infrastructure/observability/evaluation-telemetry.service';
 import { CoreReferenceQueryService } from './application/services/core-reference-query.service';
 import { ReferenceController } from './presentation/controllers/reference.controller';
 import { CapabilitiesController } from './presentation/controllers/capabilities.controller';
@@ -110,6 +111,9 @@ import { CacheMetricsService } from './infrastructure/cache/cache-metrics.servic
   providers: [
     HealthService,
     MetricsService,
+    // GT-587 — the standard (semconv) half of the evaluation signal; the private
+    // `evolith_*` series MetricsService emits stays exactly as it was.
+    EvaluationTelemetryService,
     CacheMetricsService,
     CoreReferenceQueryService,
     WorkspaceReferenceResolverService,