Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.ja.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ npm run --silent demo:repair -- --show-stop
npm run check:node -- --cwd /path/to/project --check-id regression --id before test/regression.test.mjs
```

Node のテストより前にソースの構文を確認するには、対象ごとに `--syntax FILE` を指定します。例は `tfl-check --syntax src/app.mjs test/regression.test.mjs` です。構文エラーを確認できた場合は、分かる範囲でソースの位置を添えて失敗を返します。対象ファイルが見つからない場合や構文検査の結果を確定できない場合は未確認を返します。

結果、失敗したアサーションの場所、分かる範囲で未確認の理由が出ます。修正後は同じファイル・検査IDで `--id after --resolves before` を付けて実行します。モデルや API キーは不要です。台本から使う場合は `npm run --silent check:node -- --json` に検査の引数を足し、返る `observation` を `RunContext` に渡せます。

| 終了値 | 意味 |
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ Run the check command directly on your own tests. To hand failures to a repair a
npm run check:node -- --cwd /path/to/project --check-id regression --id before test/regression.test.mjs
```

To check selected source files for parser errors before running the Node tests, repeat `--syntax FILE` for each file. For example, run `tfl-check --syntax src/app.mjs test/regression.test.mjs`. A confirmed parser error reports the source location when available and returns a failed check. A missing source file or an inconclusive parser run returns an unknown result.

The output leads with the check result, followed by the failing assertion and its location if available. After a repair, use the same files and check ID, plus `--id after --resolves before`.

| Exit code | Meaning |
Expand Down
2 changes: 2 additions & 0 deletions README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ npm run --silent demo:repair -- --show-stop
npm run check:node -- --cwd /path/to/project --check-id regression --id before test/regression.test.mjs
```

若要在运行 Node 测试前检查指定源码的语法,可对每个文件指定一次 `--syntax FILE`。例如 `tfl-check --syntax src/app.mjs test/regression.test.mjs`。确认语法错误时,检查返回失败,并尽可能报告源码位置。源码缺失或解析结果无法确定时,检查返回未知。

输出首先显示判定,然后在可用时给出失败的断言和位置。修复后,使用同一批测试文件和同一个检查 ID,并加上 `--id after --resolves before`。

| 退出码 | 含义 |
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"start": "node src/cli.mjs",
"check:node": "node src/check-cli.mjs",
"check:pytest": "node src/check-cli.mjs --runner pytest",
"test": "node test/unit.test.mjs && node test/checkers.test.mjs && node test/repair-loop.test.mjs",
"test": "node test/unit.test.mjs && node test/checkers.test.mjs && node test/repair-loop.test.mjs && node test/syntax-check.test.mjs",
"test:pytest": "node test/pytest-checker.test.mjs",
"test:integration": "node test/integration.test.mjs",
"demo:repair": "node examples/repair-loop.mjs",
Expand Down
11 changes: 10 additions & 1 deletion src/check-cli.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ const reasonText = {
pytest_report_unavailable: 'Pytest did not produce a report. Run python -m pytest with the same interpreter.',
python_launch_failed: 'Check that --python points to an executable Python interpreter.',
skipped_tests: 'The run contains skipped or deferred tests.',
syntax_input_unavailable: 'A --syntax path does not name an accessible regular file. Check the paths relative to --cwd.',
syntax_runner_unavailable: 'Could not start Node for the syntax check. Check the Node executable and --cwd.',
syntax_timeout: 'Node did not finish checking the selected source files within the time limit.',
syntax_output_limit: 'Node exceeded the syntax check output limit.',
syntax_inconclusive: 'Node did not report a recognizable parser diagnostic. Run node --check FILE directly to inspect the error.',
test_error: 'The runner reported an error outside a confirmed assertion failure. Run it directly to inspect the error.',
test_timeout: 'The test run exceeded its time limit.',
};
Expand All @@ -28,6 +33,7 @@ try {
cwd: { type: 'string', default: process.env.INIT_CWD || process.cwd() },
runner: { type: 'string', default: 'node' },
python: { type: 'string', default: 'python3' },
syntax: { type: 'string', multiple: true },
'check-id': { type: 'string' },
id: { type: 'string', default: randomUUID() },
resolves: { type: 'string' },
Expand All @@ -39,8 +45,10 @@ try {
commandName = `check:${values.runner}`;
if (values.help) {
console.log('tfl-check [--runner node|pytest] [--cwd DIR] [--check-id ID] [--id ID] [--resolves FAILURE_ID] [--json] TEST_FILE ...');
console.log('For Node, repeat --syntax FILE to check source syntax before running the tests. Paths are relative to --cwd.');
console.log('For pytest, select the interpreter with --python BIN.');
} else {
if (values.syntax && values.runner !== 'node') throw new TypeError('--syntax requires --runner node');
const { cwd, id, resolves } = values;
const runner = { node: runNodeTestCheck, pytest: runPytestCheck }[values.runner];
const checkId = values['check-id'] ?? `${values.runner}-tests`;
Expand All @@ -53,14 +61,15 @@ try {
let reason;
const files = positionals.map(file => resolve(cwd, file));
const observation = await runner({ files, cwd: resolve(cwd), python: values.python, checkId, id, resolves,
syntaxFiles: values.syntax,
onFailure: failure => evidence.push(failure), onUnknown: value => { reason = value; } });
if (values.json) {
console.log(JSON.stringify({ observation, evidence, ...(reason && { reason }) }));
} else {
console.log(`${observation.outcome}: ${observation.check_id} (${observation.id})`);
if (reason) console.log(` ${reasonText[reason]}`);
for (const item of evidence) {
console.log(` ${item.name} (${item.file}:${item.line})\n ${item.message}`);
console.log(` ${item.name} (${item.file}${item.line === undefined ? '' : `:${item.line}`})\n ${item.message}`);
}
}
process.exitCode = { success: 0, failure: 1, unknown: 2 }[observation.outcome];
Expand Down
73 changes: 72 additions & 1 deletion src/checkers/node-test.mjs
Original file line number Diff line number Diff line change
@@ -1,9 +1,80 @@
import { execFile } from 'node:child_process';
import { constants } from 'node:fs';
import { access, realpath, stat } from 'node:fs/promises';
import { resolve } from 'node:path';
import { performance } from 'node:perf_hooks';
import { promisify } from 'node:util';
import { run } from 'node:test';

const exec = promisify(execFile);

function parserDiagnostic(stderr, path, file) {
const output = stderr.replaceAll('\r\n', '\n');
if (!output.startsWith(path)) return;
const body = output.slice(path.length);
const header = /^(?::([1-9]\d*))?\n/.exec(body);
if (!header) return;
// Require the native --check stack, not just a source excerpt containing “SyntaxError”.
const diagnostic = /\n\n(SyntaxError(?:: [^\n]*)?)\n((?: at [^\n]+\n)+)\nNode\.js v[^\n]+\n$/.exec(body.slice(header[0].length - 1));
if (!diagnostic || !/^ at checkSyntax \(node:internal\/main\/check_syntax:\d+:\d+\)$/m.test(diagnostic[2])) return;
const line = Number(header[1]);
return { kind: 'code_syntax_error', name: 'Node syntax check', file,
...(Number.isSafeInteger(line) && { line }), message: diagnostic[1] };
}

async function checkSyntax(files, cwd, timeoutMs) {
const env = { ...process.env };
// Inherited preload flags can run arbitrary code before Node reaches the parser.
delete env.NODE_OPTIONS;
const deadline = performance.now() + timeoutMs;
let failure;
for (const input of files) {
let file, path;
try {
if (!input) return { reason: 'syntax_input_unavailable' };
file = resolve(cwd ?? process.cwd(), input);
path = await realpath(file);
if (!(await stat(path)).isFile()) return { reason: 'syntax_input_unavailable' };
await access(path, constants.R_OK);
} catch {
return { reason: 'syntax_input_unavailable' };
}
const timeout = Math.ceil(deadline - performance.now());
if (timeout <= 0) return { reason: 'syntax_timeout' };
try {
const { stdout, stderr } = await exec(process.execPath, ['--check', '--', path], {
cwd, env, timeout, killSignal: 'SIGKILL', maxBuffer: 64 * 1024,
});
if (stdout || stderr) return { reason: 'syntax_inconclusive' };
} catch (error) {
if (error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') return { reason: 'syntax_output_limit' };
if (error.killed) return { reason: 'syntax_timeout' };
if (typeof error.code === 'string') return { reason: 'syntax_runner_unavailable' };
const diagnostic = error.code === 1 && !error.stdout && parserDiagnostic(error.stderr, path, file);
if (!diagnostic) return { reason: 'syntax_inconclusive' };
failure ??= diagnostic;
}
}
return { failure };
}

/** Turn a trusted Node.js test run into one observation for RunContext. */
export async function runNodeTestCheck({ files, cwd, checkId, id, resolves, timeoutMs = 30_000, onFailure, onUnknown }) {
export async function runNodeTestCheck({ files, cwd, syntaxFiles = [], checkId, id, resolves,
timeoutMs = 30_000, onFailure, onUnknown }) {
if (!Array.isArray(files) || files.length === 0) throw new TypeError('Provide at least one test file');
const observation = { id, check_id: checkId };
const unknown = reason => {
onUnknown?.(reason);
return { ...observation, outcome: 'unknown' };
};
if (syntaxFiles.length) {
const { reason, failure } = await checkSyntax(syntaxFiles, cwd, timeoutMs);
if (reason) return unknown(reason);
if (failure) {
onFailure?.(failure);
return { ...observation, outcome: 'failure', category: 'reasoning' };
}
}
let summary;
let assertionFailed = false;
let uncertainFailure = false;
Expand Down
92 changes: 92 additions & 0 deletions test/syntax-check.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
import { runRepairLoop } from '../src/repair-loop.mjs';

const cli = fileURLToPath(new URL('../src/check-cli.mjs', import.meta.url));

async function fixture(t) {
const cwd = await mkdtemp(join(tmpdir(), 'tfl-syntax-'));
t.after(() => rm(cwd, { recursive: true, force: true }));
const write = (file, content) => writeFile(join(cwd, file), content);
await write('control.test.mjs', "import test from 'node:test'; test('control', () => {});\n");
await write('source.mjs', 'export const value = 42;\n');
return {
write,
check({ id = 'current', resolves, syntax = ['source.mjs'], file = 'control.test.mjs' } = {}) {
const env = { ...process.env };
// The standalone CLI starts its own node:test runner.
delete env.NODE_TEST_CONTEXT;
const run = spawnSync(process.execPath, [cli, '--json', '--cwd', cwd, '--id', id,
...syntax.flatMap(path => ['--syntax', path]), ...(resolves ? ['--resolves', resolves] : []), file],
{ encoding: 'utf8', timeout: 10_000, env });
assert.ifError(run.error);
assert.ok(run.stdout.trim().startsWith('{'), run.stderr || 'Missing JSON');
return { ...JSON.parse(run.stdout), status: run.status };
},
};
}

test('explicit syntax input reaches a Low repair and the same check verifies the fix', async t => {
const { write, check } = await fixture(t);
const bad = 'export const good = 1;\nexport const broken = ;\n';
let source = bad;
await write('source.mjs', source);
let diagnostic;
const result = await runRepairLoop({
check: async request => check(request),
readDiff: async () => source,
diagnose: async () => { assert.fail('High should not run'); },
repair: async ({ effort, evidence }) => {
assert.equal(effort, 'low');
diagnostic = evidence[0];
source = 'export const good = 1;\nexport const broken = 2;\n';
await write('source.mjs', source);
},
});
assert.equal(result.initial.status, 1);
assert.equal(result.initial.observation.outcome, 'failure');
assert.equal(result.initial.observation.category, 'reasoning');
assert.equal(diagnostic.kind, 'code_syntax_error');
assert.match(diagnostic.file, /source\.mjs$/);
assert.equal(diagnostic.line, 2);
assert.match(diagnostic.message, /SyntaxError/);
assert.equal(result.status, 'passed');
assert.equal(result.latest.status, 0);
assert.equal(result.latest.observation.resolves, 'before');
});

test('valid syntax still runs the independent tests', async t => {
const { check, write } = await fixture(t);
assert.equal(check().observation.outcome, 'success');
await write('control.test.mjs', "import test from 'node:test'; import assert from 'node:assert/strict'; test('control', () => assert.fail('control failed'));\n");
const result = check();
assert.equal(result.observation.outcome, 'failure');
assert.match(result.evidence[0].message, /control failed/);
});

test('an unavailable source or a runtime SyntaxError is inconclusive', async t => {
const { check, write } = await fixture(t);
const missing = check({ syntax: ['missing.mjs'] });
assert.equal(missing.status, 2);
assert.equal(missing.observation.outcome, 'unknown');
assert.equal(missing.evidence.length, 0);
await write('control.test.mjs', "import test from 'node:test'; test('runtime', () => { throw new SyntaxError('runtime exception'); });\n");
const runtime = check();
assert.equal(runtime.status, 2);
assert.equal(runtime.observation.outcome, 'unknown');
assert.equal(runtime.evidence.length, 0);
});

test('syntax inputs can repeat and ordinary callers keep their existing behavior', async t => {
const { check, write } = await fixture(t);
await write('second.mjs', 'let = ;\n');
assert.equal(check({ syntax: [] }).observation.outcome, 'success');
const result = check({ syntax: ['source.mjs', 'second.mjs'] });
assert.equal(result.status, 1);
assert.match(result.evidence[0].file, /second\.mjs$/);
});