Add unordered matching to expectDiagnostics

.chronus/changes/unordered-diagnostic-expectations-2026-08-18.md

@@ -0,0 +1,7 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/compiler"
+---
+
+Add a `fixedOrder` option to `expectDiagnostics`. Set it to `false` to match expected diagnostics regardless of their emitted order.

packages/compiler/src/testing/expect.ts

@@ -1,4 +1,4 @@
-import { fail, match, strictEqual } from "assert";
+import { fail, strictEqual } from "assert";
 import { getSourceLocation } from "../core/diagnostics.js";
 import { formatDiagnostic } from "../core/logger/console-sink.js";
 import { NoTarget, type Diagnostic } from "../core/types.js";
@@ -53,26 +53,50 @@ export interface DiagnosticMatch {
   end?: number;
 }
 
+export interface ExpectDiagnosticsOptions {
+  /** Require the number of diagnostics to match exactly. Defaults to true. */
+  strict?: boolean;
+
+  /** Require diagnostics to appear in the expected order. Defaults to true. */
+  fixedOrder?: boolean;
+}
+
 /**
- * Validate the diagnostic array contains exactly the given diagnostics.
- * @param diagnostics Array of the diagnostics
+ * Validate diagnostics against the given expectations. By default, the count and order must match
+ * exactly. Set `strict` to false to allow additional diagnostics, or `fixedOrder` to false to match
+ * expectations regardless of diagnostic order.
+ * @param diagnostics Array of diagnostics.
+ * @param match Expected diagnostic properties.
+ * @param options Options controlling count and order matching.
  */
 export function expectDiagnostics(
   diagnostics: readonly Diagnostic[],
   match: DiagnosticMatch | DiagnosticMatch[],
-  options = {
-    strict: true,
-  },
+  options: ExpectDiagnosticsOptions = {},
 ) {
   const array = isArray(match) ? match : [match];
+  const strict = options.strict ?? true;
+  const fixedOrder = options.fixedOrder ?? true;
 
-  if (options.strict && array.length !== diagnostics.length) {
+  if ((strict && array.length !== diagnostics.length) || array.length > diagnostics.length) {
     fail(
       `Expected ${array.length} diagnostics but found ${diagnostics.length}:\n ${formatDiagnostics(
         diagnostics,
       )}`,
     );
   }
+
+  if (!fixedOrder) {
+    if (!hasUnorderedMatch(diagnostics, array)) {
+      fail(
+        `Could not match the expected diagnostics regardless of order:\nExpected diagnostics:\n${formatDiagnosticMatches(
+          array,
+        )}\nDiagnostics found:\n${formatDiagnostics(diagnostics)}`,
+      );
+    }
+    return;
+  }
+
   for (let i = 0; i < array.length; i++) {
     const diagnostic = diagnostics[i];
     const expectation = array[i];
@@ -139,10 +163,108 @@ export function expectDiagnostics(
   }
 }
 
-function matchStrOrRegex(value: string, expectation: string | RegExp, assertMessage: string) {
+function formatDiagnosticMatches(expectations: readonly DiagnosticMatch[]): string {
+  return expectations
+    .map(
+      (expectation, index) =>
+        `${index}: ${JSON.stringify(expectation, (_, value) =>
+          value instanceof RegExp ? value.toString() : value,
+        )}`,
+    )
+    .join("\n");
+}
+
+function hasUnorderedMatch(
+  diagnostics: readonly Diagnostic[],
+  expectations: readonly DiagnosticMatch[],
+): boolean {
+  const diagnosticMatches = new Array<number>(diagnostics.length).fill(-1);
+
+  function assignExpectation(expectationIndex: number, visited: boolean[]): boolean {
+    for (let diagnosticIndex = 0; diagnosticIndex < diagnostics.length; diagnosticIndex++) {
+      if (
+        visited[diagnosticIndex] ||
+        !diagnosticMatchesExpectation(diagnostics[diagnosticIndex], expectations[expectationIndex])
+      ) {
+        continue;
+      }
+
+      visited[diagnosticIndex] = true;
+      const previousExpectation = diagnosticMatches[diagnosticIndex];
+      if (previousExpectation === -1 || assignExpectation(previousExpectation, visited)) {
+        diagnosticMatches[diagnosticIndex] = expectationIndex;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  return expectations.every((_, index) =>
+    assignExpectation(index, new Array<boolean>(diagnostics.length).fill(false)),
+  );
+}
+
+function diagnosticMatchesExpectation(
+  diagnostic: Diagnostic,
+  expectation: DiagnosticMatch,
+): boolean {
+  if (expectation.code !== undefined && diagnostic.code !== expectation.code) {
+    return false;
+  }
+  if (
+    expectation.message !== undefined &&
+    !strOrRegexMatches(diagnostic.message, expectation.message)
+  ) {
+    return false;
+  }
+  if (expectation.severity !== undefined && diagnostic.severity !== expectation.severity) {
+    return false;
+  }
+  if (
+    expectation.file === undefined &&
+    expectation.pos === undefined &&
+    expectation.end === undefined
+  ) {
+    return true;
+  }
+  if (diagnostic.target === NoTarget) {
+    return false;
+  }
+
+  const source = getSourceLocation(diagnostic.target);
+  if (
+    expectation.file !== undefined &&
+    !strOrRegexMatches(
+      source.file.path,
+      typeof expectation.file === "string"
+        ? resolveVirtualPath(expectation.file)
+        : expectation.file,
+    )
+  ) {
+    return false;
+  }
+  if (expectation.pos !== undefined && source.pos !== expectation.pos) {
+    return false;
+  }
+  if (expectation.end !== undefined && source.end !== expectation.end) {
+    return false;
+  }
+  return true;
+}
+
+function strOrRegexMatches(value: string, expectation: string | RegExp): boolean {
   if (typeof expectation === "string") {
-    strictEqual(value, expectation, assertMessage);
-  } else {
-    match(value, expectation, assertMessage);
+    return value === expectation;
+  }
+
+  const lastIndex = expectation.lastIndex;
+  const result = expectation.test(value);
+  expectation.lastIndex = lastIndex;
+  return result;
+}
+
+function matchStrOrRegex(value: string, expectation: string | RegExp, assertMessage: string) {
+  if (!strOrRegexMatches(value, expectation)) {
+    fail(assertMessage);
   }
 }

packages/compiler/test/testing/expect.test.ts

@@ -0,0 +1,73 @@
+import { expect, it } from "vitest";
+import { NoTarget, type Diagnostic } from "../../src/core/types.js";
+import { expectDiagnostics } from "../../src/testing/expect.js";
+
+function diagnostic(code: string, message: string): Diagnostic {
+  return {
+    code,
+    message,
+    severity: "error",
+    target: NoTarget,
+  };
+}
+
+it("requires diagnostics to use the expected order by default", () => {
+  const diagnostics = [diagnostic("first", "First"), diagnostic("second", "Second")];
+
+  expect(() => expectDiagnostics(diagnostics, [{ code: "second" }, { code: "first" }])).toThrow(
+    "Diagnostic at index 0 has non matching code",
+  );
+});
+
+it("can match diagnostics regardless of order", () => {
+  const diagnostics = [diagnostic("first", "First"), diagnostic("second", "Second")];
+
+  expectDiagnostics(diagnostics, [{ code: "second" }, { code: "first" }], {
+    fixedOrder: false,
+  });
+});
+
+it("matches overlapping expectations without depending on greedy order", () => {
+  const diagnostics = [diagnostic("shared", "Specific"), diagnostic("shared", "Other")];
+
+  expectDiagnostics(diagnostics, [{ code: "shared" }, { message: "Specific" }], {
+    fixedOrder: false,
+  });
+});
+
+it("matches stateful regular expressions consistently", () => {
+  const diagnostics = [diagnostic("first", "Shared one"), diagnostic("second", "Shared two")];
+  const message = /shared/gi;
+
+  expectDiagnostics(diagnostics, [{ message }, { message }], { fixedOrder: false });
+  expect(message.lastIndex).toBe(0);
+  expectDiagnostics(diagnostics, [{ message }, { message }], { fixedOrder: false });
+});
+
+it("includes unmatched expectations in unordered failure messages", () => {
+  const diagnostics = [diagnostic("actual", "Actual diagnostic")];
+
+  expect(() =>
+    expectDiagnostics(
+      diagnostics,
+      { code: "expected", message: /missing/i },
+      { fixedOrder: false },
+    ),
+  ).toThrow(
+    'Expected diagnostics:\n0: {"code":"expected","message":"/missing/i"}\nDiagnostics found:',
+  );
+});
+
+it("allows unmatched diagnostics in non-strict unordered mode", () => {
+  const diagnostics = [diagnostic("first", "First"), diagnostic("second", "Second")];
+
+  expectDiagnostics(diagnostics, { code: "second" }, { strict: false, fixedOrder: false });
+});
+
+it("retains strict count validation in unordered mode", () => {
+  const diagnostics = [diagnostic("first", "First"), diagnostic("second", "Second")];
+
+  expect(() => expectDiagnostics(diagnostics, { code: "second" }, { fixedOrder: false })).toThrow(
+    "Expected 1 diagnostics but found 2",
+  );
+});