diff --git a/CHANGELOG.md b/CHANGELOG.md index f47e609a..1a892a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [Unreleased] + +### Fixed + +- Fixed Swift Testing results being dropped for tests declared without a display name (`@Test func name()`), so their passes, failures, and failure locations were missing from test progress and summaries. Skipped tests with a display name and failures in parameterized tests whose argument is not named `value` are now parsed too ([#542](https://github.com/getsentry/MobileBuildMCP/pull/542) by [@breken-ai](https://github.com/breken-ai)). + ## [2.7.1] ### Changed diff --git a/src/utils/__tests__/swift-testing-line-parsers.test.ts b/src/utils/__tests__/swift-testing-line-parsers.test.ts index c810cba1..610241ef 100644 --- a/src/utils/__tests__/swift-testing-line-parsers.test.ts +++ b/src/utils/__tests__/swift-testing-line-parsers.test.ts @@ -109,6 +109,65 @@ describe('Swift Testing line parsers', () => { }); }); + it('should parse a passed test without a display name', () => { + const result = parseSwiftTestingResultLine('✔ Test passes() passed after 0.001 seconds.'); + expect(result).toEqual({ + status: 'passed', + rawName: 'passes()', + testName: 'passes()', + durationText: '0.001s', + }); + }); + + it('should parse a failed test without a display name', () => { + const result = parseSwiftTestingResultLine( + '✘ Test fails() failed after 0.004 seconds with 3 issues.', + ); + expect(result).toEqual({ + status: 'failed', + rawName: 'fails()', + testName: 'fails()', + durationText: '0.004s', + }); + }); + + it('should parse a failed parameterized test without a display name', () => { + const result = parseSwiftTestingResultLine( + '✘ Test param(x:) with 3 test cases failed after 0.004 seconds with 1 issue.', + ); + expect(result).toEqual({ + status: 'failed', + rawName: 'param(x:)', + testName: 'param(x:)', + durationText: '0.004s', + caseCount: 3, + }); + }); + + it('should parse a skipped test with a display name (arrow format)', () => { + const result = parseSwiftTestingResultLine( + '➜ Test "Skipped with display name" skipped: "later"', + ); + expect(result).toEqual({ + status: 'skipped', + rawName: 'Skipped with display name', + testName: 'Skipped with display name', + }); + }); + + it('should not parse run summary lines as test results', () => { + expect( + parseSwiftTestingResultLine( + '✔ Test run with 6 tests in 1 suite passed after 0.005 seconds.', + ), + ).toBeNull(); + expect( + parseSwiftTestingResultLine( + '✘ Test run with 6 tests in 1 suite failed after 0.005 seconds with 5 issues.', + ), + ).toBeNull(); + }); + it('should return null for non-matching lines', () => { expect(parseSwiftTestingResultLine('◇ Test "Foo" started.')).toBeNull(); expect(parseSwiftTestingResultLine('random text')).toBeNull(); @@ -164,6 +223,30 @@ describe('Swift Testing line parsers', () => { }); }); + it('should parse an issue for a test without a display name', () => { + const result = parseSwiftTestingIssueLine( + '✘ Test fails() recorded an issue at CountsTests.swift:6:24: Expectation failed: (one() → 1) == 2', + ); + expect(result).toEqual({ + rawTestName: 'fails()', + testName: 'fails()', + location: 'CountsTests.swift:6', + message: 'Expectation failed: (one() → 1) == 2', + }); + }); + + it('should parse a parameterized issue for a test without a display name', () => { + const result = parseSwiftTestingIssueLine( + '✘ Test param(x:) recorded an issue with 1 argument x → 2 at CountsTests.swift:10:52: Expectation failed: (x → 2) != 2', + ); + expect(result).toEqual({ + rawTestName: 'param(x:)', + testName: 'param(x:)', + location: 'CountsTests.swift:10', + message: 'Expectation failed: (x → 2) != 2', + }); + }); + it('should parse an issue without location', () => { const result = parseSwiftTestingIssueLine( '✘ Test "Some test" recorded an issue: Something went wrong', diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index a9774083..2c07b9ad 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -505,6 +505,56 @@ describe('xcodebuild-event-parser', () => { }); }); + it('parses Swift Testing results for tests without display names', () => { + // Captured from `xcodebuild test` (Xcode 26, Swift Testing 1501) on a package + // that mixes `@Test func name()` and `@Test("Display name")` tests. + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: [ + '➜ Test "Skipped with display name" skipped: "later"', + '\u200B➜ Test skippedPlain() skipped: "not now"', + '✘ Test fails() recorded an issue at CountsTests.swift:6:24: Expectation failed: (one() → 1) == 2', + '✘ Test "Display named failure" recorded an issue at CountsTests.swift:7:49: Expectation failed: (one() → 1) == 5', + '✔ Test passes() passed after 0.001 seconds.', + '✘ Test param(x:) recorded an issue with 1 argument x → 2 at CountsTests.swift:10:52: Expectation failed: (x → 2) != 2', + '✘ Test fails() recorded an issue at CountsTests.swift:6:45: Expectation failed: (one() → 1) == 3', + '✘ Test fails() recorded an issue at CountsTests.swift:6:66: Expectation failed: (one() → 1) == 4', + '✘ Test "Display named failure" failed after 0.004 seconds with 1 issue.', + '✘ Test param(x:) with 3 test cases failed after 0.004 seconds with 1 issue.', + '✘ Test fails() failed after 0.004 seconds with 3 issues.', + '✘ Suite CountsTests failed after 0.004 seconds with 5 issues.', + '✘ Test run with 6 tests in 1 suite failed after 0.005 seconds with 5 issues.', + '', + ].join('\n'), + }, + ]); + + const cases = events.filter((event) => event.fragment === 'test-case-result'); + expect(cases.map((event) => [event.test, event.status])).toEqual([ + ['Skipped with display name', 'skipped'], + ['skippedPlain', 'skipped'], + ['passes()', 'passed'], + ['Display named failure', 'failed'], + ['param(x:)', 'failed'], + ['fails()', 'failed'], + ]); + + const failures = events.filter((event) => event.fragment === 'test-failure'); + expect(failures.map((event) => event.test)).toEqual([ + 'Display named failure', + 'param(x:)', + 'fails()', + 'fails()', + 'fails()', + ]); + + expect(events.filter((event) => event.fragment === 'test-progress').at(-1)).toMatchObject({ + completed: 6, + skipped: 2, + }); + }); + it('uses Swift Testing and XCTest summaries once for mixed Calculator test output', () => { const xctestPassedLines = Array.from({ length: 21 }, (_, index) => ({ source: 'stdout' as const, diff --git a/src/utils/swift-testing-line-parsers.ts b/src/utils/swift-testing-line-parsers.ts index dddf20ef..7402ec5b 100644 --- a/src/utils/swift-testing-line-parsers.ts +++ b/src/utils/swift-testing-line-parsers.ts @@ -5,8 +5,10 @@ import { parseRawTestName, } from './xcodebuild-line-parsers.ts'; +// Test name: quoted display name ("Name") or bare function name (name(), name(x:)) // Optional verbose suffix: (aka 'funcName()') // Optional parameterized suffix: with N test cases +const TEST_NAME = `(?:"(.+)"|([^\\s"]\\S*))`; const OPTIONAL_AKA = `(?:\\s*\\(aka '[^']*'\\))?`; const OPTIONAL_PARAMETERIZED = `(?:\\s+with (\\d+) test cases?)?`; @@ -19,17 +21,21 @@ const OPTIONAL_PARAMETERIZED = `(?:\\s+with (\\d+) test cases?)?`; * ✔ Test "Name" with 3 test cases passed after 0.001 seconds. * ✘ Test "Name" failed after 0.001 seconds with 1 issue. * ✘ Test "Name" (aka 'func()') failed after 0.001 seconds with 1 issue. + * ✔ Test funcName() passed after 0.001 seconds. + * ✘ Test funcName(x:) with 3 test cases failed after 0.001 seconds with 1 issue. * ➜ Test funcName() skipped: "reason" + * ➜ Test "Name" skipped: "reason" * ➜ Test funcName() skipped */ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null { const passedRegex = new RegExp( - `^[✔] Test "(.+)"${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} passed after ([\\d.]+) seconds\\.?$`, + `^[✔] Test ${TEST_NAME}${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} passed after ([\\d.]+) seconds\\.?$`, 'u', ); const passedMatch = line.match(passedRegex); if (passedMatch) { - const [, name, caseCountStr, duration] = passedMatch; + const [, quotedName, bareName, caseCountStr, duration] = passedMatch; + const name = quotedName ?? bareName; const { suiteName, testName } = parseRawTestName(name); const caseCount = caseCountStr ? Number(caseCountStr) : undefined; return { @@ -43,12 +49,13 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null } const failedRegex = new RegExp( - `^[✘] Test "(.+)"${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} failed after ([\\d.]+) seconds`, + `^[✘] Test ${TEST_NAME}${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} failed after ([\\d.]+) seconds`, 'u', ); const failedMatch = line.match(failedRegex); if (failedMatch) { - const [, name, caseCountStr, duration] = failedMatch; + const [, quotedName, bareName, caseCountStr, duration] = failedMatch; + const name = quotedName ?? bareName; const { suiteName, testName } = parseRawTestName(name); const caseCount = caseCountStr ? Number(caseCountStr) : undefined; return { @@ -61,10 +68,12 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null }; } - // Skipped: ➜ Test funcName() skipped: "reason" + // Skipped: ➜ Test funcName() skipped: "reason" or ➜ Test "Name" skipped: "reason" // Also handle legacy format: ◇ Test "Name" skipped const skippedMatch = - line.match(/^[➜] Test (\S+?)(?:\(\))? skipped/u) ?? line.match(/^[◇] Test "(.+)" skipped/u); + line.match(/^[➜] Test "(.+?)" skipped/u) ?? + line.match(/^[➜] Test (\S+?)(?:\(\))? skipped/u) ?? + line.match(/^[◇] Test "(.+)" skipped/u); if (skippedMatch) { const rawName = skippedMatch[1]; const { suiteName, testName } = parseRawTestName(rawName); @@ -87,16 +96,19 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null * ✘ Test "Name" (aka 'func()') recorded an issue at File.swift:48:5: msg * ✘ Test "Name" recorded an issue with 1 argument value → 0 at File.swift:10:5: msg * ✘ Test "Name" recorded an issue: message + * ✘ Test funcName() recorded an issue at File.swift:48:5: msg + * ✘ Test funcName(x:) recorded an issue with 1 argument x → 2 at File.swift:10:5: msg */ export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnostic | null { // Match with location -- handle both aka suffix and parameterized argument values before "at" const locationRegex = new RegExp( - `^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue(?:\\s+with \\d+ argument values?.*?)? at (.+?):(\\d+):\\d+: (.+)$`, + `^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue(?:\\s+with \\d+ arguments?\\b.*?)? at (.+?):(\\d+):\\d+: (.+)$`, 'u', ); const locationMatch = line.match(locationRegex); if (locationMatch) { - const [, rawTestName, filePath, lineNumber, message] = locationMatch; + const [, quotedName, bareName, filePath, lineNumber, message] = locationMatch; + const rawTestName = quotedName ?? bareName; const { suiteName, testName } = parseRawTestName(rawTestName); return { rawTestName, @@ -108,10 +120,14 @@ export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnosti } // Match without location - const simpleRegex = new RegExp(`^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue: (.+)$`, 'u'); + const simpleRegex = new RegExp( + `^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue: (.+)$`, + 'u', + ); const simpleMatch = line.match(simpleRegex); if (simpleMatch) { - const [, rawTestName, message] = simpleMatch; + const [, quotedName, bareName, message] = simpleMatch; + const rawTestName = quotedName ?? bareName; const { suiteName, testName } = parseRawTestName(rawTestName); return { rawTestName, @@ -121,10 +137,14 @@ export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnosti }; } - const fallbackRegex = new RegExp(`^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue\\b.*$`, 'u'); + const fallbackRegex = new RegExp( + `^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue\\b.*$`, + 'u', + ); const fallbackMatch = line.match(fallbackRegex); if (fallbackMatch) { - const [, rawTestName] = fallbackMatch; + const [, quotedName, bareName] = fallbackMatch; + const rawTestName = quotedName ?? bareName; const { suiteName, testName } = parseRawTestName(rawTestName); return { rawTestName,