Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## [Unreleased]

### Fixed

- Fixed Swift Testing results being dropped for tests declared without a display name (`@Test func name()`), so their passes, failures, and failure locations were missing from test progress and summaries. Skipped tests with a display name and failures in parameterized tests whose argument is not named `value` are now parsed too ([#542](https://github.com/getsentry/MobileBuildMCP/pull/542) by [@breken-ai](https://github.com/breken-ai)).

## [2.7.1]

### Changed
Expand Down
83 changes: 83 additions & 0 deletions src/utils/__tests__/swift-testing-line-parsers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,65 @@ describe('Swift Testing line parsers', () => {
});
});

it('should parse a passed test without a display name', () => {
const result = parseSwiftTestingResultLine('✔ Test passes() passed after 0.001 seconds.');
expect(result).toEqual({
status: 'passed',
rawName: 'passes()',
testName: 'passes()',
durationText: '0.001s',
});
});

it('should parse a failed test without a display name', () => {
const result = parseSwiftTestingResultLine(
'✘ Test fails() failed after 0.004 seconds with 3 issues.',
);
expect(result).toEqual({
status: 'failed',
rawName: 'fails()',
testName: 'fails()',
durationText: '0.004s',
});
});

it('should parse a failed parameterized test without a display name', () => {
const result = parseSwiftTestingResultLine(
'✘ Test param(x:) with 3 test cases failed after 0.004 seconds with 1 issue.',
);
expect(result).toEqual({
status: 'failed',
rawName: 'param(x:)',
testName: 'param(x:)',
durationText: '0.004s',
caseCount: 3,
});
});

it('should parse a skipped test with a display name (arrow format)', () => {
const result = parseSwiftTestingResultLine(
'➜ Test "Skipped with display name" skipped: "later"',
);
expect(result).toEqual({
status: 'skipped',
rawName: 'Skipped with display name',
testName: 'Skipped with display name',
});
});

it('should not parse run summary lines as test results', () => {
expect(
parseSwiftTestingResultLine(
'✔ Test run with 6 tests in 1 suite passed after 0.005 seconds.',
),
).toBeNull();
expect(
parseSwiftTestingResultLine(
'✘ Test run with 6 tests in 1 suite failed after 0.005 seconds with 5 issues.',
),
).toBeNull();
});

it('should return null for non-matching lines', () => {
expect(parseSwiftTestingResultLine('◇ Test "Foo" started.')).toBeNull();
expect(parseSwiftTestingResultLine('random text')).toBeNull();
Expand Down Expand Up @@ -164,6 +223,30 @@ describe('Swift Testing line parsers', () => {
});
});

it('should parse an issue for a test without a display name', () => {
const result = parseSwiftTestingIssueLine(
'✘ Test fails() recorded an issue at CountsTests.swift:6:24: Expectation failed: (one() → 1) == 2',
);
expect(result).toEqual({
rawTestName: 'fails()',
testName: 'fails()',
location: 'CountsTests.swift:6',
message: 'Expectation failed: (one() → 1) == 2',
});
});

it('should parse a parameterized issue for a test without a display name', () => {
const result = parseSwiftTestingIssueLine(
'✘ Test param(x:) recorded an issue with 1 argument x → 2 at CountsTests.swift:10:52: Expectation failed: (x → 2) != 2',
);
expect(result).toEqual({
rawTestName: 'param(x:)',
testName: 'param(x:)',
location: 'CountsTests.swift:10',
message: 'Expectation failed: (x → 2) != 2',
});
});

it('should parse an issue without location', () => {
const result = parseSwiftTestingIssueLine(
'✘ Test "Some test" recorded an issue: Something went wrong',
Expand Down
50 changes: 50 additions & 0 deletions src/utils/__tests__/xcodebuild-event-parser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,56 @@ describe('xcodebuild-event-parser', () => {
});
});

it('parses Swift Testing results for tests without display names', () => {
// Captured from `xcodebuild test` (Xcode 26, Swift Testing 1501) on a package
// that mixes `@Test func name()` and `@Test("Display name")` tests.
const events = collectEvents('TEST', [
{
source: 'stdout',
text: [
'➜ Test "Skipped with display name" skipped: "later"',
'\u200B➜ Test skippedPlain() skipped: "not now"',
'✘ Test fails() recorded an issue at CountsTests.swift:6:24: Expectation failed: (one() → 1) == 2',
'✘ Test "Display named failure" recorded an issue at CountsTests.swift:7:49: Expectation failed: (one() → 1) == 5',
'✔ Test passes() passed after 0.001 seconds.',
'✘ Test param(x:) recorded an issue with 1 argument x → 2 at CountsTests.swift:10:52: Expectation failed: (x → 2) != 2',
'✘ Test fails() recorded an issue at CountsTests.swift:6:45: Expectation failed: (one() → 1) == 3',
'✘ Test fails() recorded an issue at CountsTests.swift:6:66: Expectation failed: (one() → 1) == 4',
'✘ Test "Display named failure" failed after 0.004 seconds with 1 issue.',
'✘ Test param(x:) with 3 test cases failed after 0.004 seconds with 1 issue.',
'✘ Test fails() failed after 0.004 seconds with 3 issues.',
'✘ Suite CountsTests failed after 0.004 seconds with 5 issues.',
'✘ Test run with 6 tests in 1 suite failed after 0.005 seconds with 5 issues.',
'',
].join('\n'),
},
]);

const cases = events.filter((event) => event.fragment === 'test-case-result');
expect(cases.map((event) => [event.test, event.status])).toEqual([
['Skipped with display name', 'skipped'],
['skippedPlain', 'skipped'],
['passes()', 'passed'],
['Display named failure', 'failed'],
['param(x:)', 'failed'],
['fails()', 'failed'],
]);

const failures = events.filter((event) => event.fragment === 'test-failure');
expect(failures.map((event) => event.test)).toEqual([
'Display named failure',
'param(x:)',
'fails()',
'fails()',
'fails()',
]);

expect(events.filter((event) => event.fragment === 'test-progress').at(-1)).toMatchObject({
completed: 6,
skipped: 2,
});
});

it('uses Swift Testing and XCTest summaries once for mixed Calculator test output', () => {
const xctestPassedLines = Array.from({ length: 21 }, (_, index) => ({
source: 'stdout' as const,
Expand Down
44 changes: 32 additions & 12 deletions src/utils/swift-testing-line-parsers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import {
parseRawTestName,
} from './xcodebuild-line-parsers.ts';

// Test name: quoted display name ("Name") or bare function name (name(), name(x:))
// Optional verbose suffix: (aka 'funcName()')
// Optional parameterized suffix: with N test cases
const TEST_NAME = `(?:"(.+)"|([^\\s"]\\S*))`;
const OPTIONAL_AKA = `(?:\\s*\\(aka '[^']*'\\))?`;
const OPTIONAL_PARAMETERIZED = `(?:\\s+with (\\d+) test cases?)?`;

Expand All @@ -19,17 +21,21 @@ const OPTIONAL_PARAMETERIZED = `(?:\\s+with (\\d+) test cases?)?`;
* ✔ Test "Name" with 3 test cases passed after 0.001 seconds.
* ✘ Test "Name" failed after 0.001 seconds with 1 issue.
* ✘ Test "Name" (aka 'func()') failed after 0.001 seconds with 1 issue.
* ✔ Test funcName() passed after 0.001 seconds.
* ✘ Test funcName(x:) with 3 test cases failed after 0.001 seconds with 1 issue.
* ➜ Test funcName() skipped: "reason"
* ➜ Test "Name" skipped: "reason"
* ➜ Test funcName() skipped
*/
export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null {
const passedRegex = new RegExp(
`^[✔] Test "(.+)"${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} passed after ([\\d.]+) seconds\\.?$`,
`^[✔] Test ${TEST_NAME}${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} passed after ([\\d.]+) seconds\\.?$`,
'u',
);
const passedMatch = line.match(passedRegex);
if (passedMatch) {
const [, name, caseCountStr, duration] = passedMatch;
const [, quotedName, bareName, caseCountStr, duration] = passedMatch;
const name = quotedName ?? bareName;
const { suiteName, testName } = parseRawTestName(name);
const caseCount = caseCountStr ? Number(caseCountStr) : undefined;
return {
Expand All @@ -43,12 +49,13 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null
}

const failedRegex = new RegExp(
`^[✘] Test "(.+)"${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} failed after ([\\d.]+) seconds`,
`^[✘] Test ${TEST_NAME}${OPTIONAL_AKA}${OPTIONAL_PARAMETERIZED} failed after ([\\d.]+) seconds`,
'u',
);
const failedMatch = line.match(failedRegex);
if (failedMatch) {
const [, name, caseCountStr, duration] = failedMatch;
const [, quotedName, bareName, caseCountStr, duration] = failedMatch;
const name = quotedName ?? bareName;
const { suiteName, testName } = parseRawTestName(name);
const caseCount = caseCountStr ? Number(caseCountStr) : undefined;
return {
Expand All @@ -61,10 +68,12 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null
};
}

// Skipped: ➜ Test funcName() skipped: "reason"
// Skipped: ➜ Test funcName() skipped: "reason" or ➜ Test "Name" skipped: "reason"
// Also handle legacy format: ◇ Test "Name" skipped
const skippedMatch =
line.match(/^[➜] Test (\S+?)(?:\(\))? skipped/u) ?? line.match(/^[◇] Test "(.+)" skipped/u);
line.match(/^[➜] Test "(.+?)" skipped/u) ??
line.match(/^[➜] Test (\S+?)(?:\(\))? skipped/u) ??
line.match(/^[◇] Test "(.+)" skipped/u);
if (skippedMatch) {
const rawName = skippedMatch[1];
const { suiteName, testName } = parseRawTestName(rawName);
Expand All @@ -87,16 +96,19 @@ export function parseSwiftTestingResultLine(line: string): ParsedTestCase | null
* ✘ Test "Name" (aka 'func()') recorded an issue at File.swift:48:5: msg
* ✘ Test "Name" recorded an issue with 1 argument value → 0 at File.swift:10:5: msg
* ✘ Test "Name" recorded an issue: message
* ✘ Test funcName() recorded an issue at File.swift:48:5: msg
* ✘ Test funcName(x:) recorded an issue with 1 argument x → 2 at File.swift:10:5: msg
*/
export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnostic | null {
// Match with location -- handle both aka suffix and parameterized argument values before "at"
const locationRegex = new RegExp(
`^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue(?:\\s+with \\d+ argument values?.*?)? at (.+?):(\\d+):\\d+: (.+)$`,
`^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue(?:\\s+with \\d+ arguments?\\b.*?)? at (.+?):(\\d+):\\d+: (.+)$`,
'u',
);
const locationMatch = line.match(locationRegex);
if (locationMatch) {
const [, rawTestName, filePath, lineNumber, message] = locationMatch;
const [, quotedName, bareName, filePath, lineNumber, message] = locationMatch;
const rawTestName = quotedName ?? bareName;
const { suiteName, testName } = parseRawTestName(rawTestName);
return {
rawTestName,
Expand All @@ -108,10 +120,14 @@ export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnosti
}

// Match without location
const simpleRegex = new RegExp(`^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue: (.+)$`, 'u');
const simpleRegex = new RegExp(
`^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue: (.+)$`,
'u',
);
const simpleMatch = line.match(simpleRegex);
if (simpleMatch) {
const [, rawTestName, message] = simpleMatch;
const [, quotedName, bareName, message] = simpleMatch;
const rawTestName = quotedName ?? bareName;
const { suiteName, testName } = parseRawTestName(rawTestName);
return {
rawTestName,
Expand All @@ -121,10 +137,14 @@ export function parseSwiftTestingIssueLine(line: string): ParsedFailureDiagnosti
};
}

const fallbackRegex = new RegExp(`^[✘] Test "(.+)"${OPTIONAL_AKA} recorded an issue\\b.*$`, 'u');
const fallbackRegex = new RegExp(
`^[✘] Test ${TEST_NAME}${OPTIONAL_AKA} recorded an issue\\b.*$`,
'u',
);
const fallbackMatch = line.match(fallbackRegex);
if (fallbackMatch) {
const [, rawTestName] = fallbackMatch;
const [, quotedName, bareName] = fallbackMatch;
const rawTestName = quotedName ?? bareName;
const { suiteName, testName } = parseRawTestName(rawTestName);
return {
rawTestName,
Expand Down