feat: add test alias support for playwright

README.md

@@ -849,10 +849,10 @@ test('resource management', () => {
 
 ## Test aliases
 
-Test aliases are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option:
+Test aliases (`test.skip()`, `test.fixme()`, `test.fail()`, `test.slow()`) are used to map tests in source code to tests in Testomat.io. By default `test` and `it` are parsed. But if you rename them or use another function to define tests (e.g. created/extended test object in Playwright), you can add alias (or multiple aliases, separated by comma) via `--test-alias` option:
 
 ```
-TESTOMATIO=11111111 npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myCustomFunction
+TESTOMATIO={API_KEY} npx check-tests Playwright "**/*{.,_}{test,spec}.ts" --test-alias myTest,myFixture
 ```
 
 ## Programmatic API

example/playwright/annotations-status.ts

@@ -0,0 +1,30 @@
+import { test, expect } from '@playwright/test';
+
+test('plain test', async () => {
+  await expect(true).toBe(true);
+});
+
+// .fail marks a test as expected to fail, but it still runs => not skipped
+test.fail('expected to fail test', async () => {
+  await expect(true).toBe(false);
+});
+
+// .slow triples the timeout, but the test still runs => not skipped
+test.slow('slow test', async () => {
+  await expect(true).toBe(true);
+});
+
+// runtime forms without a title declare no separate test
+test('runtime annotations have no title', async () => {
+  test.fail();
+  test.skip();
+  test.slow();
+  await expect(true).toBe(true);
+});
+
+// .fail inside a skipped suite => skipped (suite wins)
+test.describe.skip('skipped suite', () => {
+  test.fail('fail inside skipped suite', async () => {
+    await expect(true).toBe(false);
+  });
+});

example/playwright/custom-fixture-annotations.ts

@@ -0,0 +1,29 @@
+import { test as base } from '@playwright/test';
+
+const testFixture = base.extend<{ someFixture: any }>({
+  someFixture: async ({}, use) => {
+    await use({ name: 'custom fixture name' });
+  },
+});
+
+testFixture('plain alias test', async ({ someFixture }) => {
+  console.warn(someFixture.name);
+});
+
+testFixture.skip('skipped alias test', async ({ someFixture }) => {
+  console.warn(someFixture.name);
+});
+
+testFixture.fixme('fixme alias test', async ({ someFixture }) => {
+  console.warn(someFixture.name);
+});
+
+testFixture.fail('failing alias test', async ({ someFixture }) => {
+  console.warn(someFixture.name);
+});
+
+testFixture.describe('alias suite', () => {
+  testFixture.fixme('fixme test inside alias suite', async ({ someFixture }) => {
+    console.warn(someFixture.name);
+  });
+});

example/playwright/sibling-after-skipped-suite.ts

@@ -0,0 +1,16 @@
+import { test, expect } from '@playwright/test';
+
+test.describe.skip('skipped suite', () => {
+  test('inside skipped suite', async () => {
+    await expect(true).toBe(true);
+  });
+});
+
+// these are siblings declared AFTER the skipped suite closed - they must not inherit skipped
+test('sibling after skipped suite', async () => {
+  await expect(true).toBe(true);
+});
+
+test.fail('failing sibling after skipped suite', async () => {
+  await expect(true).toBe(false);
+});

src/lib/frameworks/playwright.js

@@ -23,6 +23,9 @@ module.exports = (ast, file = '', source = '', opts = {}) => {
   let beforeEachCode = '';
   let afterCode = '';
 
+  // built-in `test`/`it` plus any custom fixtures/aliases passed via --test-alias
+  const testNames = ['test', 'it', ...(opts?.testAlias || [])];
+
   function addSuite(path) {
     currentSuite = currentSuite.filter(s => s.loc.end.line > path.loc.start.line);
     path.tags = playwright.getTestProps({ parent: { expression: path } }).tags;
@@ -90,31 +93,33 @@ module.exports = (ast, file = '', source = '', opts = {}) => {
         }
       }
 
-      if (path.isIdentifier({ name: 'skip' })) {
+      // `.skip`/`.fixme` tests are skipped; `.fail`/`.slow` tests still run;
+      // runtime forms without a title (e.g. `test.skip()` inside a body) declare no test
+      if (path.isIdentifier() && ['skip', 'fixme', 'fail', 'slow'].includes(path.node.name)) {
         if (!path.parent || !path.parent.object) {
           return;
         }
         const name =
-          path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name;
+          path.parent.object.name || path.parent.object.property?.name || path.parent.object.callee?.object?.name;
 
-        if (name === 'test' || name === 'it') {
+        if (testNames.includes(name)) {
           // test or it
           if (!hasStringOrTemplateArgument(path.parentPath.container)) return;
 
           const testName = getStringValue(path.parentPath.container);
+          const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path));
           tests.push({
             name: testName,
-            suites: currentSuite
-              .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path))
-              .map(s => getStringValue(s)),
+            suites: suites.map(s => getStringValue(s)),
             line: getLineNumber(path),
-            code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber),
+            // end line comes from the enclosing call to capture the full test body
+            code: getCode(source, getLineNumber(path), getEndLineNumber(path.parentPath), isLineNumber),
             file,
-            skipped: true,
+            skipped: ['skip', 'fixme'].includes(path.node.name) || suites.some(s => s.skipped),
           });
         }
 
-        if (name === 'describe') {
+        if (name === 'describe' && (path.node.name === 'skip' || path.node.name === 'fixme')) {
           // suite
           if (!hasStringOrTemplateArgument(path.parentPath.container)) return;
           const suite = path.parentPath.container;
@@ -125,66 +130,7 @@ module.exports = (ast, file = '', source = '', opts = {}) => {
         // todo: handle "context"
       }
 
-      if (path.isIdentifier({ name: 'fixme' })) {
-        if (!path.parent || !path.parent.object) {
-          return;
-        }
-        const name =
-          path.parent.object.name || path.parent.object.property.name || path.parent.object.callee.object.name;
-
-        if (name === 'test' || name === 'it') {
-          // test or it
-          if (!hasStringOrTemplateArgument(path.parentPath.container)) return;
-
-          const testName = getStringValue(path.parentPath.container);
-          tests.push({
-            name: testName,
-            suites: currentSuite
-              .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path))
-              .map(s => getStringValue(s)),
-            line: getLineNumber(path),
-            code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber),
-            file,
-            skipped: true,
-          });
-        }
-
-        if (name === 'describe') {
-          // suite
-          if (!hasStringOrTemplateArgument(path.parentPath.container)) return;
-          const suite = path.parentPath.container;
-          suite.skipped = true;
-          addSuite(suite);
-        }
-
-        // todo: handle "context"
-      }
-
-      if (path.isIdentifier({ name: 'todo' })) {
-        if (!path.parent || !path.parent.object) {
-          return;
-        }
-        // todo tests => skipped tests
-        if (path.parent.object.name === 'test') {
-          // test
-          if (!hasStringOrTemplateArgument(path.parentPath.container)) return;
-
-          const testName = getStringValue(path.parentPath.container);
-          tests.push({
-            name: testName,
-            suites: currentSuite
-              .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path))
-              .map(s => getStringValue(s)),
-            line: getLineNumber(path),
-            code: getCode(source, getLineNumber(path), getEndLineNumber(path), isLineNumber),
-            file,
-            skipped: true,
-          });
-        }
-      }
-
-      const fixtureNames = [...['test', 'it'], ...(opts?.testAlias || [])];
-      for (const fiixtureName of fixtureNames || []) {
+      for (const fiixtureName of testNames) {
         if (path.isIdentifier({ name: fiixtureName })) {
           if (!hasStringOrTemplateArgument(path.parent)) return;
 
@@ -202,19 +148,19 @@ module.exports = (ast, file = '', source = '', opts = {}) => {
               afterCode;
 
           const testName = getStringValue(path.parent);
+          const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path));
 
           tests.push({
             name: testName,
-            suites: currentSuite
-              .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path))
-              .map(s => getStringValue(s)),
+            suites: suites.map(s => getStringValue(s)),
             updatePoint: getUpdatePoint(path.parent),
             line: getLineNumber(path),
             code,
             file,
             tags: [...getAllSuiteTags(currentSuite), ...playwright.getTestProps(path.parentPath).tags],
             annotations: playwright.getTestProps(path.parentPath).annotations,
-            skipped: !!currentSuite.filter(s => s.skipped).length,
+            // only suites still enclosing this line can mark it skipped (not closed siblings)
+            skipped: suites.some(s => s.skipped),
           });
 
           // stop the loop if the test is found
@@ -227,16 +173,15 @@ module.exports = (ast, file = '', source = '', opts = {}) => {
 
         if (!hasStringOrTemplateArgument(currentPath.parent)) return;
         const testName = getStringValue(currentPath.parent);
+        const suites = currentSuite.filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path));
         tests.push({
           name: testName,
-          suites: currentSuite
-            .filter(s => getEndLineNumber({ container: s }) >= getLineNumber(path))
-            .map(s => getStringValue(s)),
+          suites: suites.map(s => getStringValue(s)),
           updatePoint: getUpdatePoint(path.parent),
           line: getLineNumber(currentPath),
           code: getCode(source, getLineNumber(currentPath), getEndLineNumber(currentPath), isLineNumber),
           file,
-          skipped: !!currentSuite.filter(s => s.skipped).length,
+          skipped: suites.some(s => s.skipped),
         });
       }
     },

tests/playwright_test.js

@@ -248,19 +248,23 @@ test.describe.only('my test', () => {
     ast = jsParser.parse(source, { sourceType: 'unambiguous' });
     const tests = playwrightParser(ast, '', source);
 
-    expect(tests[1].code.trim()).to.equal("test.skip('my skip test @first', async ({ page }) => {".trim());
     expect(tests[1].name).to.equal('my skip test @first');
     expect(tests[1].suites.length).to.eql(1);
+    // code captures the full test body, not just the signature line
+    expect(tests[1].code).to.include("test.skip('my skip test @first', async ({ page }) => {");
+    expect(tests[1].code).to.include("await expect(page).toHaveURL('https://my.start.url/');");
   });
 
   it('should parse playwright-js tests with annotation including fixme', () => {
     source = fs.readFileSync('./example/playwright/annotations.js').toString();
     ast = jsParser.parse(source, { sourceType: 'unambiguous' });
     const tests = playwrightParser(ast, '', source);
 
-    expect(tests[2].code.trim()).to.equal("test.fixme('my fixme test @third', async ({ page }) => {".trim());
     expect(tests[2].name).to.equal('my fixme test @third');
     expect(tests[2].suites.length).to.eql(1);
+    // code captures the full test body, not just the signature line
+    expect(tests[2].code).to.include("test.fixme('my fixme test @third', async ({ page }) => {");
+    expect(tests[2].code).to.include("await expect(page).toHaveURL('https://my.start.url/');");
   });
 
   it('should parse playwright-ts tests with annotations', () => {
@@ -505,6 +509,90 @@ test.describe.only('my test', () => {
     expect(tests.length).to.equal(1);
   });
 
+  it('should parse annotations (.skip/.fixme/.fail) on a custom test alias', () => {
+    source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString();
+    ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] });
+    const tests = playwrightParser(ast, '', source, { testAlias: ['testFixture'] });
+
+    const byName = name => tests.find(t => t.name === name);
+
+    expect(tests.length).to.equal(5);
+    expect(byName('plain alias test').skipped).to.be.false;
+    expect(byName('skipped alias test').skipped).to.be.true;
+    expect(byName('fixme alias test').skipped).to.be.true;
+    expect(byName('failing alias test').skipped).to.be.false; // .fail still runs
+    expect(byName('fixme test inside alias suite').skipped).to.be.true;
+    expect(byName('fixme test inside alias suite').suites).to.deep.equal(['alias suite']);
+  });
+
+  it('should not parse custom alias annotations when the alias is not configured', () => {
+    source = fs.readFileSync('./example/playwright/custom-fixture-annotations.ts').toString();
+    ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] });
+    const tests = playwrightParser(ast, '', source);
+
+    expect(tests.length).to.equal(0);
+  });
+
+  describe('annotations status (.skip/.fixme/.fail/.slow)', () => {
+    let tests;
+
+    beforeEach(() => {
+      source = fs.readFileSync('./example/playwright/annotations-status.ts').toString();
+      ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] });
+      tests = playwrightParser(ast, '', source);
+    });
+
+    const byName = name => tests.find(t => t.name === name);
+
+    it('registers every named test exactly once (runtime no-title forms excluded)', () => {
+      // 5 named tests; inline `test.fail()` / `test.skip()` / `test.slow()` without a title add nothing
+      expect(tests.length).to.equal(5);
+      expect(tests.map(t => t.name)).to.deep.equal([
+        'plain test',
+        'expected to fail test',
+        'slow test',
+        'runtime annotations have no title',
+        'fail inside skipped suite',
+      ]);
+    });
+
+    it('keeps .fail tests runnable (not skipped)', () => {
+      expect(byName('expected to fail test').skipped).to.be.false;
+    });
+
+    it('keeps .slow tests runnable (not skipped)', () => {
+      expect(byName('slow test').skipped).to.be.false;
+    });
+
+    it('treats .fail inside a skipped suite as skipped', () => {
+      const test = byName('fail inside skipped suite');
+      expect(test.skipped).to.be.true;
+      expect(test.suites).to.deep.equal(['skipped suite']);
+    });
+
+    it('ignores runtime `test.fail()` / `test.skip()` / `test.slow()` calls without a title', () => {
+      const test = byName('runtime annotations have no title');
+      expect(test).to.not.be.undefined;
+      expect(test.skipped).to.be.false;
+    });
+  });
+
+  it('should not leak a skipped suite onto sibling tests declared after it', () => {
+    source = fs.readFileSync('./example/playwright/sibling-after-skipped-suite.ts').toString();
+    ast = jsParser.parse(source, { sourceType: 'unambiguous', plugins: ['typescript'] });
+    const tests = playwrightParser(ast, '', source);
+
+    const byName = name => tests.find(t => t.name === name);
+
+    expect(byName('inside skipped suite').skipped).to.be.true;
+    expect(byName('inside skipped suite').suites).to.deep.equal(['skipped suite']);
+    // siblings declared after the suite closed must not inherit it
+    expect(byName('sibling after skipped suite').skipped).to.be.false;
+    expect(byName('sibling after skipped suite').suites).to.deep.equal([]);
+    expect(byName('failing sibling after skipped suite').skipped).to.be.false;
+    expect(byName('failing sibling after skipped suite').suites).to.deep.equal([]);
+  });
+
   it('should not crash when test is assigned to a variable or inside an array (regression for issue #1)', () => {
     const source = `
       // This works normally