diff --git a/changelog.d/3083-hardener-shoulds-docs.changed.md b/changelog.d/3083-hardener-shoulds-docs.changed.md new file mode 100644 index 0000000000..c682826392 --- /dev/null +++ b/changelog.d/3083-hardener-shoulds-docs.changed.md @@ -0,0 +1 @@ +- Guides for `wheels test` now state the CLI throws `Wheels.TestsFailed` on `directoryRejected`, `bundlesDiscovered=0`, and compile-skipped specs — not only Fail/Error — so a vacuous or rejected scope is not documented as exit 0 (#3083) diff --git a/changelog.d/3083-hardener-shoulds.changed.md b/changelog.d/3083-hardener-shoulds.changed.md new file mode 100644 index 0000000000..85ffbbae2b --- /dev/null +++ b/changelog.d/3083-hardener-shoulds.changed.md @@ -0,0 +1 @@ +- Legacy CommandBox `wheels test` / `browser:test` refuse with a deprecation `error()` then `return` and no longer invoke TestBox (`testbox run`). Unrouted `vendor/wheels/controllers/Tests.cfc` is removed; `/wheels/core/tests` and `/wheels/app/tests` stay on `Public.cfc` (#3083) diff --git a/cli/lucli/tests/specs/commands/HardenerShouldsSpec.cfc b/cli/lucli/tests/specs/commands/HardenerShouldsSpec.cfc new file mode 100644 index 0000000000..21bcd5706a --- /dev/null +++ b/cli/lucli/tests/specs/commands/HardenerShouldsSpec.cfc @@ -0,0 +1,182 @@ +/** + * Hardener SHOULDs 4–6 (WheelsTest review slice). + * + * Source-scan / existence locks — no live CommandBox, no live runTests HTTP. + * Same altitude as MainCommandSpec / TestExitFailClosedSpec. + * + * SHOULD 4 — guides must teach CLI fail-closed (Wheels.TestsFailed), not + * "no Fail/Error means exit 0" / ignore directoryRejected. + * SHOULD 5 — legacy CommandBox cli/src test runners must not be a weaker + * exit path than LuCLI `wheels test` / `wheels browser test`. + * SHOULD 6 — vendor/wheels/controllers/Tests.cfc is unrouted and must not + * ship; allowlisted runners stay on Public.cfc. + */ +component extends="wheels.wheelstest.system.BaseSpec" { + + function beforeAll() { + variables.repoRoot = expandPath("/cli/../"); + variables.guidesRoot = variables.repoRoot & "web/sites/guides/src/content/docs/v4-0-0/"; + } + + function run() { + + describe("SHOULD 4 — docs teach fail-closed, not silent full-suite / vacuous exit 0", () => { + + it("testing.mdx names Wheels.TestsFailed and the ##3083 honesty signals", () => { + var src = fileRead(guidesRoot & "command-line-tools/wheels-commands/testing.mdx"); + expect(src).toInclude("Wheels.TestsFailed"); + expect(src).toInclude("directoryRejected"); + expect(src).toInclude("bundlesDiscovered"); + }); + + it("running-framework-tests.mdx says wheels test throws Wheels.TestsFailed", () => { + var src = fileRead(guidesRoot & "contributing/running-framework-tests.mdx"); + expect(src).toInclude("Wheels.TestsFailed"); + }); + + it("quick-start.mdx does not teach that a run with no Fail/Error always exits 0", () => { + var src = fileRead(guidesRoot & "command-line-tools/quick-start.mdx"); + expect(src).notToInclude("a run with no failures exits `0`"); + }); + + it("running-tests-locally.mdx names Wheels.TestsFailed for the CLI path", () => { + var src = fileRead(guidesRoot & "testing/running-tests-locally.mdx"); + expect(src).toInclude("Wheels.TestsFailed"); + }); + + it("ci-integration.mdx gates the CommandBox curl example on directoryRejected / bundlesDiscovered", () => { + var src = fileRead(guidesRoot & "testing/ci-integration.mdx"); + expect(src).toInclude("directoryRejected"); + expect(src).toInclude("bundlesDiscovered"); + expect(src).toInclude("Wheels.TestsFailed"); + }); + + }); + + describe("SHOULD 5 — CommandBox cli/src test runners are not a weaker exit path", () => { + + it("test/run.cfc does not swallow TestBox failing exit codes", () => { + var src = fileRead(expandPath("/cli/src/commands/wheels/test/run.cfc")); + expect(findNoCase("failing exit code", src)).toBe( + 0, + "CommandBox wheels test run must not catch-and-ignore TestBox failing exit codes." + ); + }); + + it("test/all.cfc does not swallow TestBox failing exit codes", () => { + var src = fileRead(expandPath("/cli/src/commands/wheels/test/all.cfc")); + expect(findNoCase("failing exit code", src)).toBe( + 0, + "CommandBox wheels test:all must not catch-and-ignore TestBox failing exit codes." + ); + }); + + it("test/unit.cfc does not swallow TestBox failing exit codes", () => { + var src = fileRead(expandPath("/cli/src/commands/wheels/test/unit.cfc")); + expect(findNoCase("failing exit code", src)).toBe( + 0, + "CommandBox wheels test:unit must not catch-and-ignore TestBox failing exit codes." + ); + }); + + it("test/integration.cfc does not swallow TestBox failing exit codes", () => { + var src = fileRead(expandPath("/cli/src/commands/wheels/test/integration.cfc")); + expect(findNoCase("failing exit code", src)).toBe( + 0, + "CommandBox wheels test:integration must not catch-and-ignore TestBox failing exit codes." + ); + }); + + it("browser/test.cfc refuses with a deprecation error instead of returning after Fail/Error", () => { + var src = fileRead(expandPath("/cli/src/commands/wheels/browser/test.cfc")); + expect(src).toInclude("DEPRECATED"); + expect(reFindNoCase("error\s*\(", src)).toBeGT( + 0, + "CommandBox wheels browser:test must error() so the process cannot exit 0 after Fail/Error." + ); + }); + + it("legacy CommandBox test runners point operators at LuCLI wheels test", () => { + var files = [ + "test/run.cfc", + "test/all.cfc", + "test/unit.cfc", + "test/integration.cfc", + "test/coverage.cfc", + "test/watch.cfc", + "browser/test.cfc" + ]; + for (var rel in files) { + var src = fileRead(expandPath("/cli/src/commands/wheels/" & rel)); + expect(src).toInclude( + "DEPRECATED", + rel & " must refuse with a deprecation instead of offering a weaker exit path." + ); + expect(src).toInclude("LuCLI"); + } + }); + + it("CommandBox test runners do not invoke testbox run after deprecation error()", () => { + // CommandBox error() prints red and does NOT abort. A later + // command("testbox run") still executes. C5. + var files = [ + "test/run.cfc", + "test/all.cfc", + "test/unit.cfc", + "test/integration.cfc", + "test/coverage.cfc", + "test/watch.cfc", + "browser/test.cfc" + ]; + for (var rel in files) { + var src = fileRead(expandPath("/cli/src/commands/wheels/" & rel)); + expect(findNoCase("testbox run", src)).toBe( + 0, + rel & " must not contain testbox run after error(); CommandBox error() does not abort." + ); + } + }); + + it("deprecation error() is followed by return so CommandBox cannot fall through", () => { + var files = [ + "test/run.cfc", + "test/all.cfc", + "test/unit.cfc", + "test/integration.cfc", + "test/coverage.cfc", + "test/watch.cfc", + "browser/test.cfc" + ]; + for (var rel in files) { + var src = fileRead(expandPath("/cli/src/commands/wheels/" & rel)); + expect(reFindNoCase("error\s*\(\s*""DEPRECATED[^;]*;\s*return\s*;", src)).toBeGT( + 0, + rel & " must return immediately after error(""DEPRECATED..."") — error() does not abort." + ); + } + }); + + }); + + describe("SHOULD 6 — orphan vendor/wheels/controllers/Tests.cfc is gone", () => { + + it("does not ship vendor/wheels/controllers/Tests.cfc", () => { + expect(fileExists(expandPath("/vendor/wheels/controllers/Tests.cfc"))).toBeFalse( + "Unrouted Tests.cfc is not on the ##3083 allowlisted runners; delete it rather than leave an orphan." + ); + }); + + it("allowlisted runners route to Public.cfc, not a Tests controller", () => { + var publicRoutes = fileRead(expandPath("/vendor/wheels/public/routes.cfm")); + var testRoutes = fileRead(expandPath("/vendor/wheels/tests/routes.cfm")); + expect(publicRoutes).toInclude("public####tests_testbox"); + expect(publicRoutes).toInclude("wheels####public####testbox"); + expect(reFindNoCase("to\s*=\s*""Tests####", publicRoutes)).toBe(0); + expect(reFindNoCase("to\s*=\s*""Tests####", testRoutes)).toBe(0); + }); + + }); + + } + +} diff --git a/cli/src/commands/wheels/browser/test.cfc b/cli/src/commands/wheels/browser/test.cfc index 620ab99226..d9b9ce9b34 100644 --- a/cli/src/commands/wheels/browser/test.cfc +++ b/cli/src/commands/wheels/browser/test.cfc @@ -23,89 +23,8 @@ component aliases="wheels browser:test, wheels browser test" extends="../base" { boolean verbose = false, string directory = "wheels.tests.specs.wheelstest" ) { - var projectRoot = getCWD(); - - try { - var manifest = browserService.getManifest(projectRoot); - var installDir = browserService.resolveInstallDir(); - var status = browserService.verifyInstall( - manifest=manifest, - installDir=installDir - ); - if (!status.installed) { - print.redLine("Playwright not installed."); - if (arrayLen(status.missing)) { - print.yellowLine("Missing: " & arrayToList(status.missing, ", ")); - } - if (arrayLen(status.mismatched)) { - print.yellowLine("SHA mismatch: " & arrayToList(status.mismatched, ", ")); - } - print.line(""); - print.line("Run: wheels browser:install"); - return; - } - } catch (any e) { - print.redLine("Error: " & e.message); - return; - } - - print.line("Running browser tests..."); - print.line("Directory: " & arguments.directory); - print.line(""); - - var serverInfo = command("server info").params(property="host").run(returnOutput=true); - var port = command("server info").params(property="port").run(returnOutput=true); - var host = trim(serverInfo) ?: "localhost"; - var portNum = trim(port) ?: "8080"; - var baseUrl = "http://" & host & ":" & portNum; - - var testUrl = baseUrl - & "/wheels/core/tests?db=sqlite&format=json&directory=" - & arguments.directory; - - try { - cfhttp(url=testUrl, method="GET", timeout=300, result="local.response"); - } catch (any e) { - print.redLine("Failed to reach test runner at: " & testUrl); - print.redLine("Is the server running? Try: server start"); - return; - } - - if (arguments.format == "json") { - print.line(local.response.fileContent); - return; - } - - try { - var data = deserializeJSON(local.response.fileContent); - print.line("Pass: " & data.totalPass & " Fail: " & data.totalFail & " Error: " & data.totalError); - print.line(""); - - for (var bundle in (data.bundleStats ?: [])) { - for (var suite in (bundle.suiteStats ?: [])) { - for (var spec in (suite.specStats ?: [])) { - if (listFindNoCase("Failed,Error", spec.status ?: "")) { - print.redLine( - " " & (spec.status ?: "") & ": " - & (spec.name ?: "unknown") - ); - if (arguments.verbose && len(spec.failMessage ?: "")) { - print.line(" " & left(spec.failMessage, 200)); - } - } - } - } - } - - if (data.totalFail == 0 && data.totalError == 0) { - print.greenLine("All browser tests passed."); - } - } catch (any e) { - print.redLine("Failed to parse test results: " & e.message); - if (arguments.verbose) { - print.line(left(local.response.fileContent ?: "", 500)); - } - } + error("DEPRECATED: CommandBox `wheels browser:test` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels browser test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } } diff --git a/cli/src/commands/wheels/test/all.cfc b/cli/src/commands/wheels/test/all.cfc index fa3f2f968d..317622f9d7 100644 --- a/cli/src/commands/wheels/test/all.cfc +++ b/cli/src/commands/wheels/test/all.cfc @@ -45,77 +45,7 @@ component aliases='wheels test:all' extends="../base" { string servername = "" ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core", "plugin"], - format=["txt", "json", "junit", "html"], - coverageReporter=["html", "json", "xml"] - } - ); - arguments.directory = resolveTestDirectory(arguments.type, arguments.directory); - - // Build the test URL - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Add coverage parameters if enabled - if (arguments.coverage) { - testUrl &= "&coverage=true"; - testUrl &= "&coverageBrowserOutputDir=#encodeForURL(arguments.coverageOutputDir)#"; - // Add coverage reporter format to URL - testUrl &= "&coverageReporter=#encodeForURL(arguments.coverageReporter)#"; - } - - // Add fail-fast parameter if specified - if (arguments.failFast) { - testUrl &= "&bail=true"; - } - - // Build TestBox command parameters - var params = { - runner = testUrl, - recurse = arguments.recurse, - verbose = arguments.verbose - }; - - // Add directory parameter if specified - if (len(arguments.directory)) { - params.directory = arguments.directory; - } - // Add optional filtering parameters - if (len(arguments.bundles)) { - params.testbundles = arguments.bundles; - } - - if (len(arguments.labels)) { - params.labels = arguments.labels; - } - - if (len(arguments.excludes)) { - params.excludes = arguments.excludes; - } - - if (len(arguments.filter)) { - // Handle filter parameter - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - try { - // Execute TestBox command - command('testbox run').params(argumentCollection=params).run(); - } catch (any e) { - // Let TestBox handle its own output and errors - if (!findNoCase("failing exit code", e.message)) { - rethrow; - } - } + error("DEPRECATED: CommandBox `wheels test:all` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } } \ No newline at end of file diff --git a/cli/src/commands/wheels/test/coverage.cfc b/cli/src/commands/wheels/test/coverage.cfc index 958eae8b18..deab986dad 100644 --- a/cli/src/commands/wheels/test/coverage.cfc +++ b/cli/src/commands/wheels/test/coverage.cfc @@ -46,107 +46,8 @@ component aliases='wheels test:coverage' extends="../base" { string outputFile = "test-results-coverage" ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core", "plugin"], - format=["txt", "json", "junit", "html"] - }, - numericRanges={ - threshold={min=0, max=1000} - } - ); - // Use relative path for outputDir to avoid issues with TestBox - var outputPath = arguments.outputDir; - - // Create output directory if it doesn't exist (using relative path) - var fullOutputPath = getCWD() & outputPath; - if (!directoryExists(fullOutputPath)) { - try { - directoryCreate(fullOutputPath, true, true); - detailOutput.create("output directory: #fullOutputPath#"); - } catch (any e) { - detailOutput.error("Failed to create output directory: #e.message#"); - outputPath = ""; // Use current directory - } - } - - // Build the test URL with coverage parameters - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Add coverage parameters to URL - testUrl &= "&coverage=true"; - testUrl &= "&coveragePathToCapture=#encodeForURL(arguments.pathsToCapture)#"; - testUrl &= "&coverageWhitelist=#encodeForURL(arguments.whitelist)#"; - testUrl &= "&coverageBlacklist=#encodeForURL(arguments.blacklist)#"; - testUrl &= "&coverageBrowserOutputDir=#encodeForURL(outputPath)#"; - - if (arguments.threshold > 0) { - testUrl &= "&coverageThreshold=#arguments.threshold#"; - } - - // Build TestBox command parameters - var params = { - runner = testUrl, - recurse = true, - verbose = arguments.verbose - }; - - // Add test filtering parameters - if (len(arguments.bundles)) { - params.testbundles = arguments.bundles; - } - - if (len(arguments.directory) && arguments.directory != "tests/specs") { - params.directory = arguments.directory; - } - - if (len(arguments.labels)) { - params.labels = arguments.labels; - } - - if (len(arguments.excludes)) { - params.excludes = arguments.excludes; - } - - if (len(arguments.filter)) { - // Handle filter parameter - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - // Use relative path for outputFile - if (len(outputPath)) { - params.outputFile = outputPath & "/" & arguments.outputFile; - } else { - params.outputFile = arguments.outputFile; - } - - // Add JSON output format to get structured results - params.outputFormats = "json,junit"; - - - var testsPassed = true; - - try { - // Execute TestBox command - command('testbox run').params(argumentCollection=params).run(); - - detailOutput.line(); - detailOutput.statusSuccess("Tests completed successfully!"); - - } catch (any e) { - detailOutput.statusFailed("Test execution failed: #e.message#"); - testsPassed = false; - } - + error("DEPRECATED: CommandBox `wheels test:coverage` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } } \ No newline at end of file diff --git a/cli/src/commands/wheels/test/integration.cfc b/cli/src/commands/wheels/test/integration.cfc index 35d00c6b6b..fdd808db3e 100644 --- a/cli/src/commands/wheels/test/integration.cfc +++ b/cli/src/commands/wheels/test/integration.cfc @@ -36,69 +36,8 @@ component aliases='wheels test:integration' extends="../base" { string servername = "" ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core", "plugin"], - format=["txt", "json", "junit", "html"] - } - ); - arguments.directory = resolveTestDirectory(arguments.type, arguments.directory); - - // Check if integration test directory exists, create if not - var integrationTestPath = fileSystemUtil.resolvePath(arguments.directory); - if (!directoryExists(integrationTestPath)) { - directoryCreate(integrationTestPath, true, true); - createSampleIntegrationTest(integrationTestPath); - detailOutput.create("integration test directory: #arguments.directory#"); - } - - // Build the test URL - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Build TestBox command parameters - var params = { - runner = testUrl, - directory = arguments.directory, - recurse = true, - verbose = arguments.verbose - }; - - // Add optional filtering parameters - if (len(arguments.bundles)) { - params.testbundles = arguments.bundles; - } - - if (len(arguments.labels)) { - params.labels = arguments.labels; - } - - if (len(arguments.excludes)) { - params.excludes = arguments.excludes; - } - - if (len(arguments.filter)) { - // Handle filter parameter - if it ends with "Test", treat as bundle - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - try { - // Execute TestBox command - command("testbox run").params(argumentCollection = params).run(); - } catch (any e) { - // Let TestBox handle its own output and errors - if (!findNoCase("failing exit code", e.message)) { - rethrow; - } - } + error("DEPRECATED: CommandBox `wheels test:integration` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } /** diff --git a/cli/src/commands/wheels/test/run.cfc b/cli/src/commands/wheels/test/run.cfc index a7c40c1a41..5c53fb0149 100644 --- a/cli/src/commands/wheels/test/run.cfc +++ b/cli/src/commands/wheels/test/run.cfc @@ -35,130 +35,8 @@ component extends="../base" { string reporter = "", ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core"], - format=["txt", "json", "junit", "html"], - reporter=["text", "json", "junit", "tap", "antjunit", "console", ""] - } - ); - arguments.directory = resolveTestDirectory(arguments.type, arguments.directory); - - // Map reporter to format if reporter is specified - if (structKeyExists(arguments, "reporter") && len(arguments.reporter)) { - // Map common reporter names to formats your runner expects - switch(arguments.reporter) { - case "console": - case "tap": - case "text": - arguments.format = "txt"; - break; - case "json": - arguments.format = "json"; - break; - case "junit": - case "antjunit": - arguments.format = "junit"; - - } - } - - // Build the test URL - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Build TestBox command parameters - var params = { - runner = testUrl - }; - params.recurse = arguments.recurse; - params.verbose = arguments.verbose; - - // Add bundles if specified - if (len(arguments.bundles)) { - params.testbundles = arguments.bundles; - } - - // Add directory if specified - if (len(arguments.directory)) { - params.directory = arguments.directory; - } - - // Handle filter parameter - if (len(arguments.filter)) { - // Filter can be used for testSpecs or testBundles depending on pattern - // If it looks like a bundle name (e.g., UserTest), use testBundles - // If it looks like a spec pattern, use testSpecs - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - // Handle lables parameter - if (len(arguments.lables)) { - params.labels = arguments.lables; - } - - // Handle coverage parameter - if (arguments.coverage) { - // Add coverage parameters to the URL since TestBox CLI doesn't directly support coverage - // You'll need to handle this in your runner.cfm - testUrl &= "&coverage=true"; - } - - // Update the runner URL in params - params.runner = testUrl; - - // Display test header - detailOutput.header("#ucase(arguments.type)# Tests"); - - // Display additional info if verbose - if (arguments.verbose) { - detailOutput.subHeader("Test Configuration"); - detailOutput.metric("Test URL", testUrl); - if (len(arguments.filter)) { - detailOutput.metric("Filter", arguments.filter); - } - if (len(arguments.lables)) { - detailOutput.metric("Labels", arguments.lables); - } - if (arguments.coverage) { - detailOutput.metric("Coverage", "Enabled"); - } - detailOutput.line(); - } - - try { - // Try using runCommand which should handle the CommandBox command properly - local.testboxCommand = command("testbox run").params(argumentCollection = params); - - // Execute without throwing on non-zero exit codes - try { - local.testboxCommand.run(); - } catch (any commandError) { - // If it's just an exit code error, ignore it and continue - // The actual test output should have been displayed already - if (findNoCase("failing exit code", commandError.message)) { - detailOutput.statusWarning("TestBox completed (exit code indicates test results)"); - } else { - // Re-throw if it's a genuine error - rethrow; - } - } - - } catch (any e) { - detailOutput.error("Error executing TestBox command: #e.message#"); - return; - } - - detailOutput.line(); - detailOutput.statusSuccess("#ucase(arguments.type)# Tests Completed"); + error("DEPRECATED: CommandBox `wheels test run` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } } \ No newline at end of file diff --git a/cli/src/commands/wheels/test/unit.cfc b/cli/src/commands/wheels/test/unit.cfc index b3d1c1afdd..477e430783 100644 --- a/cli/src/commands/wheels/test/unit.cfc +++ b/cli/src/commands/wheels/test/unit.cfc @@ -36,69 +36,8 @@ component aliases='wheels test:unit' extends="../base" { string servername = "" ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core", "plugin"], - format=["txt", "json", "junit", "html"] - } - ); - arguments.directory = resolveTestDirectory(arguments.type, arguments.directory); - - // Check if unit test directory exists, create if not - var unitTestPath = fileSystemUtil.resolvePath(arguments.directory); - if (!directoryExists(unitTestPath)) { - directoryCreate(unitTestPath, true, true); - createSampleUnitTest(unitTestPath); - detailOutput.create("unit test directory: #arguments.directory#"); - } - - // Build the test URL using arguments - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Build TestBox command parameters - var params = { - runner = testUrl, - directory = arguments.directory, - recurse = true, - verbose = arguments.verbose - }; - - // Add optional filtering parameters - if (len(arguments.bundles)) { - params.testbundles = arguments.bundles; - } - - if (len(arguments.labels)) { - params.labels = arguments.labels; - } - - if (len(arguments.excludes)) { - params.excludes = arguments.excludes; - } - - if (len(arguments.filter)) { - // Handle filter parameter - if it ends with "Test", treat as bundle - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - try { - // Execute TestBox command - command("testbox run").params(argumentCollection = params).run(); - } catch (any e) { - // Let TestBox handle its own output and errors - if (!findNoCase("failing exit code", e.message)) { - rethrow; - } - } + error("DEPRECATED: CommandBox `wheels test:unit` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } /** diff --git a/cli/src/commands/wheels/test/watch.cfc b/cli/src/commands/wheels/test/watch.cfc index db063ab59d..a64f4ec114 100644 --- a/cli/src/commands/wheels/test/watch.cfc +++ b/cli/src/commands/wheels/test/watch.cfc @@ -38,92 +38,7 @@ component aliases='wheels test:watch' extends="../base" { string servername = "" ) { requireWheelsApp(getCWD()); - arguments = reconstructArgs( - argStruct=arguments, - allowedValues={ - type=["app", "core", "plugin"], - format=["txt", "json", "junit", "html"] - }, - numericRanges={ - delay={min=100, max=60000} - } - ); - arguments.directory = resolveTestDirectory(arguments.type, arguments.directory); - - detailOutput.header("Starting Test Watcher"); - detailOutput.divider("=", 40); - detailOutput.line(); - detailOutput.statusInfo("Watching for file changes..."); - detailOutput.output("Press Ctrl+C to stop watching"); - detailOutput.line(); - - // Build the test URL - var testUrl = buildTestUrl( - type = arguments.type, - servername = arguments.servername, - format = arguments.format - ); - - // Build TestBox watch command parameters - var params = { - runner = testUrl, - directory = arguments.directory, - delay = arguments.delay, - verbose = arguments.verbose - }; - - // Add optional filtering parameters - if (len(arguments.bundles)) { - params.bundles = arguments.bundles; - } - - if (len(arguments.labels)) { - params.labels = arguments.labels; - } - - if (len(arguments.excludes)) { - params.excludes = arguments.excludes; - } - - if (len(arguments.filter)) { - // Handle filter parameter - if (reFindNoCase("Test$", arguments.filter)) { - params.testBundles = arguments.filter; - } else { - params.testSpecs = arguments.filter; - } - } - - // Show watching configuration - detailOutput.subHeader("Configuration"); - detailOutput.metric("Type", "#arguments.type# tests"); - detailOutput.metric("Directory", arguments.directory); - detailOutput.metric("Format", arguments.format); - detailOutput.metric("Delay", "#arguments.delay#ms"); - - if (len(arguments.filter)) { - detailOutput.metric("Filter", arguments.filter); - } - - if (len(arguments.labels)) { - detailOutput.metric("Labels", arguments.labels); - } - - detailOutput.line(); - detailOutput.output("Executing: testbox watch"); - detailOutput.line(); - - try { - // Execute TestBox watch command - command('testbox watch').params(argumentCollection=params).run(); - } catch (any e) { - // Handle interruption gracefully - if (findNoCase("interrupted", e.message) || findNoCase("ctrl", e.message)) { - detailOutput.line(); - detailOutput.statusInfo("Watch mode stopped by user"); - } else { - detailOutput.error("Error in watch mode: #e.message#"); - } - } + error("DEPRECATED: CommandBox `wheels test:watch` is frozen and does not fail-closed. Use the LuCLI `wheels` binary (`wheels test`). Removal scheduled for Wheels 5.0. See cli/src/README.md."); + return; } } \ No newline at end of file diff --git a/vendor/wheels/controllers/Tests.cfc b/vendor/wheels/controllers/Tests.cfc deleted file mode 100644 index 0954b6d4a8..0000000000 --- a/vendor/wheels/controllers/Tests.cfc +++ /dev/null @@ -1,146 +0,0 @@ -component extends="Controller" { - - /** - * Initialize the controller - */ - public void function config() { - // This controller provides JSON responses for test running - provides("json"); - } - - /** - * Run tests and return results - */ - public void function index() { - // Set long timeout for test execution - setting requesttimeout="300"; - - // Get test parameters - param name="params.type" default="app"; - param name="params.format" default="json"; - param name="params.reporter" default="json"; - param name="params.filter" default=""; - param name="params.group" default=""; - param name="params.coverage" default="false"; - param name="params.failFast" default="false"; - param name="params.watch" default="false"; - - local.result = { - success = false, - message = "", - tests = {}, - coverage = {} - }; - - try { - // Determine test directory based on type - local.testDirectory = ""; - switch(params.type) { - case "core": - local.testDirectory = expandPath("/wheels/tests"); - break; - case "app": - default: - local.testDirectory = expandPath("/tests"); - break; - } - - // Check if test directory exists - if (!directoryExists(local.testDirectory)) { - local.result.message = "Test directory not found: #local.testDirectory#"; - renderWith(local.result); - return; - } - - // Check if WheelsTest is available - if (!structKeyExists(application, "testbox") && !fileExists(expandPath("/wheels/wheelstest/system/TestBox.cfc"))) { - local.result.message = "WheelsTest is not installed. Please ensure the wheels test framework is available."; - renderWith(local.result); - return; - } - - // Build WheelsTest options - local.testboxOptions = { - directory = local.testDirectory, - recurse = true, - reporter = params.reporter, - labels = params.group, - testBundles = params.filter, - coverageEnabled = params.coverage, - coveragePathToCapture = expandPath("/app"), - coverageWhitelist = "", - coverageBlacklist = "tests,wheelstest,vendor,wheels" - }; - - // Run tests using WheelsTest - if (fileExists(expandPath("/wheels/wheelstest/system/TestBox.cfc"))) { - local.testbox = new wheels.wheelstest.system.TestBox(); - local.testResults = local.testbox.run(argumentCollection=local.testboxOptions); - - // Format results - local.result.success = true; - local.result.tests = { - totalSpecs = local.testResults.getTotalSpecs(), - totalPass = local.testResults.getTotalPass(), - totalFail = local.testResults.getTotalFail(), - totalError = local.testResults.getTotalError(), - totalSkipped = local.testResults.getTotalSkipped(), - totalDuration = local.testResults.getTotalDuration(), - bundles = [] - }; - - // Add bundle details - for (local.bundle in local.testResults.getBundleStats()) { - arrayAppend(local.result.tests.bundles, { - name = local.bundle.name, - totalSpecs = local.bundle.totalSpecs, - totalPass = local.bundle.totalPass, - totalFail = local.bundle.totalFail, - totalError = local.bundle.totalError, - totalSkipped = local.bundle.totalSkipped - }); - } - - // Add coverage if enabled - if (params.coverage && structKeyExists(local.testResults, "getCoverageData")) { - local.result.coverage = local.testResults.getCoverageData(); - } - - local.result.message = "Tests completed successfully"; - } else { - // Fallback for when WheelsTest isn't properly installed - local.result.message = "WheelsTest installation not found. Please ensure the wheels test framework is properly installed."; - } - - } catch (any e) { - local.result.success = false; - local.result.message = "Error running tests: #e.message# #e.detail#"; - } - - // Return JSON response - renderWith(local.result); - } - - /** - * Run a single test bundle - */ - public void function run() { - // Redirect to index with parameters - params.type = "single"; - index(); - } - - /** - * Get test coverage report - */ - public void function coverage() { - local.result = { - success = false, - message = "Coverage reporting not yet implemented", - coverage = {} - }; - - renderWith(local.result); - } - -} \ No newline at end of file diff --git a/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/quick-start.mdx b/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/quick-start.mdx index ef75bd1b3b..9fd5917df4 100644 --- a/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/quick-start.mdx +++ b/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/quick-start.mdx @@ -73,7 +73,7 @@ With the server still running in the background: wheels test posts ``` -The trailing positional is a directory scope, not a filename match — `posts` expands to `tests.specs.posts`. The scaffold generated `PostSpec.cfc` under `tests/specs/models/` and `PostsControllerSpec.cfc` under `tests/specs/controllers/`, so on a fresh scaffold the `posts` scope matches neither directory and the run reports `0 passed`. Use `wheels test models` or `wheels test controllers` to scope to the generated specs — though note that plain `wheels test` also reports `0 passed` here, because the generated specs are empty `describe` stubs until you write expectations into them. A run with failing or erroring specs exits non-zero; a run with no failures exits `0`. +The trailing positional is a directory scope, not a filename match — `posts` expands to `tests.specs.posts`. The scaffold generated `PostSpec.cfc` under `tests/specs/models/` and `PostsControllerSpec.cfc` under `tests/specs/controllers/`, so on a fresh scaffold the `posts` scope matches neither directory. The CLI treats that vacuous run as a failure (`Wheels.TestsFailed` — `bundlesDiscovered=0`). Use `wheels test models` or `wheels test controllers` to scope to the generated specs. Empty `describe` stubs under those directories still report `0 passed` and exit 0, because they loaded; a rejected `directory=`, a 0-bundle discovery, compile-skipped specs, or Fail/Error all throw `Wheels.TestsFailed`. ## 7. What's next diff --git a/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/wheels-commands/testing.mdx b/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/wheels-commands/testing.mdx index be4ae24edd..95581c7886 100644 --- a/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/wheels-commands/testing.mdx +++ b/web/sites/guides/src/content/docs/v4-0-0/command-line-tools/wheels-commands/testing.mdx @@ -90,7 +90,7 @@ The engine identifiers are `postgres` (not `postgresql`) and `sqlserver` (not `m wheels test --filter=models ``` -Runs every spec under `tests/specs/models/`. Exit code is non-zero if any spec fails or errors. +Runs every spec under `tests/specs/models/`. The CLI throws `Wheels.TestsFailed` (non-zero exit) when any spec fails or errors, when the runner reports `directoryRejected` or `bundlesDiscovered=0` (a rejected or vacuous `directory=` scope — [#3083](https://github.com/wheels-dev/wheels/issues/3083)), or when compile-skipped `*Spec.cfc` files were skipped by the runner. #### Testing against different engines diff --git a/web/sites/guides/src/content/docs/v4-0-0/contributing/running-framework-tests.mdx b/web/sites/guides/src/content/docs/v4-0-0/contributing/running-framework-tests.mdx index 0a32044227..ba06b26630 100644 --- a/web/sites/guides/src/content/docs/v4-0-0/contributing/running-framework-tests.mdx +++ b/web/sites/guides/src/content/docs/v4-0-0/contributing/running-framework-tests.mdx @@ -29,7 +29,7 @@ Environment knobs: - `DB=mysql bash tools/test-local.sh` — run against a different database; bring up the container yourself first (see the Docker matrix below). - `WHEELS_BROWSER_TEST_BASE_URL` — auto-set to `http://localhost:${PORT}` so browser specs hit the same server the script starts. -You can also run the framework suite through the CLI: `wheels test --core`, with `--db=` selecting between the `wheelstestdb_*` datasources the matrix wires up (`--db` is only honoured with `--core`). +You can also run the framework suite through the CLI: `wheels test --core`, with `--db=` selecting between the `wheelstestdb_*` datasources the matrix wires up (`--db` is only honoured with `--core`). The CLI throws `Wheels.TestsFailed` (non-zero exit) on Fail/Error, on `directoryRejected` / `bundlesDiscovered=0` ([#3083](https://github.com/wheels-dev/wheels/issues/3083)), and on compile-skipped `*Spec.cfc` files. Do not treat a curl of `/wheels/core/tests` as pass/fail from `totalFail`/`totalError` alone — check those honesty fields too. ## The core test-runner URL @@ -70,12 +70,14 @@ docker compose up -d lucee7 adobe2025 curl -s -o /tmp/lucee7.json "http://localhost:60007/wheels/core/tests?db=sqlite&format=json" curl -s -o /tmp/adobe2025.json "http://localhost:62025/wheels/core/tests?db=sqlite&format=json" -# Summarize +# Summarize — fail closed on #3083 honesty fields, not just Fail/Error for f in /tmp/lucee7.json /tmp/adobe2025.json; do python3 -c " -import json +import json, sys d = json.load(open('$f')) -print('$f', d['totalPass'], 'pass', d['totalFail'], 'fail', d['totalError'], 'error') +print('$f', d.get('totalPass',0), 'pass', d.get('totalFail',0), 'fail', d.get('totalError',0), 'error', 'rejected=', d.get('directoryRejected'), 'bundles=', d.get('bundlesDiscovered')) +bad = d.get('directoryRejected') or d.get('bundlesDiscovered', 1) == 0 or d.get('totalFail',0) or d.get('totalError',0) +sys.exit(1 if bad else 0) " done ``` diff --git a/web/sites/guides/src/content/docs/v4-0-0/testing/ci-integration.mdx b/web/sites/guides/src/content/docs/v4-0-0/testing/ci-integration.mdx index e7047ce7a8..ee685d7df1 100644 --- a/web/sites/guides/src/content/docs/v4-0-0/testing/ci-integration.mdx +++ b/web/sites/guides/src/content/docs/v4-0-0/testing/ci-integration.mdx @@ -59,7 +59,7 @@ Treat this as the shape, not gospel — the step that varies per app is the data Two details worth knowing: - **`WHEELS_CI=true`** is a signal for the test harness — most visibly it gates browser specs (below). Set it on every CI job. -- **`wheels test --ci`** exits non-zero on failure and emits one GitHub Actions `::error` annotation per failed or errored spec, so failures surface inline in the PR checks ([#3113](https://github.com/wheels-dev/wheels/issues/3113)). +- **`wheels test --ci`** throws `Wheels.TestsFailed` (non-zero exit) on Fail/Error, on `directoryRejected` / `bundlesDiscovered=0`, and on compile-skipped specs, and emits one GitHub Actions `::error` annotation per failed or errored spec so failures surface inline in the PR checks ([#3113](https://github.com/wheels-dev/wheels/issues/3113), [#3083](https://github.com/wheels-dev/wheels/issues/3083)). ## Minimum viable — without the wheels CLI @@ -93,7 +93,10 @@ jobs: import json d = json.load(open('results.json')) print(d.get('totalPass',0), 'pass', d.get('totalFail',0), 'fail', d.get('totalError',0), 'error') - raise SystemExit(0 if d.get('totalFail',0)==0 and d.get('totalError',0)==0 else 1) + rejected = d.get('directoryRejected') + vacuous = d.get('bundlesDiscovered', 1) == 0 + failed = d.get('totalFail',0) or d.get('totalError',0) + raise SystemExit(0 if not (rejected or vacuous or failed) else 1) " ``` @@ -102,7 +105,7 @@ Adjust the port to whatever your `server.json` declares. Two constraints: - **The app must run as `development`** — the `/wheels/*` runner surfaces are development-only ([#2903](https://github.com/wheels-dev/wheels/pull/2903)). A CI job is exactly the place that's fine; just don't point this at a production-configured deployment. - **The runner uses the app's configured datasource** (no `_test` auto-swap without the CLI) — configure the CI app's datasource to a throwaway database and let `tests/populate.cfm` build it. -A filtered run that names a directory the runner doesn't recognize silently runs the **full** suite ([#3083](https://github.com/wheels-dev/wheels/issues/3083)) — when you filter in CI (`&directory=tests.specs.models`), also assert the run wasn't vacuous (a suspiciously low spec count or a zero-bundle warning in the payload). +A filtered run that names a directory the runner doesn't recognize silently runs the **full** suite ([#3083](https://github.com/wheels-dev/wheels/issues/3083)) — the payload sets `directoryRejected: true`. When you filter in CI (`&directory=tests.specs.models`), fail the job on `directoryRejected` or `bundlesDiscovered=0` (the snippet above does). Prefer `wheels test --ci`, which throws `Wheels.TestsFailed` for those cases. ## Reporter output for CI diff --git a/web/sites/guides/src/content/docs/v4-0-0/testing/running-tests-locally.mdx b/web/sites/guides/src/content/docs/v4-0-0/testing/running-tests-locally.mdx index 7d7e3e6dbb..a4aa42ce50 100644 --- a/web/sites/guides/src/content/docs/v4-0-0/testing/running-tests-locally.mdx +++ b/web/sites/guides/src/content/docs/v4-0-0/testing/running-tests-locally.mdx @@ -23,7 +23,7 @@ Your app's specs live under `tests/specs/` and run through the framework's built ## With the wheels CLI -`wheels test` detects your running dev server, reloads the app, hits the test runner, and prints pass/fail counts. No ceremony: +`wheels test` detects your running dev server, reloads the app, hits the test runner, and prints pass/fail counts. A failing, errored, rejected, vacuous, or compile-skipped run throws `Wheels.TestsFailed` (non-zero exit) — do not treat a quiet `0 passed` as success. No ceremony: ```bash title="your shell — in your app root" # Run the full suite @@ -67,7 +67,7 @@ Keep the dev loop as: change a spec, refresh the URL. Two things to know: - **The datasource is whatever the app is using.** The `_test` auto-swap is a CLI convenience; the raw runner exercises the datasource your app is currently configured with. If you drive tests by URL routinely, point your development datasource at a database you're happy to have `tests/populate.cfm` rebuild — or keep populate idempotent (it should be anyway; see [Fixtures & Test Data](/v4-0-0/testing/fixtures-and-test-data/)). - **Isolated application scope.** With the stock `Application.cfc` snippet (`include` of `vendor/wheels/events/testcontext.cfm` after `config/app.cfm`), `/wheels/app/tests` and `/wheels/core/tests` bind a separate CFML application name. Concurrent browsing of the same server keeps live config. See [Fixtures & Test Data](/v4-0-0/testing/fixtures-and-test-data/). -An unrecognized `directory=` value is ignored and the **full** suite runs instead ([#3083](https://github.com/wheels-dev/wheels/issues/3083)) — if a filtered run looks suspiciously slow or broad, check the value against your `tests/specs/` layout. +An unrecognized `directory=` value is ignored and the **full** suite runs instead ([#3083](https://github.com/wheels-dev/wheels/issues/3083)) — the JSON flags this with `directoryRejected: true`. A `directory=` that names a single spec file (or a folder with no `*Spec.cfc` files) yields `bundlesDiscovered: 0`. `wheels test` throws `Wheels.TestsFailed` in both cases. If you curl the runner yourself, check those fields; if a filtered run looks suspiciously slow or broad, check the value against your `tests/specs/` layout. ## Filtering