From 9e42dded2a81e501751b48af9ee32c7047fe5d57 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Wed, 4 Mar 2026 13:52:06 +0100 Subject: [PATCH 01/24] New branch and updated output format --- .../main/kotlin/org/evomaster/core/output/OutputFormat.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index c5aeb96248..73e2bf1d32 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -19,7 +19,8 @@ enum class OutputFormat { KOTLIN_JUNIT_5, JS_JEST, //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed - PYTHON_UNITTEST + PYTHON_UNITTEST, + JS_JEST_PLAYWRIGHT ; fun isJava() = this.name.startsWith("java_", true) @@ -41,4 +42,6 @@ enum class OutputFormat { fun isPython() = this.name.startsWith("python_", true) + fun isPlaywright() = this.name.startsWith("js_jest_playwright", true) + } From 43c2d8f918bfbb95377931c0e9f09b8f73b5c641 Mon Sep 17 00:00:00 2001 From: FredRaw Date: Thu, 5 Mar 2026 15:34:11 +0100 Subject: [PATCH 02/24] Add initial Playwright output support --- .../org/evomaster/core/output/OutputFormat.kt | 3 + .../output/service/HttpWsTestCaseWriter.kt | 90 +++++++++++++------ .../core/output/service/TestCaseWriter.kt | 5 +- .../core/output/service/TestSuiteWriter.kt | 33 ++++--- 4 files changed, 89 insertions(+), 42 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index 73e2bf1d32..350169af69 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,6 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, + JS_JEST_PLAYWRIGHT, // Testing new playwright imp //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST, JS_JEST_PLAYWRIGHT @@ -35,6 +36,8 @@ enum class OutputFormat { fun isJUnit4() = this.name.endsWith("junit_4", true) + fun isPlaywright() = this.name.endsWith("_playwright", true) // Testing new playwright imp + fun isJUnit() = this.name.contains("_junit_", true) @Deprecated("No longer supported") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 260ed3e894..168c1711a1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -60,7 +60,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") + format.isJavaScript() && format.isPlaywright() -> lines.append(" -64 HttpWsTestCaseWriter.kt- ") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -103,7 +104,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") - format.isJavaScript() -> lines.append("const $resVarName = ") + format.isJavaScript() && !format.isPlaywright()-> lines.append("const $resVarName = ") + format.isJavaScript() && format.isPlaywright() -> lines.append(" -108 HttpWsTestCaseWriter.kt- ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") } @@ -111,7 +113,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") + format.isJavaScript() && format.isPlaywright() -> lines.append(" -117 HttpWsTestCaseWriter.kt- ") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -186,7 +189,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" - format.isJavaScript() -> ".set('Accept', " + format.isJavaScript() && !format.isPlaywright()-> ".set('Accept', " + format.isJavaScript() && format.isPlaywright() -> " -193 HttpWsTestCaseWriter.kt- " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " else -> throw IllegalArgumentException("Invalid format: $format") @@ -227,7 +231,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" - format.isJavaScript() -> "set" + format.isJavaScript() && !format.isPlaywright()-> "set" + format.isJavaScript() && format.isPlaywright() -> " -235 HttpWsTestCaseWriter.kt- " format.isPython() -> "headers = {}" else -> throw IllegalArgumentException("Not supported format: $format") } @@ -276,7 +281,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isJavaScript() && !format.isPlaywright()-> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isJavaScript() && format.isPlaywright() -> lines.add(" -285 HttpWsTestCaseWriter.kt- ") // Python cookies are set alongside the headers and body when performing the request } } @@ -303,7 +309,17 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val code = res.getStatusCode() when { - format.isJavaScript() -> { + + format.isJavaScript() && format.isPlaywright() -> { // This need a re-write! + val statusAssert = " - 314 - HttpWsTestCaseWriter.kt- " + if (res.getFlakyStatusCode() == null){ + lines.add(statusAssert) + }else{ + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(statusAssert) + } + } + format.isJavaScript() && !format.isPlaywright()-> { // This might need a second look! val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -436,8 +452,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $varName = System.currentTimeMillis()") } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") - } else if(format.isJavaScript()) { + } else if(format.isJavaScript() && !format.isPlaywright()) { lines.addStatement("$varName = performance.now()") + } else if(format.isJavaScript() && format.isPlaywright()) { + lines.addStatement(" -448 HttpWsTestCaseWriter.kt- ") } lines.addEmpty(1) @@ -455,8 +473,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $finalVarName = System.currentTimeMillis() - $varName") } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") - } else if(format.isJavaScript()) { + } else if(format.isJavaScript() && !format.isPlaywright()) { lines.addStatement("$finalVarName = performance.now() - $varName") + } else if(format.isJavaScript() && format.isPlaywright()) { + lines.addStatement(" -469 HttpWsTestCaseWriter.kt- ") } lines.addEmpty(1) @@ -466,7 +486,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: SQL Injection vulnerability detected in this call. Expected response time (sqliInjectedSleepDurationMs) should be greater than ${config.sqliInjectedSleepDurationMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName > ${config.sqliInjectedSleepDurationMs})") - format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") + format.isJavaScript() && !format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") + format.isJavaScript() && format.isPlaywright() -> lines.addStatement(" -480 HttpWsTestCaseWriter.kt- ") format.isPython() -> lines.addStatement("assert $finalVarName > ${config.sqliInjectedSleepDurationMs}") else -> {} } @@ -474,9 +495,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript() && !format.isPlaywright()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript() && format.isPlaywright() -> lines.addStatement(" -489 HttpWsTestCaseWriter.kt- ") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") - else -> {} + else -> { } // This should have some information? } } } @@ -508,11 +530,15 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isJavaScript() -> { lines.indent(2) - //in SuperAgent, verb must be first - handleVerbEndpoint(baseUrlOfSut, call, lines) - lines.append(getAcceptHeader(call, res)) - handleHeaders(call, lines) - handleBody(call, lines) + if (format.isPlaywright()) { // This needs to be checked! + lines.append(" -524 HttpWsTestCaseWriter.kt- ") + } else { + //in SuperAgent, verb must be first + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.append(getAcceptHeader(call, res)) + handleHeaders(call, lines) + handleBody(call, lines) + } } format.isCsharp() -> { @@ -540,7 +566,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() -> "send" + format.isJavaScript() && !format.isPlaywright()-> "send" + format.isPython() && format.isPlaywright() -> " - 570 - HttpWsTestCaseWriter.kt" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -562,7 +589,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isPlaywright() && format.isPlaywright() -> lines.add(" - 593 - HttpWsTestCaseWriter.kt- ") format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -660,7 +688,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = ${bodyLines.first()}") } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) // Needs to be checked! fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, false) } } else { @@ -685,7 +713,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("${bodyLines.last()}") } } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) // Needs to be checked fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, true) } } @@ -774,8 +802,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isJavaScript() -> - "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" + format.isJavaScript() && format.isPlaywright() -> "- 794 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() && !format.isPlaywright()-> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") } @@ -806,7 +834,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isJavaScript() -> + format.isJavaScript() && format.isPlaywright() -> "- 826 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() && !format.isPlaywright()-> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" format.isCsharp() -> "Assert.Contains(\"$bodyTypeSimplified\", $responseVariableName.Content.Headers.GetValues(\"Content-Type\").First());" @@ -871,7 +900,14 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - if (format.isJavaScript()) { + if (format.isJavaScript() && format.isPlaywright()) { + /* + This is to deal with very weird behavior in SuperAgent that crashes the tests + for status codes different from 2xx... + so, here we make it passes as long as a status was present + */ + lines.add(" - 886 - HttpWsTestCaseWriter.kt - ") + } else if(format.isJavaScript() && !format.isPlaywright()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... @@ -880,7 +916,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add(".ok(res => res.status)") } - if (lines.shouldUseSemicolon()) { /* FIXME this is wrong when // is in a string of response, like a URL. @@ -928,7 +963,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" + format.isJavaScript() && format.isPlaywright() -> " - 955 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() && !format.isPlaywright()-> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index b36b04d0f4..de88733a81 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -116,7 +116,8 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && !format.isPlaywright()-> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && format.isPlaywright() -> lines.add(" - 120 TestCaseWriter.kt - ") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } @@ -137,7 +138,7 @@ abstract class TestCaseWriter { lines.add("}") } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // add to code block - ok for playwright? lines.append(");") } return lines diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 51f87e7288..d2cb8edf72 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -511,17 +511,23 @@ class TestSuiteWriter { } if (format.isJavaScript()) { - lines.add("const superagent = require(\"superagent\");") + if (format.isPlaywright()) { + lines.add(" - 515 - TestSuiteWriter.kt)") + } else { + lines.add("const superagent = require(\"superagent\");") + } val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) lines.add("const $jsImport = require(\"./$javascriptUtilsFilename\");") - + lines.add(" - 523 - TestSuiteWriter.kt - What about this const EM thing?. Do we need it?") if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") + lines.add(" - 526 - TestSuiteWriter.kt - What about controllerName. Do we need it?") } if (config.testTimeout > 0) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") + lines.add(" - 529 - TestSuiteWriter.kt - Do we need the jest.setTimeout line above?") } } @@ -706,7 +712,7 @@ class TestSuiteWriter { lines.add("private lateinit var $driver : RemoteWebDriver") } - } else if (config.outputFormat.isJavaScript()) { + } else if (config.outputFormat.isJavaScript()) { // Leave as is for playwright? if (!config.blackBox || config.bbExperiments) { lines.add("const $controller = new $controllerName();") @@ -763,13 +769,14 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() -> lines.add("beforeAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("- 771 - TestSuiteWriter.kt") } lines.block { if (!config.blackBox) { when { - config.outputFormat.isJavaScript() -> { + config.outputFormat.isJavaScript() -> { // Add something for playwright white box testing! addStatement("await $controller.setupForGeneratedTest()", lines) addStatement("$baseUrlOfSut = await $controller.startSut()", lines) } @@ -795,7 +802,7 @@ class TestSuiteWriter { when { format.isJavaOrKotlin() -> addStatement("assertNotNull(baseUrlOfSut)", lines) - format.isJavaScript() -> addStatement("expect(baseUrlOfSut).toBeTruthy()", lines) + format.isJavaScript() -> addStatement("expect(baseUrlOfSut).toBeTruthy()", lines) // Add something for playwright white box testing } } @@ -895,7 +902,7 @@ class TestSuiteWriter { testCaseWriter.addExtraInitStatement(lines) } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // End statement of blocks. Valid for playwright too lines.append(");") } } @@ -918,13 +925,13 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") + format.isJavaScript() -> lines.add("afterAll( async () =>") // white box part? } if (!format.isCsharp()) { lines.block { when { - format.isJavaScript() -> { + format.isJavaScript() -> { // white box part? addStatement("await $controller.stopSut()", lines) } else -> { @@ -949,7 +956,7 @@ class TestSuiteWriter { } } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // white box part? lines.append(");") } } @@ -971,7 +978,7 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") + format.isJavaScript() -> lines.add("beforeEach(async () => ") // white box part? //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } @@ -979,7 +986,7 @@ class TestSuiteWriter { lines.block { - if (format.isJavaScript()) { + if (format.isJavaScript()) { // white box part? //TODO add resetDatabase addStatement("await $controller.resetStateOfSUT()", lines) } else if (format.isJavaOrKotlin()) { @@ -1018,7 +1025,7 @@ class TestSuiteWriter { } } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // white box part? lines.append(");") } } From 444d2807622666f6891376c6e5113d9fa301bf50 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Mon, 16 Mar 2026 16:46:33 +0100 Subject: [PATCH 03/24] Add support for Playwright. HttpWsTestCaseWriter still work in progress. --- .../output/service/HttpWsTestCaseWriter.kt | 143 ++++++++++++------ .../core/output/service/TestCaseWriter.kt | 17 ++- .../core/output/service/TestSuiteWriter.kt | 18 ++- 3 files changed, 123 insertions(+), 55 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 168c1711a1..645cd9a662 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -60,8 +60,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") - format.isJavaScript() && format.isPlaywright() -> lines.append(" -64 HttpWsTestCaseWriter.kt- ") + format.isPlaywright() -> lines.append("await request") + format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -83,7 +83,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handlePreCallSetup(call: HttpWsAction, lines: Lines, res: HttpWsCallResult) { /* This is needed when we need to execute some code before each HTTP call. - Cannot be in @Before/Fixture, as it must be done for each HTTP call , and not + Cannot be in @Before/Fixture, as it must be done for each HTTP call , and not just once for test */ @@ -104,8 +104,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") - format.isJavaScript() && !format.isPlaywright()-> lines.append("const $resVarName = ") - format.isJavaScript() && format.isPlaywright() -> lines.append(" -108 HttpWsTestCaseWriter.kt- ") + format.isJavaScript() && format.isPlaywright()-> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") } @@ -113,8 +112,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") - format.isJavaScript() && format.isPlaywright() -> lines.append(" -117 HttpWsTestCaseWriter.kt- ") + format.isPlaywright() -> lines.append("await request") + format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -189,8 +188,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" - format.isJavaScript() && !format.isPlaywright()-> ".set('Accept', " - format.isJavaScript() && format.isPlaywright() -> " -193 HttpWsTestCaseWriter.kt- " + format.isPlaywright() -> "'Accept': " + format.isJavaScript() -> ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " else -> throw IllegalArgumentException("Invalid format: $format") @@ -199,8 +198,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun closeAcceptHeader(openedHeader: String): String { var result = openedHeader - if (!config.outputFormat.isPython()) { - result += ")" + when { + format.isPlaywright() -> result += "," + !config.outputFormat.isPython() -> result += ")" } if (format.isCsharp()){ result = "$result;" @@ -215,7 +215,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { but that is not the case for the other libraries used for example in JS and C# */ return config.enableBasicAssertions && - (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.PYTHON_UNITTEST) + (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.JS_JEST_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) } protected fun handleHeaders(call: HttpWsAction, lines: Lines) { @@ -232,7 +232,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" format.isJavaScript() && !format.isPlaywright()-> "set" - format.isJavaScript() && format.isPlaywright() -> " -235 HttpWsTestCaseWriter.kt- " format.isPython() -> "headers = {}" else -> throw IllegalArgumentException("Not supported format: $format") } @@ -241,10 +240,18 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add(set) } + val bodyParam = call.parameters.find { p -> p is BodyParam } as BodyParam? + + if (format.isPlaywright() && bodyParam != null) { + lines.add("'Content-Type': '${bodyParam.contentType()}',") + } + //headers in specified auth info call.auth.headers.forEach { if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${it.value}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${it.value}', // ${call.auth.name}") } else { lines.add(".$set(\"${it.name}\", \"${it.value}\") // ${call.auth.name}") } @@ -262,6 +269,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escapedHeader = GeneUtils.applyEscapes(x, GeneUtils.EscapeMode.BODY, format) if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${escapedHeader}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${escapedHeader}',") } else { lines.add(".$set(\"${it.name}\", \"${escapedHeader}\")") @@ -275,14 +284,16 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val tokenHeader = elc.token!!.sendName if (format.isPython()) { lines.add("headers[\"$tokenHeader\"] = ${TokenWriter.authPayloadName(elc)} # ${call.auth.name}") + } else if (format.isPlaywright()) { + lines.add("'$tokenHeader': ${TokenWriter.authPayloadName(elc)}, // ${call.auth.name}") } else { lines.add(".$set(\"$tokenHeader\", ${TokenWriter.authPayloadName(elc)}) // ${call.auth.name}") } } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() && !format.isPlaywright()-> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") - format.isJavaScript() && format.isPlaywright() -> lines.add(" -285 HttpWsTestCaseWriter.kt- ") + format.isPlaywright() -> lines.add("'Cookie': ${CookieWriter.cookiesName(elc)},") + format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") // Python cookies are set alongside the headers and body when performing the request } } @@ -310,8 +321,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { - format.isJavaScript() && format.isPlaywright() -> { // This need a re-write! - val statusAssert = " - 314 - HttpWsTestCaseWriter.kt- " + format.isJavaScript() && format.isPlaywright() -> { + val statusAssert = "expect($responseVariableName.status()).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) }else{ @@ -319,7 +330,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine(statusAssert) } } - format.isJavaScript() && !format.isPlaywright()-> { // This might need a second look! + format.isJavaScript() -> { val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -452,13 +463,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $varName = System.currentTimeMillis()") } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") - } else if(format.isJavaScript() && !format.isPlaywright()) { + } else if(format.isPlaywright()) { + lines.addStatement("const $varName = performance.now()") + } else if(format.isJavaScript()) { lines.addStatement("$varName = performance.now()") - } else if(format.isJavaScript() && format.isPlaywright()) { - lines.addStatement(" -448 HttpWsTestCaseWriter.kt- ") - } lines.addEmpty(1) - + } return varName } @@ -473,10 +483,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $finalVarName = System.currentTimeMillis() - $varName") } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") - } else if(format.isJavaScript() && !format.isPlaywright()) { + } else if(format.isPlaywright()) { + lines.addStatement("const $finalVarName = performance.now() - $varName") + } else if(format.isJavaScript()) { lines.addStatement("$finalVarName = performance.now() - $varName") - } else if(format.isJavaScript() && format.isPlaywright()) { - lines.addStatement(" -469 HttpWsTestCaseWriter.kt- ") } lines.addEmpty(1) @@ -486,8 +496,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: SQL Injection vulnerability detected in this call. Expected response time (sqliInjectedSleepDurationMs) should be greater than ${config.sqliInjectedSleepDurationMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName > ${config.sqliInjectedSleepDurationMs})") - format.isJavaScript() && !format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") - format.isJavaScript() && format.isPlaywright() -> lines.addStatement(" -480 HttpWsTestCaseWriter.kt- ") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") + format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isPython() -> lines.addStatement("assert $finalVarName > ${config.sqliInjectedSleepDurationMs}") else -> {} } @@ -495,8 +505,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() && !format.isPlaywright()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() && format.isPlaywright() -> lines.addStatement(" -489 HttpWsTestCaseWriter.kt- ") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") else -> { } // This should have some information? } @@ -530,8 +540,20 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isJavaScript() -> { lines.indent(2) - if (format.isPlaywright()) { // This needs to be checked! - lines.append(" -524 HttpWsTestCaseWriter.kt- ") + if (format.isPlaywright()) { + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.append(", {") + lines.addEmpty() + lines.indented { + lines.add("headers: {") + lines.indented { + lines.add("${openAcceptHeader()}${getAcceptHeader(call, res)}") + handleHeaders(call, lines) + } + lines.add("},") + handleBody(call, lines, dtoVar) + } + lines.add("}") } else { //in SuperAgent, verb must be first handleVerbEndpoint(baseUrlOfSut, call, lines) @@ -566,8 +588,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() && !format.isPlaywright()-> "send" - format.isPython() && format.isPlaywright() -> " - 570 - HttpWsTestCaseWriter.kt" + format.isJavaScript() -> "send" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -589,8 +610,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") - format.isPlaywright() && format.isPlaywright() -> lines.add(" - 593 - HttpWsTestCaseWriter.kt- ") + format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -618,7 +638,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else { lines.add("body = $body") } - } + } + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: $body,") + } else -> lines.add(".$send($body)") } } else { @@ -629,6 +652,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"\"") } + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"${"""\"\""""}\",") + } else -> lines.add(".$send(\"${"""\"\""""}\")") } } @@ -649,6 +675,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$body\"") } + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"$body\",") + } else -> lines.add(".$send(\"$body\")") } } else if (bodyParam.isXml()) { @@ -661,6 +690,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$escapedXml\"") } + format.isPlaywright() -> { + lines.add("data: \"$escapedXml\",") + } else -> lines.add(".$send(\"$escapedXml\")") } } else { @@ -688,6 +720,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = ${bodyLines.first()}") } + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, false) + } format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) // Needs to be checked! fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, false) } @@ -713,6 +748,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("${bodyLines.last()}") } } + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, true) + } format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) // Needs to be checked fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, true) } @@ -747,6 +785,20 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(")") } + private fun writePlaywrightPayload(lines: Lines, bodyLines: List, isMultiLine: Boolean) { + lines.add("data: ${bodyLines.first()}") + if (isMultiLine) { + lines.append(" + ") + lines.indented { + (1 until bodyLines.lastIndex).forEach { i -> + lines.add("${bodyLines[i]} + ") + } + lines.add("${bodyLines.last()}") + } + } + lines.append(",") + } + /** * This is done mainly for RestAssured */ @@ -802,8 +854,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isJavaScript() && format.isPlaywright() -> "- 794 - HttpWsTestCaseWriter.kt - " - format.isJavaScript() && !format.isPlaywright()-> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" + format.isPlaywright() -> "- 794 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") } @@ -834,8 +886,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isJavaScript() && format.isPlaywright() -> "- 826 - HttpWsTestCaseWriter.kt - " - format.isJavaScript() && !format.isPlaywright()-> + format.isPlaywright() -> "- 826 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" format.isCsharp() -> "Assert.Contains(\"$bodyTypeSimplified\", $responseVariableName.Content.Headers.GetValues(\"Content-Type\").First());" @@ -899,15 +951,14 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - - if (format.isJavaScript() && format.isPlaywright()) { + if (format.isPlaywright()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... so, here we make it passes as long as a status was present */ lines.add(" - 886 - HttpWsTestCaseWriter.kt - ") - } else if(format.isJavaScript() && !format.isPlaywright()) { + } else if(format.isJavaScript()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... @@ -963,8 +1014,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isJavaScript() && format.isPlaywright() -> " - 955 - HttpWsTestCaseWriter.kt - " - format.isJavaScript() && !format.isPlaywright()-> "$resVarName.body.$jsonPath.toString()" + format.isPlaywright() -> " - 955 - HttpWsTestCaseWriter.kt - " + format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index de88733a81..676652f11b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -117,7 +117,7 @@ abstract class TestCaseWriter { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") format.isJavaScript() && !format.isPlaywright()-> lines.add("test(\"${test.name}\", async () => {") - format.isJavaScript() && format.isPlaywright() -> lines.add(" - 120 TestCaseWriter.kt - ") + format.isJavaScript() && format.isPlaywright() -> lines.add("test(\"${test.name}\", async ({ request }) => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } @@ -310,6 +310,12 @@ abstract class TestCaseWriter { testSuitePath: Path?, baseUrlOfSut: String ) { + val playwrightExpectException = format.isPlaywright() && shouldFailIfExceptionNotThrown(res) + val hasThrownVar = if (playwrightExpectException) "hasThrown_${counter++}" else "" + + if (playwrightExpectException) { + lines.add("let $hasThrownVar = false;") + } when { /* TODO do we need to handle differently in JS due to Promises? @@ -365,6 +371,12 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } + if (playwrightExpectException) { + lines.indented { + lines.add("$hasThrownVar = true;") + } + } + res.getErrorMessage()?.let { lines.indented { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") @@ -378,6 +390,9 @@ abstract class TestCaseWriter { } else { lines.add("}") } + if (playwrightExpectException) { + lines.add("expect($hasThrownVar).toBe(true);") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index d2cb8edf72..bbfd6595ce 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -512,7 +512,7 @@ class TestSuiteWriter { if (format.isJavaScript()) { if (format.isPlaywright()) { - lines.add(" - 515 - TestSuiteWriter.kt)") + lines.add("const { test, expect, request } = require('@playwright/test');") } else { lines.add("const superagent = require(\"superagent\");") } @@ -520,14 +520,13 @@ class TestSuiteWriter { val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) lines.add("const $jsImport = require(\"./$javascriptUtilsFilename\");") - lines.add(" - 523 - TestSuiteWriter.kt - What about this const EM thing?. Do we need it?") + if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") - lines.add(" - 526 - TestSuiteWriter.kt - What about controllerName. Do we need it?") } - if (config.testTimeout > 0) { + + if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") - lines.add(" - 529 - TestSuiteWriter.kt - Do we need the jest.setTimeout line above?") } } @@ -770,7 +769,7 @@ class TestSuiteWriter { lines.add("fun initClass()") } format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") - format.isJavaScript() && format.isPlaywright() -> lines.add("- 771 - TestSuiteWriter.kt") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeAll(async () =>") } lines.block { @@ -925,7 +924,8 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") // white box part? + format.isJavaScript() && !format.isPlaywright()-> lines.add("afterAll( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.afterAll(async () =>") } if (!format.isCsharp()) { @@ -978,7 +978,9 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") // white box part? + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeEach( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeEach(async () =>") + //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } From e780704aa95931bdbcd238f6fe67e7dec4519001 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Wed, 18 Mar 2026 11:11:41 +0100 Subject: [PATCH 04/24] Fix header handling, send body, and test initialization for Playwright --- .../output/service/HttpWsTestCaseWriter.kt | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 645cd9a662..4e18f61ec5 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -61,7 +61,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") format.isPlaywright() -> lines.append("await request") - format.isJavaScript() -> lines.append("await superagent") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -113,7 +113,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") format.isPlaywright() -> lines.append("await request") - format.isJavaScript() -> lines.append("await superagent") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -232,7 +232,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" format.isJavaScript() && !format.isPlaywright()-> "set" + format.isPlaywright() -> "" // headers are handled in a map in the options object format.isPython() -> "headers = {}" + format.isCsharp() -> "" // handled in handlePreCallSetup else -> throw IllegalArgumentException("Not supported format: $format") } @@ -292,8 +294,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") format.isPlaywright() -> lines.add("'Cookie': ${CookieWriter.cookiesName(elc)},") - format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") // Python cookies are set alongside the headers and body when performing the request } } @@ -329,8 +331,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) lines.addSingleCommentLine(statusAssert) } + lines.addEmpty() + lines.add("const body = await $responseVariableName.json();") + lines.add("$responseVariableName.body = body;") + lines.addEmpty() } - format.isJavaScript() -> { + + format.isJavaScript() && !format.isPlaywright() -> { val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -588,7 +595,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() -> "send" + format.isJavaScript() && !format.isPlaywright() -> "send" + format.isPlaywright() -> "data" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -610,7 +618,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isPlaywright() -> { + // handled in makeHttpCall through options object + } format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -818,6 +829,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } + format.isPlaywright() -> { + // assertions for Playwright are handled in handleResponseAfterTheCall, + // as they cannot be chained directly in the request call + } + else -> throw IllegalStateException("No assertion in calls for format: $format") } @@ -834,7 +850,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleResponseAssertions(lines: Lines, res: HttpWsCallResult, responseVariableName: String?) { - assert(responseVariableName != null || format.isJavaOrKotlin()) + assert(responseVariableName != null || format.isJavaOrKotlin() || format.isPlaywright()) /* there are 2 cases: @@ -854,7 +870,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isPlaywright() -> "- 794 - HttpWsTestCaseWriter.kt - " + format.isPlaywright() -> "expect($responseVariableName.headers()[\"allow\"]?.includes(\"$allow\")).toBe(true)" format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") @@ -886,7 +902,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isPlaywright() -> "- 826 - HttpWsTestCaseWriter.kt - " + format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]?.includes(\"$bodyTypeSimplified\")).toBe(true)" format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" @@ -952,12 +968,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { if (format.isPlaywright()) { - /* - This is to deal with very weird behavior in SuperAgent that crashes the tests - for status codes different from 2xx... - so, here we make it passes as long as a status was present - */ - lines.add(" - 886 - HttpWsTestCaseWriter.kt - ") + // No extra processing for Playwright } else if(format.isJavaScript()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests @@ -1014,7 +1025,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isPlaywright() -> " - 955 - HttpWsTestCaseWriter.kt - " + format.isPlaywright() -> " (await $resVarName.json())${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)}.toString()" format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") From a3452755b8a8cc9d7ab0721e5599fbfdb3f19f2f Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 19 Mar 2026 12:20:42 +0100 Subject: [PATCH 05/24] Fix duplicate playwright entry --- .../org/evomaster/core/output/OutputFormat.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index 350169af69..21d4809802 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -20,8 +20,7 @@ enum class OutputFormat { JS_JEST, JS_JEST_PLAYWRIGHT, // Testing new playwright imp //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed - PYTHON_UNITTEST, - JS_JEST_PLAYWRIGHT + PYTHON_UNITTEST ; fun isJava() = this.name.startsWith("java_", true) @@ -30,21 +29,22 @@ enum class OutputFormat { fun isJavaScript() = this.name.startsWith("js_", true) + /** + * Return true if the output format is Playwright. + * Playwright is currently only supported for JavaScript (or TypeScript). + */ + fun isPlaywright() = this.name.endsWith("_playwright", true) // Testing new playwright imp + fun isJavaOrKotlin() = isJava() || isKotlin() fun isJUnit5() = this.name.endsWith("junit_5", true) fun isJUnit4() = this.name.endsWith("junit_4", true) - fun isPlaywright() = this.name.endsWith("_playwright", true) // Testing new playwright imp - fun isJUnit() = this.name.contains("_junit_", true) @Deprecated("No longer supported") fun isCsharp() = this.name.startsWith("csharp",ignoreCase = true) fun isPython() = this.name.startsWith("python_", true) - - fun isPlaywright() = this.name.startsWith("js_jest_playwright", true) - } From c5842f5b0221a3b228be1a91c466735edcf86049 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 19 Mar 2026 15:42:47 +0100 Subject: [PATCH 06/24] Add support for Playwright --- .../core/output/service/ApiTestCaseWriter.kt | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 310c3d89d1..94672a6e4d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -317,9 +317,14 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { + val field = when { + format.isPlaywright() -> "await $responseVariableName.json()" + format.isJavaScript() -> "$responseVariableName.body" + else -> "" + } val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isJavaScript() -> "expect($responseVariableName.body$fieldPath).toBe(null);" + format.isJavaScript() -> "expect(($field)$fieldPath).toBe(null);" format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") @@ -369,10 +374,15 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { + val field = when { + format.isPlaywright() -> "await $responseVariableName.json()" + format.isJavaScript() -> "$responseVariableName.body" + else -> "" + } val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" }else { // javascript - "expect($responseVariableName.body$fieldPath).toBe($toPrint);" + "expect(($field)$fieldPath).toBe($toPrint);" } if (flakyValue == null || flakyValue == value){ @@ -481,6 +491,9 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { + if (format.isPlaywright()) { + return "expect(await $responseVariableName.text()).toBe(\"\");" + } /* This is super ugly... but there is no clean solution for this in Jest nor SuperAgent... :( @@ -516,8 +529,14 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val path = if (fieldPath.isEmpty()) "" else "$fieldPath." ".body(\"${path}size()\", equalTo($expectedSize))" } - format.isJavaScript() -> - "expect($responseVariableName.body$fieldPath.length).toBe($expectedSize);" + format.isJavaScript() -> { + val field = when { + format.isPlaywright() -> "await $responseVariableName.json()" + format.isJavaScript() -> "$responseVariableName.body" + else -> "" + } + "expect(($field)$fieldPath.length).toBe($expectedSize);" + } format.isCsharp() -> "Assert.True($responseVariableName$fieldPath.Count == $expectedSize);" format.isPython() -> @@ -537,7 +556,8 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { - return "expect($responseVariableName.text).toBe(\"$content\");" + val contentCall = if (format.isPlaywright()) "await $responseVariableName.text()" else "$responseVariableName.text" + return "expect($contentCall).toBe(\"$content\");" } if (format.isCsharp()) { From cbf837bc613195761c745e7e583aba9ee26cf720 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 19 Mar 2026 16:00:15 +0100 Subject: [PATCH 07/24] Add support for Playwright --- .../core/output/service/RestTestCaseWriter.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 0f5b4a5869..1428b7899e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -205,7 +205,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJavaScript() -> lines.add("const $name = ") + format.isJavaScript() || format.isPlaywright() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -417,7 +417,11 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { - lines.add("const $location = $resVarName.header['location'];") + if (format.isPlaywright()) { + lines.add("const $location = $resVarName.headers()['location'];") + } else { + lines.add("const $location = $resVarName.header['location'];") + } val validCheck = "${TestSuiteWriter.jsImport}.isValidURIorEmpty($location)" lines.add("expect($validCheck).toBe(true);") } @@ -449,7 +453,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJavaScript() -> lines.add("const ") + format.isJavaScript() || format.isPlaywright() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ From 2ac346d97e5411aff14d9b7e611780d30c304752 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 19 Mar 2026 16:45:34 +0100 Subject: [PATCH 08/24] Add support for Playwright --- .../evomaster/core/output/auth/CookieWriter.kt | 16 ++++++++++++++-- .../evomaster/core/output/auth/TokenWriter.kt | 6 ++++++ .../core/output/service/TestSuiteWriter.kt | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index b3791fe751..8574e06f01 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -76,12 +76,17 @@ object CookieWriter { format.isPython() -> lines.append(".cookies") } - if(format.isJavaScript()){ + if(format.isJavaScript() && !format.isPlaywright()){ lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") lines.appendSemicolon() } + if (format.isPlaywright()) { + lines.add(".then(async (res) => (await res.headerValue('set-cookie'))?.split(';')[0])") + lines.appendSemicolon() + } + if (format.isPython()) { lines.add("${cookiesName(k)} = requests.utils.dict_from_cookiejar($targetCookieVariable)") } @@ -122,6 +127,10 @@ object CookieWriter { if(contentType != null) { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${contentType.defaultValue}\")") + format.isPlaywright() -> { + // handled in request options 'data' or similar if needed, + // but usually Playwright sets it automatically if passed as object + } format.isJavaScript() -> lines.add(".set(\"content-type\", \"${contentType.defaultValue}\")") format.isPython() -> { lines.add("headers[\"content-type\"] = \"${contentType.defaultValue}\"") @@ -150,6 +159,9 @@ object CookieWriter { for(header in k.headers) { when { format.isJavaOrKotlin() -> lines.add(".header(\"${header.name}\", \"${header.value}\")") + format.isPlaywright() -> { + // Playwright headers for login are not yet supported in this simplified call + } format.isJavaScript() -> lines.add(".set(\"${header.name}\", \"${header.value}\")") format.isPython() -> { lines.add("headers[\"${header.name}\"] = \"${header.value}\"") @@ -157,7 +169,7 @@ object CookieWriter { } } - if (format.isJavaScript()){ + if (format.isJavaScript() && !format.isPlaywright()){ // disable redirections lines.add(".redirects(0)") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index ec2df92b7e..c3d1adba7e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -57,6 +57,12 @@ object TokenWriter { when{ format.isJavaOrKotlin() -> lines.append("given()") + format.isPlaywright() -> { + lines.append("\"\"") + lines.appendSemicolon() + lines.addEmpty() + lines.append("await request") + } format.isJavaScript() -> { lines.append("\"\"") lines.appendSemicolon() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index bbfd6595ce..70fac281aa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -769,7 +769,7 @@ class TestSuiteWriter { lines.add("fun initClass()") } format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") - format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeAll(async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeAll(async ({ request }) =>") } lines.block { From 1f0563bff165c458e68144255cc121166a52f4a4 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Fri, 20 Mar 2026 14:08:18 +0100 Subject: [PATCH 09/24] Rewrite of existing support for Playwright, add Playwright support for cookie and token handling, and for GraphQl. --- .../core/output/auth/CookieWriter.kt | 43 +++++++++++++++++-- .../evomaster/core/output/auth/TokenWriter.kt | 15 ++++--- .../core/output/service/ApiTestCaseWriter.kt | 8 +++- .../output/service/GraphQLTestCaseWriter.kt | 11 ++++- .../output/service/HttpWsTestCaseWriter.kt | 12 ++++-- .../core/output/service/RestTestCaseWriter.kt | 2 + .../core/output/service/TestSuiteWriter.kt | 20 +++++---- 7 files changed, 87 insertions(+), 24 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index 8574e06f01..844bda8cf0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -115,7 +115,7 @@ object CookieWriter { targetVariable: String ) { - if(format.isJavaScript()) { + if(format.isJavaScript() && !format.isPlaywright()) { callEndpoint(lines, k, format, baseUrlOfSut) } @@ -160,7 +160,7 @@ object CookieWriter { when { format.isJavaOrKotlin() -> lines.add(".header(\"${header.name}\", \"${header.value}\")") format.isPlaywright() -> { - // Playwright headers for login are not yet supported in this simplified call + // handled in callEndpoint for Playwright } format.isJavaScript() -> lines.add(".set(\"${header.name}\", \"${header.value}\")") format.isPython() -> { @@ -182,6 +182,10 @@ object CookieWriter { callEndpoint(lines, k, format, baseUrlOfSut) } + if (format.isPlaywright()) { + callEndpoint(lines, k, format, baseUrlOfSut) + } + if (format.isPython()) { lines.add("$targetVariable = requests \\") lines.indent(2) @@ -201,7 +205,13 @@ object CookieWriter { baseUrlOfSut: String ) { val verb = k.verb.name.lowercase() - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("request.$verb(") + } else { + lines.add(".$verb(") + } + if (k.externalEndpointURL != null) { lines.append("\"${k.externalEndpointURL}\"") } else { @@ -212,6 +222,33 @@ object CookieWriter { } lines.append("${k.endpoint}\"") } + + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + if (k.headers.isNotEmpty() || k.contentType != null) { + lines.add("headers: {") + lines.indented { + if (k.contentType != null) { + lines.add("'Content-Type': '${k.contentType.defaultValue}',") + } + for (header in k.headers) { + lines.add("'${header.name}': '${header.value}',") + } + } + lines.add("},") + } + if (k.payload != null) { + if (k.contentType == ContentType.JSON) { + lines.add("data: ${k.payload},") + } else { + lines.add("data: '${k.payload}',") + } + } + } + lines.add("}") + } + if (!format.isPython()) { lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index c3d1adba7e..5c58b2c74d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -58,10 +58,7 @@ object TokenWriter { when{ format.isJavaOrKotlin() -> lines.append("given()") format.isPlaywright() -> { - lines.append("\"\"") - lines.appendSemicolon() - lines.addEmpty() - lines.append("await request") + lines.append("await ") } format.isJavaScript() -> { lines.append("\"\"") @@ -91,7 +88,10 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;},") + lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { @@ -106,7 +106,10 @@ object TokenWriter { } TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = await res.headerValue(\"$header\");},") + lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 94672a6e4d..ace9bc97e6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -251,6 +251,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { //TODO would not this fail on recursive/nested calls??? format.isJava() -> ".body(\"${k}isEmpty()\", is(true))" format.isKotlin() -> ".body(\"${k}isEmpty()\", `is`(true))" //'is' is a keyword in Kotlin + format.isPlaywright() -> "expect(Object.keys(await $responseVariableName.json()${k}).length).toBe(0);" format.isJavaScript() -> "expect(Object.keys($responseVariableName.body${k}).length).toBe(0);" format.isCsharp() -> "Assert.True($responseVariableName${k}.ToString() == \"{}\");" format.isPython() -> "assert len($responseVariableName.json()${k}) == 0" @@ -324,6 +325,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" + format.isPlaywright() -> "expect(($field)${if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else ".$fieldPath"}).toBe(null);" format.isJavaScript() -> "expect(($field)$fieldPath).toBe(null);" format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" @@ -535,7 +537,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { format.isJavaScript() -> "$responseVariableName.body" else -> "" } - "expect(($field)$fieldPath.length).toBe($expectedSize);" + if (format.isPlaywright()) { + "expect(await $responseVariableName.json()).toHaveLength($expectedSize);" + } else { + "expect(($field)$fieldPath.length).toBe($expectedSize);" + } } format.isCsharp() -> "Assert.True($responseVariableName$fieldPath.Count == $expectedSize);" diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 30962d25e7..59ce50244f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -52,6 +52,9 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"application/json\")") + format.isPlaywright() -> { + // Handled in callEndpoint for Playwright + } format.isJavaScript() -> lines.add(".set('Content-Type','application/json')") format.isPython() -> lines.add("headers[\"content-type\"] = \"application/json\"") // format.isCsharp() -> lines.add("Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(\"application/json\"));") @@ -95,10 +98,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } override fun handleVerbEndpoint(baseUrlOfSut: String, _call: HttpWsAction, lines: Lines) { - // TODO maybe in future might want to have GET for QUERY types val verb = "post" - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("request.$verb(") + } else { + lines.add(".$verb(") + } if(config.blackBox){ /* diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 4e18f61ec5..5cda6de2b0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -245,7 +245,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val bodyParam = call.parameters.find { p -> p is BodyParam } as BodyParam? if (format.isPlaywright() && bodyParam != null) { - lines.add("'Content-Type': '${bodyParam.contentType()}',") + val contentType = bodyParam.contentType() + if (contentType != null) { + lines.add("'Content-Type': '$contentType',") + } } //headers in specified auth info @@ -870,7 +873,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isPlaywright() -> "expect($responseVariableName.headers()[\"allow\"]?.includes(\"$allow\")).toBe(true)" + format.isPlaywright() -> "expect(await $responseVariableName.headerValue(\"allow\")).toContain(\"$allow\")" format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") @@ -902,7 +905,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]?.includes(\"$bodyTypeSimplified\")).toBe(true)" + format.isPlaywright() -> "expect(await $responseVariableName.headerValue(\"content-type\")).toContain(\"$bodyTypeSimplified\")" format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" @@ -1022,10 +1025,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } val jsonPath = JsonUtils.fromPointerToPath(jsonPointer) + val dictAccess = JsonUtils.fromPointerToDictionaryAccess(jsonPointer) return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isPlaywright() -> " (await $resVarName.json())${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)}.toString()" + format.isPlaywright() -> " ((await $resVarName.json())$dictAccess)?.toString()" format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 1428b7899e..3a9b863130 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -223,6 +223,8 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { if (format.isCsharp()) { lines.append(".${StringUtils.capitalization(verb)}Async(") + } else if (format.isPlaywright()) { + lines.add("request.$verb(") } else { if (verb == "trace" && format.isJavaOrKotlin()) { //currently, RestAssured does not have a trace() method diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 70fac281aa..e6a4ad73e2 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -711,13 +711,17 @@ class TestSuiteWriter { lines.add("private lateinit var $driver : RemoteWebDriver") } - } else if (config.outputFormat.isJavaScript()) { // Leave as is for playwright? + } else if (config.outputFormat.isJavaScript()) { if (!config.blackBox || config.bbExperiments) { lines.add("const $controller = new $controllerName();") lines.add("let $baseUrlOfSut;") } else { - lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + if (config.outputFormat.isPlaywright()) { + lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } else { + lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } } } else if (config.outputFormat.isCsharp()) { lines.add("private static readonly HttpClient Client = new HttpClient ();") @@ -775,7 +779,7 @@ class TestSuiteWriter { lines.block { if (!config.blackBox) { when { - config.outputFormat.isJavaScript() -> { // Add something for playwright white box testing! + config.outputFormat.isJavaScript() -> { addStatement("await $controller.setupForGeneratedTest()", lines) addStatement("$baseUrlOfSut = await $controller.startSut()", lines) } @@ -801,7 +805,7 @@ class TestSuiteWriter { when { format.isJavaOrKotlin() -> addStatement("assertNotNull(baseUrlOfSut)", lines) - format.isJavaScript() -> addStatement("expect(baseUrlOfSut).toBeTruthy()", lines) // Add something for playwright white box testing + format.isJavaScript() -> addStatement("expect(baseUrlOfSut).toBeTruthy()", lines) } } @@ -931,7 +935,7 @@ class TestSuiteWriter { if (!format.isCsharp()) { lines.block { when { - format.isJavaScript() -> { // white box part? + format.isJavaScript() -> { addStatement("await $controller.stopSut()", lines) } else -> { @@ -956,7 +960,7 @@ class TestSuiteWriter { } } - if (format.isJavaScript()) { // white box part? + if (format.isJavaScript()) { lines.append(");") } } @@ -988,7 +992,7 @@ class TestSuiteWriter { lines.block { - if (format.isJavaScript()) { // white box part? + if (format.isJavaScript()) { //TODO add resetDatabase addStatement("await $controller.resetStateOfSUT()", lines) } else if (format.isJavaOrKotlin()) { @@ -1027,7 +1031,7 @@ class TestSuiteWriter { } } - if (format.isJavaScript()) { // white box part? + if (format.isJavaScript()) { lines.append(");") } } From 4969fb6fdf3479136089431cf3cfe1ae33c390da Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Tue, 24 Mar 2026 13:33:46 +0100 Subject: [PATCH 10/24] Add lombok --- core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml index b41f9d7211..84a7544350 100644 --- a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml @@ -35,6 +35,7 @@ org.projectlombok lombok + provided org.apache.commons @@ -133,6 +134,16 @@ org.apache.maven.plugins maven-compiler-plugin + 3.14.1 + + + + org.projectlombok + lombok + 1.18.30 + + + From 8234fc3b64c76ca1d07751f75440586d74c6ad3e Mon Sep 17 00:00:00 2001 From: FredRaw Date: Mon, 13 Apr 2026 09:32:34 +0200 Subject: [PATCH 11/24] added possible fixes for playwright code --- .../org/evomaster/core/output/service/ApiTestCaseWriter.kt | 6 +++--- .../evomaster/core/output/service/HttpWsTestCaseWriter.kt | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index c481d08093..ac4cafe8e5 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -328,7 +328,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" format.isPlaywright() -> "expect(($field)${if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else ".$fieldPath"}).toBe(null);" - format.isJavaScript() -> "expect(($field)$fieldPath).toBe(null);" + format.isJavaScript() -> "expect($field$fieldPath).toBe(null);" // ($field$)fieldPath format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") @@ -386,7 +386,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" }else { // javascript - "expect(($field)$fieldPath).toBe($toPrint);" + "expect($field$fieldPath).toBe($toPrint);" // ($field$)fieldPath } if (flakyValue == null || flakyValue == value){ @@ -578,7 +578,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (format.isPlaywright()) { "expect(await $responseVariableName.json()).toHaveLength($expectedSize);" } else { - "expect(($field)$fieldPath.length).toBe($expectedSize);" + "expect($field$fieldPath.length).toBe($expectedSize);" // ($field$)fieldPath } } format.isCsharp() -> diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 15efaf4c67..daf7675ea6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -104,7 +104,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") - format.isJavaScript() && format.isPlaywright()-> lines.append("const $resVarName = ") + format.isPlaywright() -> lines.append("const $resVarName = ") + format.isJavaScript() -> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") } @@ -113,7 +114,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") format.isPlaywright() -> lines.append("await request") - format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") + format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -333,7 +334,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } lines.addEmpty() lines.add("const body = await $responseVariableName.json();") - lines.add("$responseVariableName.body = body;") + lines.add(" = body;") lines.addEmpty() } From 158e195ed1f5bccb93f5f5864c483a3a3d5c37ee Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 23 Apr 2026 10:46:28 +0200 Subject: [PATCH 12/24] Add Playwright library to core-tests/e2e-tests/spring/spring-rest-bb/javascript and hopefully fix playwright syntax error. --- .../javascript/package-lock.json | 94 +++++++++++++++++++ .../spring-rest-bb/javascript/package.json | 1 + .../output/service/HttpWsTestCaseWriter.kt | 8 +- .../core/output/service/RestTestCaseWriter.kt | 22 +++-- .../core/output/service/TestSuiteWriter.kt | 2 +- docs/options.md | 2 +- 6 files changed, 112 insertions(+), 17 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json index c85a0aca99..4333fffe1d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json @@ -7,6 +7,7 @@ "name": "evomaster-client-js-e2e-tests", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", @@ -933,6 +934,21 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -3248,6 +3264,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -4567,6 +4627,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "requires": { + "playwright": "1.59.1" + } + }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -6292,6 +6361,31 @@ "find-up": "^4.0.0" } }, + "playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.59.1" + }, + "dependencies": { + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + } + } + }, + "playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true + }, "pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index 5c69b06726..aa42e14ae8 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -6,6 +6,7 @@ "author": "EvoMaster Team", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index daf7675ea6..eda55ac430 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -333,9 +333,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine(statusAssert) } lines.addEmpty() - lines.add("const body = await $responseVariableName.json();") - lines.add(" = body;") - lines.addEmpty() } format.isJavaScript() && !format.isPlaywright() -> { @@ -551,18 +548,19 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.indent(2) if (format.isPlaywright()) { handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.replaceInCurrent(Regex("\\)$"), "") lines.append(", {") lines.addEmpty() lines.indented { lines.add("headers: {") lines.indented { - lines.add("${openAcceptHeader()}${getAcceptHeader(call, res)}") + lines.add(getAcceptHeader(call, res)) handleHeaders(call, lines) } lines.add("},") handleBody(call, lines, dtoVar) } - lines.add("}") + lines.add("})") } else { //in SuperAgent, verb must be first handleVerbEndpoint(baseUrlOfSut, call, lines) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 3a9b863130..3f11bb8fc9 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -221,16 +221,18 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val call = _call as RestCallAction val verb = call.verb.name.lowercase() - if (format.isCsharp()) { - lines.append(".${StringUtils.capitalization(verb)}Async(") - } else if (format.isPlaywright()) { - lines.add("request.$verb(") - } else { - if (verb == "trace" && format.isJavaOrKotlin()) { - //currently, RestAssured does not have a trace() method - lines.add(".request(io.restassured.http.Method.TRACE, ") - } else { - lines.add(".$verb(") + when { + format.isPlaywright() -> { + lines.add("request.$verb(") + } + format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") + else -> { + if (verb == "trace" && format.isJavaOrKotlin()) { + //currently, RestAssured does not have a trace() method + lines.add(".request(io.restassured.http.Method.TRACE, ") + } else { + lines.add(".$verb(") + } } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 0efbbbb1d7..585b3378ae 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -787,7 +787,7 @@ class TestSuiteWriter { lines.add("fun initClass()") } format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") - format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeAll(async ({ request }) =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("(async ({ request }) =>") } lines.block { diff --git a/docs/options.md b/docs/options.md index f9a13d5c5a..4529b034ef 100644 --- a/docs/options.md +++ b/docs/options.md @@ -39,7 +39,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_JEST_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`blackBox`| __Boolean__. Use EvoMaster in black-box mode. This does not require an EvoMaster Driver up and running. However, you will need to provide further option to specify how to connect to the SUT. *Default value*: `false`.| |`bbSwaggerUrl`| __String__. When in black-box mode for REST APIs, specify the URL of where the OpenAPI/Swagger schema can be downloaded from. If the schema is on the local machine, you can use a URL starting with 'file://'. If the given URL is neither starting with 'file' nor 'http', then it will be treated as a local file path. *Default value*: `""`.| From e666fb6db9f536041b9cc93813e69f639521e985 Mon Sep 17 00:00:00 2001 From: FredRaw Date: Fri, 24 Apr 2026 09:38:03 +0200 Subject: [PATCH 13/24] updated release.yml to include changes from master branch --- .github/workflows/release.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f4b304a0d..d4be9c806e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,7 @@ env: evomaster-version: 5.1.1 jdk-jar: 17 jdk-jpackage: 21 + java-distribution: temurin retention-days: 1 ## Doesn't work, ie, use env in env # installer-windows: evomaster-${evomaster-version}.msi @@ -24,9 +25,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup JDK ${{env.jdk-jar}} - uses: actions/setup-java@v1 + uses: actions/setup-java@v5 with: java-version: ${{env.jdk-jar}} + distribution: ${{env.java-distribution}} - name: Cache Maven packages uses: actions/cache@v3 with: @@ -74,9 +76,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup JDK ${{env.jdk-jpackage}} - uses: actions/setup-java@v1 + uses: actions/setup-java@v5 with: java-version: ${{env.jdk-jpackage}} + distribution: ${{env.java-distribution}} - name: Download fat jar uses: actions/download-artifact@v4 with: @@ -99,9 +102,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup JDK ${{env.jdk-jpackage}} - uses: actions/setup-java@v1 + uses: actions/setup-java@v5 with: java-version: ${{env.jdk-jpackage}} + distribution: ${{env.java-distribution}} - name: Download fat jar uses: actions/download-artifact@v4 with: @@ -124,9 +128,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup JDK ${{env.jdk-jpackage}} - uses: actions/setup-java@v1 + uses: actions/setup-java@v5 with: java-version: ${{env.jdk-jpackage}} + distribution: ${{env.java-distribution}} - name: Download fat jar uses: actions/download-artifact@v4 with: From b8dc0a39bb03c18795bfd7a58593de69bee7299f Mon Sep 17 00:00:00 2001 From: FredRaw Date: Mon, 27 Apr 2026 11:15:11 +0200 Subject: [PATCH 14/24] re-introduced if(nody.isNotBlank()) test to avoid failure of testInActiveBodyParamlnTest() --- .../output/service/HttpWsTestCaseWriter.kt | 61 +++++++++++-------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index eda55ac430..6cc6a2acf6 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -638,35 +638,46 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else if (bodyParam.isTextPlain()) { val body = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.TEXT, targetFormat = format) - if (body != "\"\"") { - when { - format.isCsharp() -> { - lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")") + // handle body only if it is not black + if (body.isNotBlank()) { + if (body != "\"\"") { + when { + format.isCsharp() -> { + lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")") + } + + format.isPython() -> { + if (body.trim().isNullOrBlank()) { + lines.add("body = \"\"") + } else { + lines.add("body = $body") + } + } + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: $body,") + } + + else -> { + lines.add(".$send($body)") + } } - format.isPython() -> { - if (body.trim().isNullOrBlank()) { + } else { + when { + format.isCsharp() -> { + lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")") + } + + format.isPython() -> { lines.add("body = \"\"") - } else { - lines.add("body = $body") } + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"${"""\"\""""}\",") + } + + else -> lines.add(".$send(\"${"""\"\""""}\")") } - format.isJavaScript() && format.isPlaywright() -> { - lines.add("data: $body,") - } - else -> lines.add(".$send($body)") - } - } else { - when { - format.isCsharp() -> { - lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")") - } - format.isPython() -> { - lines.add("body = \"\"") - } - format.isJavaScript() && format.isPlaywright() -> { - lines.add("data: \"${"""\"\""""}\",") - } - else -> lines.add(".$send(\"${"""\"\""""}\")") } } From 188b96aed618704b04ee2bb92dc9ae0e7ea0e8d3 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 7 May 2026 15:28:02 +0200 Subject: [PATCH 15/24] BlackBoxUtils.kt: Add base location and relative path for Playwright and add function for running PLaywright tests. HttpsWsTestCaseWriter.kt: Remove await request handling for Playwright and hopefully fix header handling for response. RestTestCaseWriter.kt: Add await for playwright SpringTestBase.kt: Add playwright support Add playwright.config --- .../evomaster/e2etests/utils/BlackBoxUtils.kt | 20 +++++++++++++++++-- .../spring/graphql/bb/SpringTestBase.kt | 2 ++ .../javascript/playwright.config.js | 6 ++++++ .../e2etests/spring/rest/bb/SpringTestBase.kt | 5 +++-- .../output/service/HttpWsTestCaseWriter.kt | 6 +++--- .../core/output/service/RestTestCaseWriter.kt | 2 +- 6 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt index a3577df23b..a6089e694b 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt @@ -14,12 +14,15 @@ object BlackBoxUtils { private const val GENERATED_FOLDER_NAME = "generated" const val baseLocationForJavaScript = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME" + const val baseLocationForPlaywright = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME/playwright" const val baseLocationForPython = "$PY_BASE_PATH/$GENERATED_FOLDER_NAME" const val baseLocationForJava = "$MAVEN_BASE_PATH/src/test/java" const val baseLocationForKotlin = "$MAVEN_BASE_PATH/src/test/kotlin" fun relativePath(folderName: String) = "$GENERATED_FOLDER_NAME/$folderName" + fun relativePathPlaywright(folderName: String) = "$GENERATED_FOLDER_NAME/playwright/$folderName" + fun checkCoveredTargets(targetLabels: Collection) { targetLabels.forEach { assertTrue(CoveredTargets.isCovered(it), "Target '$it' is not covered") @@ -36,7 +39,6 @@ object BlackBoxUtils { private fun mvn() = if (isWindows()) "mvn.cmd" else "mvn" - private fun runNpmInstall() { val command = listOf(npm(), "ci") @@ -120,6 +122,20 @@ object BlackBoxUtils { runTestsCommand(command, JS_BASE_PATH, "NPM") } + fun runPlaywrightTests(folderRelativePath: String) { + runNpmInstall() + + val path = if(folderRelativePath.endsWith("/")){ + folderRelativePath + } else { + "$folderRelativePath/" + } + + val npx = if (isWindows()) "npx.cmd" else "npx" + val command = listOf(npx, "playwright", "test", path) + runTestsCommand(command, JS_BASE_PATH, "Playwright") + } + fun runPythonTests(folderRelativePath: String) { installPythonRequirements() @@ -151,4 +167,4 @@ object BlackBoxUtils { fun getOutputFilePrefixKotlin(outputFolderName: String) = "com.kotlin.$outputFolderName.EM" -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index 30291759dc..ee78d106d8 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -90,6 +90,7 @@ abstract class SpringTestBase : GraphQLTestBase() { lambda: Consumer> ){ val baseLocation = when { + outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython else -> throw IllegalArgumentException("Not supported output type $outputFormat") @@ -100,6 +101,7 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js new file mode 100644 index 0000000000..ce35c56d60 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js @@ -0,0 +1,6 @@ +const { defineConfig } = require('@playwright/test'); + +module.exports = defineConfig({ + testDir: './generated', + testMatch: /.*[tT]est\.js/, +}); diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index 3586ed69c1..24cb85b04c 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -116,7 +116,6 @@ abstract class SpringTestBase : RestTestBase() { BlackBoxUtils.checkCoveredTargets(targetLabels) } - fun runBlackBoxEM( outputFormat: OutputFormat, outputFolderName: String, @@ -125,6 +124,7 @@ abstract class SpringTestBase : RestTestBase() { lambda: Consumer> ){ val baseLocation = when { + outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython outputFormat.isJava() -> BlackBoxUtils.baseLocationForJava @@ -137,6 +137,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) @@ -189,4 +190,4 @@ abstract class SpringTestBase : RestTestBase() { } } -} +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 6cc6a2acf6..ec1dc91ab4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -113,7 +113,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") - format.isPlaywright() -> lines.append("await request") + format.isPlaywright() -> {} // already handled in handleVerbEndpoint in RestTestCaseWriter format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") @@ -881,7 +881,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isPlaywright() -> "expect(await $responseVariableName.headerValue(\"allow\")).toContain(\"$allow\")" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"allow\"]).toContain(\"$allow\")" format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") @@ -913,7 +913,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isPlaywright() -> "expect(await $responseVariableName.headerValue(\"content-type\")).toContain(\"$bodyTypeSimplified\")" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 1a592aa985..3bd63c272c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -219,7 +219,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isPlaywright() -> { - lines.add("request.$verb(") + lines.add("await request.$verb(") } format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") else -> { From ce127974d0901d683387c18540fc52e06971ac8d Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Tue, 12 May 2026 14:30:40 +0200 Subject: [PATCH 16/24] All changes only made to playwright code. Will hopefully fix BB test errors for generating playwright output: ApiTestCaseWriter.kt: update json response size assertions, CookieWriter.kt: update assignment of cookie value, httpWsTestCaseWriter.kt: update header and add verb handling, RestTestCaseWriter.kt: update verb handling incl. fetch method, TokenWriter.kt: update token handling --- .../core/output/auth/CookieWriter.kt | 17 +++++++++++------ .../evomaster/core/output/auth/TokenWriter.kt | 6 ++++-- .../core/output/service/ApiTestCaseWriter.kt | 19 ++++++++++--------- .../output/service/HttpWsTestCaseWriter.kt | 5 +++++ .../core/output/service/RestTestCaseWriter.kt | 7 ++++++- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index 844bda8cf0..378d6790a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -50,12 +50,15 @@ object CookieWriter { when { format.isJava() -> lines.add("final Map ${cookiesName(k)} = ") format.isKotlin() -> lines.add("val ${cookiesName(k)} : Map = ") + format.isPlaywright() -> lines.add("let ${cookiesName(k)};") format.isJavaScript() -> lines.add("const ${cookiesName(k)} = ") } if (!format.isPython()) { // TODO: should we use DTO for cookie related requests? - testCaseWriter.startRequest(lines) + if (!format.isPlaywright()) { + testCaseWriter.startRequest(lines) + } lines.indent() } @@ -83,7 +86,7 @@ object CookieWriter { } if (format.isPlaywright()) { - lines.add(".then(async (res) => (await res.headerValue('set-cookie'))?.split(';')[0])") + lines.add(".then(async (res) => { ${cookiesName(k)} = res.headers()['set-cookie']?.split('\\n')?.find(c => c.trim().length > 0)?.split(';')[0]; })") lines.appendSemicolon() } @@ -115,6 +118,11 @@ object CookieWriter { targetVariable: String ) { + if (format.isPlaywright()) { + callEndpoint(lines, k, format, baseUrlOfSut) + return + } + if(format.isJavaScript() && !format.isPlaywright()) { callEndpoint(lines, k, format, baseUrlOfSut) } @@ -182,9 +190,6 @@ object CookieWriter { callEndpoint(lines, k, format, baseUrlOfSut) } - if (format.isPlaywright()) { - callEndpoint(lines, k, format, baseUrlOfSut) - } if (format.isPython()) { lines.add("$targetVariable = requests \\") @@ -207,7 +212,7 @@ object CookieWriter { val verb = k.verb.name.lowercase() if (format.isPlaywright()) { - lines.add("request.$verb(") + lines.add("await request.$verb(") } else { lines.add(".$verb(") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index 5c58b2c74d..81f92ef44f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -52,13 +52,15 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") + format.isPlaywright() -> { + lines.add("let ${tokenName(k)};") + } format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") format.isPlaywright() -> { - lines.append("await ") } format.isJavaScript() -> { lines.append("\"\"") @@ -107,7 +109,7 @@ object TokenWriter { TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector if (format.isPlaywright()) { - lines.add(".then(async res => {${tokenName(k)} = await res.headerValue(\"$header\");},") + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];},") lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 2eca7989b0..61de4bd224 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -345,13 +345,14 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (value == null) { val field = when { - format.isPlaywright() -> "await $responseVariableName.json()" + format.isPlaywright() -> "(await $responseVariableName.json())" format.isJavaScript() -> "$responseVariableName.body" else -> "" } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isPlaywright() -> "expect(($field)${if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else ".$fieldPath"}).toBe(null);" + format.isPlaywright() -> "expect(($field)$fieldWithDot).toBe(null);" format.isJavaScript() -> "expect($field$fieldPath).toBe(null);" // ($field$)fieldPath format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" @@ -403,12 +404,15 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { val field = when { - format.isPlaywright() -> "await $responseVariableName.json()" + format.isPlaywright() -> "(await $responseVariableName.json())" format.isJavaScript() -> "$responseVariableName.body" else -> "" } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" + }else if (format.isPlaywright()){ // playwright + "expect($field$fieldWithDot).toBe($toPrint);" }else { // javascript "expect($field$fieldPath).toBe($toPrint);" // ($field$)fieldPath } @@ -594,14 +598,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { ".body(\"${path}size()\", equalTo($expectedSize))" } format.isJavaScript() -> { - val field = when { - format.isPlaywright() -> "await $responseVariableName.json()" - format.isJavaScript() -> "$responseVariableName.body" - else -> "" - } if (format.isPlaywright()) { - "expect(await $responseVariableName.json()).toHaveLength($expectedSize);" + val field = if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" + "expect((await $responseVariableName.json())$field).toHaveLength($expectedSize);" } else { + val field = "$responseVariableName.body" "expect($field$fieldPath.length).toBe($expectedSize);" // ($field$)fieldPath } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 12194309bc..da04e9393b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -274,6 +274,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escapedHeader = GeneUtils.applyEscapes(x, GeneUtils.EscapeMode.BODY, format) if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${escapedHeader}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${escapedHeader}',") } else { lines.add(".$set(\"${it.name}\", \"${escapedHeader}\")") @@ -552,6 +554,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(", {") lines.addEmpty() lines.indented { + if (call is org.evomaster.core.problem.rest.data.RestCallAction) { + lines.add("method: \"${call.verb.name.uppercase()}\",") + } lines.add("headers: {") lines.indented { lines.add(getAcceptHeader(call, res)) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 3bd63c272c..0aaf8ecb9b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -219,7 +219,12 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isPlaywright() -> { - lines.add("await request.$verb(") + val verbToUse = call.verb.name.lowercase() + if (verbToUse.uppercase() == "OPTIONS") { + lines.add("await request.fetch(") + } else { + lines.add("await request.$verbToUse(") + } } format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") else -> { From baec9316df2dd3f8655440fe10186ba17a5df071 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Mon, 15 Jun 2026 17:08:42 +0200 Subject: [PATCH 17/24] Improve Playwright output format and refactor E2E test utils post PR feedback. - Refactor 'CookieWriter' and 'TokenWriter' for better Playwright/JS auth handling. - Fix missing 'await' and improve location header extraction in JS/Playwright. - Enhance 'ApiTestCaseWriter' field assertions for JavaScript-based formats. - Simplify 'BlackBoxUtils' and consolidate NPM/Playwright test execution logic. - Update E2E test bases and 'package.json' to support new Playwright scripts. - General cleanup of output format helpers and minor build script fixes. --- .../evomaster/e2etests/utils/BlackBoxUtils.kt | 23 ++---- .../spring/graphql/bb/SpringTestBase.kt | 3 +- .../spring-rest-bb/javascript/package.json | 3 +- .../spring/spring-rest-bb/maven/pom.xml | 2 +- .../e2etests/spring/rest/bb/SpringTestBase.kt | 3 +- .../e2e-tests/spring/spring-rest-rsa/pom.xml | 12 +--- .../org/evomaster/core/output/OutputFormat.kt | 11 ++- .../core/output/auth/CookieWriter.kt | 71 +++++++++++++------ .../evomaster/core/output/auth/TokenWriter.kt | 13 ++-- .../core/output/service/ApiTestCaseWriter.kt | 39 ++++++++-- .../output/service/GraphQLTestCaseWriter.kt | 10 ++- .../output/service/HttpWsTestCaseWriter.kt | 11 ++- .../core/output/service/RestTestCaseWriter.kt | 8 ++- .../core/output/service/TestCaseWriter.kt | 2 +- .../core/output/service/TestSuiteWriter.kt | 9 +-- 15 files changed, 127 insertions(+), 93 deletions(-) diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt index a6089e694b..31a5e277cb 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt @@ -14,15 +14,12 @@ object BlackBoxUtils { private const val GENERATED_FOLDER_NAME = "generated" const val baseLocationForJavaScript = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME" - const val baseLocationForPlaywright = "$JS_BASE_PATH/$GENERATED_FOLDER_NAME/playwright" const val baseLocationForPython = "$PY_BASE_PATH/$GENERATED_FOLDER_NAME" const val baseLocationForJava = "$MAVEN_BASE_PATH/src/test/java" const val baseLocationForKotlin = "$MAVEN_BASE_PATH/src/test/kotlin" fun relativePath(folderName: String) = "$GENERATED_FOLDER_NAME/$folderName" - fun relativePathPlaywright(folderName: String) = "$GENERATED_FOLDER_NAME/playwright/$folderName" - fun checkCoveredTargets(targetLabels: Collection) { targetLabels.forEach { assertTrue(CoveredTargets.isCovered(it), "Target '$it' is not covered") @@ -108,7 +105,7 @@ object BlackBoxUtils { } } - fun runNpmTests(folderRelativePath: String) { + fun runNpmTests(folderRelativePath: String, isPlaywright: Boolean = false) { runNpmInstall() val path = if(folderRelativePath.endsWith("/")){ @@ -118,22 +115,12 @@ object BlackBoxUtils { "$folderRelativePath/" } - val command = listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") - runTestsCommand(command, JS_BASE_PATH, "NPM") - } - - fun runPlaywrightTests(folderRelativePath: String) { - runNpmInstall() - - val path = if(folderRelativePath.endsWith("/")){ - folderRelativePath + val command = if (isPlaywright) { + listOf(npm(), "run", "test:playwright", "--", path) } else { - "$folderRelativePath/" + listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") } - - val npx = if (isWindows()) "npx.cmd" else "npx" - val command = listOf(npx, "playwright", "test", path) - runTestsCommand(command, JS_BASE_PATH, "Playwright") + runTestsCommand(command, JS_BASE_PATH, if (isPlaywright) "Playwright" else "NPM") } fun runPythonTests(folderRelativePath: String) { diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index ee78d106d8..399d12cd72 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -90,7 +90,6 @@ abstract class SpringTestBase : GraphQLTestBase() { lambda: Consumer> ){ val baseLocation = when { - outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython else -> throw IllegalArgumentException("Not supported output type $outputFormat") @@ -101,7 +100,7 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) + outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index aa42e14ae8..21e7571566 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -13,7 +13,8 @@ "urijs": "1.19.6" }, "scripts": { - "test": "jest" + "test": "jest", + "test:playwright": "playwright test" }, "jest": { "testEnvironment": "node", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml index 664c3532c2..a0736e954e 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml @@ -11,7 +11,7 @@ --> org.evomaster - evomaster-e2e-tests + evomaster-e2e-tests-spring 6.0.1-SNAPSHOT ../../pom.xml diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index e00b2bed9c..2434bbc8fb 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -159,7 +159,6 @@ abstract class SpringTestBase : RestTestBase() { lambda: Consumer> ){ val baseLocation = when { - outputFormat.isPlaywright() -> BlackBoxUtils.baseLocationForPlaywright outputFormat.isJavaScript() -> BlackBoxUtils.baseLocationForJavaScript outputFormat.isPython() -> BlackBoxUtils.baseLocationForPython outputFormat.isJava() -> BlackBoxUtils.baseLocationForJava @@ -172,7 +171,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isPlaywright() -> BlackBoxUtils.runPlaywrightTests(BlackBoxUtils.relativePathPlaywright(outputFolderName)) + outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) diff --git a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml index caf3412ad5..4cacc8b78b 100644 --- a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml @@ -35,7 +35,6 @@ org.projectlombok lombok - provided org.apache.commons @@ -134,16 +133,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.14.1 - - - - org.projectlombok - lombok - 1.18.30 - - - + diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index 21d4809802..d18bf4a177 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,7 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, - JS_JEST_PLAYWRIGHT, // Testing new playwright imp + JS_JEST_PLAYWRIGHT, //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST ; @@ -29,11 +29,7 @@ enum class OutputFormat { fun isJavaScript() = this.name.startsWith("js_", true) - /** - * Return true if the output format is Playwright. - * Playwright is currently only supported for JavaScript (or TypeScript). - */ - fun isPlaywright() = this.name.endsWith("_playwright", true) // Testing new playwright imp + fun isPlaywright() = this.name.endsWith("_playwright", true) fun isJavaOrKotlin() = isJava() || isKotlin() @@ -47,4 +43,7 @@ enum class OutputFormat { fun isCsharp() = this.name.startsWith("csharp",ignoreCase = true) fun isPython() = this.name.startsWith("python_", true) + + fun isJsBased() = isJavaScript() || isPlaywright() + // Helper method for JavaScript based formats. Playwright is currently only supported for JavaScript (or TypeScript) // } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index 378d6790a4..a106052d94 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -50,15 +50,17 @@ object CookieWriter { when { format.isJava() -> lines.add("final Map ${cookiesName(k)} = ") format.isKotlin() -> lines.add("val ${cookiesName(k)} : Map = ") - format.isPlaywright() -> lines.add("let ${cookiesName(k)};") - format.isJavaScript() -> lines.add("const ${cookiesName(k)} = ") + format.isJavaScript() -> when { + format.isPlaywright() -> + lines.add("let ${cookiesName(k)};") + else -> + lines.add("const ${cookiesName(k)} = ") + } } if (!format.isPython()) { // TODO: should we use DTO for cookie related requests? - if (!format.isPlaywright()) { - testCaseWriter.startRequest(lines) - } + testCaseWriter.startRequest(lines) lines.indent() } @@ -79,17 +81,32 @@ object CookieWriter { format.isPython() -> lines.append(".cookies") } - if(format.isJavaScript() && !format.isPlaywright()){ - lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") - lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") - lines.appendSemicolon() - } + if (format.isJavaScript()) { + + // SuperAgent/Playwright cookie extraction + if (format.isPlaywright()) { + lines.add(".then((res) => {") + lines.add("const setCookie = res.headers()['set-cookie'];") + lines.add("if (setCookie) {") + lines.add("return setCookie.split('\\n')[0].split(';')[0];") + lines.add("}") + lines.add("return null;") + lines.add("})") + } else { + lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") + lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") + } - if (format.isPlaywright()) { - lines.add(".then(async (res) => { ${cookiesName(k)} = res.headers()['set-cookie']?.split('\\n')?.find(c => c.trim().length > 0)?.split(';')[0]; })") + if (format.isPlaywright()) { + // Playwright cookie extraction must NOT assume res is a response object + lines.add(".then(async (cookie) => {") + lines.add("${cookiesName(k)} = cookie;") + lines.add("})") + } lines.appendSemicolon() } + if (format.isPython()) { lines.add("${cookiesName(k)} = requests.utils.dict_from_cookiejar($targetCookieVariable)") } @@ -118,13 +135,12 @@ object CookieWriter { targetVariable: String ) { - if (format.isPlaywright()) { - callEndpoint(lines, k, format, baseUrlOfSut) - return - } - - if(format.isJavaScript() && !format.isPlaywright()) { + if(format.isJavaScript() ) { callEndpoint(lines, k, format, baseUrlOfSut) + if (format.isPlaywright()) { + return + // to avoid two request calls for the same endpoint + } } if(format.isPython()) { @@ -167,19 +183,27 @@ object CookieWriter { for(header in k.headers) { when { format.isJavaOrKotlin() -> lines.add(".header(\"${header.name}\", \"${header.value}\")") - format.isPlaywright() -> { - // handled in callEndpoint for Playwright + format.isJavaScript() -> when { + format.isPlaywright() -> + { + // handled in callEndPoint for Playwright + } + else -> + lines.add(".set(\"${header.name}\", \"${header.value}\")") } - format.isJavaScript() -> lines.add(".set(\"${header.name}\", \"${header.value}\")") format.isPython() -> { lines.add("headers[\"${header.name}\"] = \"${header.value}\"") } } } - if (format.isJavaScript() && !format.isPlaywright()){ + if (format.isJavaScript()){ // disable redirections - lines.add(".redirects(0)") + if (format.isPlaywright()) { + // handled in callEndPoint + } else { + lines.add(".redirects(0)") + } } /* @@ -250,6 +274,7 @@ object CookieWriter { lines.add("data: '${k.payload}',") } } + lines.add("maxRedirects: 0,") } lines.add("}") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index 81f92ef44f..fb911ed9cd 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -52,16 +52,13 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") - format.isPlaywright() -> { - lines.add("let ${tokenName(k)};") - } + format.isPlaywright() -> lines.add("let ${tokenName(k)};") format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") - format.isPlaywright() -> { - } + format.isPlaywright() -> testCaseWriter.startRequest(lines) //handled in HttpWsTestCaseWriter with startRequest(lines) and await request format.isJavaScript() -> { lines.append("\"\"") lines.appendSemicolon() @@ -91,8 +88,7 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { if (format.isPlaywright()) { - lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;},") - lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;})") } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } @@ -109,8 +105,7 @@ object TokenWriter { TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector if (format.isPlaywright()) { - lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];},") - lines.indented { lines.add("async error => {console.log(await error.response.text()); throw Error(\"Auth failed.\")})") } + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];})") } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 61de4bd224..61716e6400 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -341,14 +341,33 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return text.replace("\$", "\\\$") } + private fun formatFieldPath(fieldPath: String): String { + if (fieldPath.isEmpty()) { + return "" + } + if (format.isJavaOrKotlin()) { + return if (fieldPath.startsWith("'")) "$fieldPath." else "'$fieldPath'." + } + return if (fieldPath.startsWith("[") || fieldPath.startsWith(".")) { + fieldPath + } else { + ".$fieldPath" + } + } + private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { val field = when { - format.isPlaywright() -> "(await $responseVariableName.json())" - format.isJavaScript() -> "$responseVariableName.body" - else -> "" - } + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } + else -> "" + } val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" @@ -404,10 +423,16 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { val field = when { - format.isPlaywright() -> "(await $responseVariableName.json())" - format.isJavaScript() -> "$responseVariableName.body" + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } else -> "" } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" @@ -599,7 +624,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } format.isJavaScript() -> { if (format.isPlaywright()) { - val field = if (fieldPath.isEmpty()) "" else if (fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" + val field = formatFieldPath(fieldPath) "expect((await $responseVariableName.json())$field).toHaveLength($expectedSize);" } else { val field = "$responseVariableName.body" diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 59ce50244f..090947a5b5 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -102,7 +102,7 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { val verb = "post" if (format.isPlaywright()) { - lines.add("request.$verb(") + lines.add("await request.$verb(") } else { lines.add(".$verb(") } @@ -138,6 +138,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } } + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + lines.add("maxRedirects: 0,") + } + lines.add("}") + } + lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index da04e9393b..8e8b11930e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -297,8 +297,14 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") - format.isPlaywright() -> lines.add("'Cookie': ${CookieWriter.cookiesName(elc)},") + format.isJavaScript() -> when { + format.isPlaywright() -> { + val cookieVar = CookieWriter.cookiesName(elc) + lines.add("'Cookie': $cookieVar,") + } + else -> + lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + } // Python cookies are set alongside the headers and body when performing the request } } @@ -564,6 +570,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } lines.add("},") handleBody(call, lines, dtoVar) + lines.add("maxRedirects: 0,") } lines.add("})") } else { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 8dc94680bf..dc4b61e532 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -201,7 +201,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJavaScript() || format.isPlaywright() -> lines.add("const $name = ") + format.isJsBased() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -422,8 +422,10 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { + if (format.isPlaywright()) { - lines.add("const $location = $resVarName.headers()['location'];") + lines.add("const $location = $resVarName") + lines.append(".headers()['location'];") } else { lines.add("const $location = $resVarName.header['location'];") } @@ -458,7 +460,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJavaScript() || format.isPlaywright() -> lines.add("const ") + format.isJsBased() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index e12d28b0a4..5be50cd510 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -139,7 +139,7 @@ abstract class TestCaseWriter { lines.add("}") } - if (format.isJavaScript()) { // add to code block - ok for playwright? + if (format.isJavaScript()) { lines.append(");") } return lines diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 9c09379941..cfdfe05de0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -745,11 +745,8 @@ class TestSuiteWriter { lines.add("const $controller = new $controllerName();") lines.add("let $baseUrlOfSut;") } else { - if (config.outputFormat.isPlaywright()) { - lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") - } else { - lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") - } + lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } } else if (config.outputFormat.isCsharp()) { lines.add("private static readonly HttpClient Client = new HttpClient ();") @@ -800,7 +797,7 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll(async () =>") format.isJavaScript() && format.isPlaywright() -> lines.add("(async ({ request }) =>") } From 65cc4b6917d6ca622417da3a10ef7bd2e2f9079a Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Fri, 19 Jun 2026 14:05:26 +0200 Subject: [PATCH 18/24] Merge master, fix JS_JEST certificate failures and isolate naming side effects TestSuiteWriter: - Reordered 'convertToCompilableTestCode' to trigger naming before header, hopefully preventing side effects from Playwright format swaps (LLM context). - Added global TLS rejection override for JS_JEST to support self-signed certs and removed redundant superagent imports from Playwright suites. ApiTestCaseWriter & HttpWsTestCaseWriter: - Refactored 'baseUrlOfSut' initialization for consistent availability across all modes. HttpWsTestCaseWriter: - Added ignoreHTTPSErrors for makeHttpCall --- .../core/output/service/ApiTestCaseWriter.kt | 3 + .../output/service/HttpWsTestCaseWriter.kt | 2 + .../core/output/service/TestSuiteWriter.kt | 74 +++++++++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 61716e6400..66362d0696 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -21,6 +21,8 @@ import java.math.BigDecimal import java.math.BigInteger abstract class ApiTestCaseWriter : TestCaseWriter() { + + protected var baseUrlOfSut: String = "" companion object{ private val mapper = ObjectMapper() @@ -45,6 +47,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { insertionVars: MutableList>, testName: String ) { + this.baseUrlOfSut = baseUrlOfSut //TODO: REFACTOR TO HANDLE MULTIPLE DATABASES val initializingSqlActions = ind.individual.seeInitializingActions().filterIsInstance() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 987caf47b1..7eddb33bbb 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -396,6 +396,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { baseUrlOfSut: String, addTimeMeasurement: Boolean, ) { + this.baseUrlOfSut = baseUrlOfSut val exActions = mutableListOf() // add all used external service actions for the action @@ -571,6 +572,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("},") handleBody(call, lines, dtoVar) lines.add("maxRedirects: 0,") + lines.add("ignoreHTTPSErrors: true,") } lines.add("})") } else { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index fe08c169ec..91e24790fb 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -6,12 +6,12 @@ import org.evomaster.client.java.controller.api.dto.database.operations.MongoIns import org.evomaster.client.java.controller.api.dto.database.operations.RedisInsertionDto import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils import org.evomaster.core.EMConfig +import org.evomaster.core.llm.service.LlmService import org.evomaster.core.output.* import org.evomaster.core.output.TestWriterUtils.getWireMockVariableName import org.evomaster.core.output.TestWriterUtils.handleDefaultStubForAsJavaOrKotlin import org.evomaster.core.output.dto.DtoWriter import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy -import org.evomaster.core.output.naming.TestCaseNamingStrategyFactory import org.evomaster.core.problem.api.ApiWsIndividual import org.evomaster.core.problem.enterprise.service.EnterpriseSampler import org.evomaster.core.problem.externalservice.httpws.HttpWsExternalService @@ -60,7 +60,7 @@ class TestSuiteWriter { private val log: Logger = LoggerFactory.getLogger(TestSuiteWriter::class.java) - private const val baseUrlOfSut = "baseUrlOfSut" + const val baseUrlOfSut = "baseUrlOfSut" private const val fixtureClass = "ControllerFixture" private const val fixture = "_fixture" private const val browser = "browser" @@ -95,6 +95,9 @@ class TestSuiteWriter { @Inject private lateinit var httpCallbackVerifier: HttpCallbackVerifier + @Inject + private lateinit var llmService: LlmService + fun writeTests(testSuiteCode: TestSuiteCode){ saveToDisk(testSuiteCode.code, Paths.get(config.outputFolder, testSuiteCode.testSuitePath)) @@ -136,9 +139,22 @@ class TestSuiteWriter { controllerInput: String? ): TestSuiteCode { + val testSuiteOrganizer = TestSuiteOrganizer(config, llmService) + + //catch any sorting problems (see NPE is SortingHelper on Trello) + val tests = try { + testSuiteOrganizer.createSortedTestCases(solution, testCaseWriter) + } catch (ex: Exception) { + log.warn( + "A failure has occurred with the test sorting. Reverting to default settings. \n" + + "Exception: ${ex.localizedMessage} \n" + + "At ${ex.stackTrace.joinToString(separator = " \n -> ")}. " + ) + // fallback to numbered naming strategy upon failure + NumberedTestCaseNamingStrategy(solution).getTestCases() + } + val lines = Lines(config.outputFormat) - val testSuiteOrganizer = TestSuiteOrganizer() - val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) header(solution, testSuiteFileName, lines, timestamp, controllerName) @@ -153,20 +169,6 @@ class TestSuiteWriter { beforeAfterMethods(solution, controllerName, controllerInput, lines, config.outputFormat, testSuiteFileName) - //catch any sorting problems (see NPE is SortingHelper on Trello) - val tests = try { - // TODO skip to sort RPC for the moment - testSuiteOrganizer.sortTests(solution, namingStrategy, config.testCaseSortingStrategy) - } catch (ex: Exception) { - log.warn( - "A failure has occurred with the test sorting. Reverting to default settings. \n" - + "Exception: ${ex.localizedMessage} \n" - + "At ${ex.stackTrace.joinToString(separator = " \n -> ")}. " - ) - // fallback to numbered naming strategy upon failure - NumberedTestCaseNamingStrategy(solution).getTestCases() - } - val testSuitePath = getTestSuitePath(testSuiteFileName, config) val tc = tests.mapNotNull { test -> @@ -539,7 +541,11 @@ class TestSuiteWriter { } if (format.isJavaScript()) { - lines.add("const superagent = require(\"superagent\");") + if (format.isPlaywright()) { + lines.add("const { test, expect } = require(\"@playwright/test\");") + } else { + lines.add("const superagent = require(\"superagent\");") + } val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -548,7 +554,7 @@ class TestSuiteWriter { if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") } - if (config.testTimeout > 0) { + if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") } } @@ -791,7 +797,13 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() -> lines.add("beforeAll( async () =>") + format.isJavaScript() -> { + if (format.isPlaywright()) { + lines.add("test.beforeAll( async () =>") + } else { + lines.add("beforeAll( async () =>") + } + } } lines.block { @@ -847,6 +859,10 @@ class TestSuiteWriter { addStatement("RestAssured.urlEncodingEnabled = false", lines) } + if (format.isJavaScript() && !format.isPlaywright()) { + addStatement("process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'", lines) + } + if (config.enableBasicAssertions && format.isJavaOrKotlin()) { lines.add("RestAssured.config = RestAssured.config()") lines.indented { @@ -946,7 +962,13 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") + format.isJavaScript() -> { + if (format.isPlaywright()) { + lines.add("test.afterAll( async () =>") + } else { + lines.add("afterAll( async () =>") + } + } } if (!format.isCsharp()) { @@ -999,7 +1021,13 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") + format.isJavaScript() -> { + if (format.isPlaywright()) { + lines.add("test.beforeEach(async () => ") + } else { + lines.add("beforeEach(async () => ") + } + } //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } From 6ae9d66d91f6cd56a11f0cc1c5d54a7e520aa7ab Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Mon, 22 Jun 2026 16:47:58 +0200 Subject: [PATCH 19/24] Merge master and fix failing BB Playwright tests by using request options object and removing .redirects(0) chain for Playwright in AuthWriter. --- .../evomaster/core/output/auth/AuthWriter.kt | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt index 6bc5ce2200..bb843cff41 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt @@ -28,6 +28,57 @@ object AuthWriter { targetVariable: String? ) { + // Special handling for Playwright: build request with options object instead of chained calls + if (format.isPlaywright()) { + val verb = k.verb.name.lowercase() + lines.add(".$verb(") + if (k.externalEndpointURL != null) { + lines.append("\"${k.externalEndpointURL}\"") + } else { + lines.append("$baseUrlOfSut + \"") + lines.append("${k.endpoint}\"") + } + lines.append(", {") + lines.addEmpty() + lines.indented { + // headers block + lines.add("headers: {") + lines.indented { + // content-type header if present + val contentType = k.contentType + if (contentType != null) { + lines.add("'content-type': \"${contentType.defaultValue}\",") + } + // any additional headers + for (header in k.headers) { + lines.add("'${header.name}': \"${header.value}\",") + } + } + lines.add("},") + + // body/payload + val contentType = k.contentType + if (contentType != null) { + when (contentType) { + ContentType.X_WWW_FORM_URLENCODED -> { + lines.add("data: \"${k.payload}\",") + } + ContentType.JSON -> { + testCaseWriter.printSendJsonBody(k.payload!!, lines) + // printSendJsonBody for Playwright appends a trailing comma + } + else -> throw IllegalStateException("Currently not supporting yet ${k.contentType} in login") + } + } + + // disable redirections and ignore HTTPS errors + lines.add("maxRedirects: 0,") + lines.add("ignoreHTTPSErrors: true,") + } + lines.add("})") + return + } + if(format.isJavaScript()) { callEndpoint(lines, k, format, baseUrlOfSut) } @@ -75,8 +126,8 @@ object AuthWriter { } } - if (format.isJavaScript()){ - // disable redirections + // Disable redirections where supported (SuperAgent/Frisby style only) + if (format.isJavaScript() && !format.isPlaywright()){ lines.add(".redirects(0)") } From fc4a3ceac8c70249ed588831c59676daac6a971e Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Mon, 6 Jul 2026 16:37:28 +0200 Subject: [PATCH 20/24] - Rename OutputFormat JS_JEST_PLAYWRIGHT to JS_PLAYWRIGHT; streamline Playwright codegen and cleanup - Drop OutputFormat.isJsBased; use local helper in RestTestCaseWriter - Unify JS/Playwright const emission; handle non-standard verbs via request.fetch - Add playwrightFixturesFor to control fixtures in test signatures - Remove unused baseUrlOfSut assignments; skip jest.setTimeout for Playwright - Remove whitespace tweak in spring-rest-rsa pom.xml --- .../e2e-tests/spring/spring-rest-rsa/pom.xml | 1 - .../org/evomaster/core/output/OutputFormat.kt | 5 +---- .../core/output/service/ApiTestCaseWriter.kt | 4 +--- .../core/output/service/HttpWsTestCaseWriter.kt | 6 ++---- .../core/output/service/RestTestCaseWriter.kt | 13 ++++++++++--- .../core/output/service/TestCaseWriter.kt | 14 ++++++++++++-- .../core/output/service/TestSuiteWriter.kt | 1 + 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml index 4cacc8b78b..2fe734f2a0 100644 --- a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml @@ -133,7 +133,6 @@ org.apache.maven.plugins maven-compiler-plugin - diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index d18bf4a177..d46f77115a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,7 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, - JS_JEST_PLAYWRIGHT, + JS_PLAYWRIGHT, //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST ; @@ -43,7 +43,4 @@ enum class OutputFormat { fun isCsharp() = this.name.startsWith("csharp",ignoreCase = true) fun isPython() = this.name.startsWith("python_", true) - - fun isJsBased() = isJavaScript() || isPlaywright() - // Helper method for JavaScript based formats. Playwright is currently only supported for JavaScript (or TypeScript) // } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 66362d0696..d11d023cad 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -21,8 +21,7 @@ import java.math.BigDecimal import java.math.BigInteger abstract class ApiTestCaseWriter : TestCaseWriter() { - - protected var baseUrlOfSut: String = "" + companion object{ private val mapper = ObjectMapper() @@ -47,7 +46,6 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { insertionVars: MutableList>, testName: String ) { - this.baseUrlOfSut = baseUrlOfSut //TODO: REFACTOR TO HANDLE MULTIPLE DATABASES val initializingSqlActions = ind.individual.seeInitializingActions().filterIsInstance() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 7eddb33bbb..07c6fe7704 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -104,8 +104,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") - format.isPlaywright() -> lines.append("const $resVarName = ") - format.isJavaScript() -> lines.append("const $resVarName = ") + format.isJavaScript() || format.isPlaywright() -> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") } @@ -216,7 +215,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { but that is not the case for the other libraries used for example in JS and C# */ return config.enableBasicAssertions && - (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.JS_JEST_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) + (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat. JS_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) } protected fun handleHeaders(call: HttpWsAction, lines: Lines) { @@ -396,7 +395,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { baseUrlOfSut: String, addTimeMeasurement: Boolean, ) { - this.baseUrlOfSut = baseUrlOfSut val exActions = mutableListOf() // add all used external service actions for the action diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index dc4b61e532..359e1f748d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines import org.evomaster.core.output.SqlWriter import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestWriterUtils +import org.evomaster.core.output.OutputFormat import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.httpws.HttpWsAction @@ -35,6 +36,9 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { private val log = LoggerFactory.getLogger(RestTestCaseWriter::class.java) } + // Local extension/helper to avoid depending on OutputFormat.isJsBased from elsewhere + private fun OutputFormat.isJsBasedLocal(): Boolean = this.isJavaScript() || this.isPlaywright() + @Inject private lateinit var partialOracles: PartialOracles @@ -201,7 +205,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJsBased() -> lines.add("const $name = ") + format.isJsBasedLocal() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -220,11 +224,14 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isPlaywright() -> { val verbToUse = call.verb.name.lowercase() - if (verbToUse.uppercase() == "OPTIONS") { + // Playwright provides direct helpers for common verbs. For the rest, use fetch with explicit method. + val hasDirectHelper = setOf("get", "post", "put", "patch", "delete", "head").contains(verbToUse) + if (!hasDirectHelper) { lines.add("await request.fetch(") } else { lines.add("await request.$verbToUse(") } + // Note: the options object is appended in HttpWsTestCaseWriter.makeHttpCall. } format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") else -> { @@ -460,7 +467,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJsBased() -> lines.add("const ") + format.isJsBasedLocal() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 5be50cd510..aa541fe9f0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -73,6 +73,13 @@ abstract class TestCaseWriter { protected abstract fun addTestCommentBlock(lines: Lines, test: TestCase) + /** + * Compute the list of Playwright fixtures to destructure in the test signature. + * For REST-only generation we currently need only the `request` fixture. + * This helper makes it easy to extend later (e.g., include `page`) without touching call sites. + */ + private fun playwrightFixturesFor(test: TestCase): String = "request" + fun convertToCompilableTestCode( test: TestCase, baseUrlOfSut: String, @@ -117,8 +124,11 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isJavaScript() && !format.isPlaywright()-> lines.add("test(\"${test.name}\", async () => {") - format.isJavaScript() && format.isPlaywright() -> lines.add("test(\"${test.name}\", async ({ request }) => {") + format.isPlaywright() -> { + val fixtures = playwrightFixturesFor(test) + lines.add("test(\"${test.name}\", async ({ $fixtures }) => {") + } + format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 91e24790fb..52e0be1503 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -554,6 +554,7 @@ class TestSuiteWriter { if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") } + // Playwright has its own test runner and does not use Jest if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") } From 1becef80f8d827a8aa4cda8c77baa7e2399aedc0 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 9 Jul 2026 11:51:17 +0200 Subject: [PATCH 21/24] Isolate auth/request generation for Playwright: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AuthWriter: apply placeholder replacements and options-object requests. HttpWsTestCaseWriter: keep Playwright-specific body/headers handling. ApiTestCaseWriter: use Playwright’s response API for plain-text assertions. CreateUsersWriter: do not emit SuperAgent .ok(...) in Playwright path. --- .../evomaster/core/output/auth/AuthWriter.kt | 17 ++++- .../core/output/auth/CreateUsersWriter.kt | 2 +- .../core/output/service/ApiTestCaseWriter.kt | 8 +-- .../output/service/HttpWsTestCaseWriter.kt | 71 +++++++++++++++---- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt index a46ec7038b..54ff8dc97c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt @@ -31,8 +31,12 @@ object AuthWriter { placeHolderResolver: PlaceHolderResolver? ) { - // Special handling for Playwright: build request with options object instead of chained calls + // Compute placeholder replacements for PLaywright (used for dynamic user data) if (format.isPlaywright()) { + val replacements: List? = placeHolderResolver?.placeHolders?.entries?.map { + val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) + ".replace(\"${placeholder}\", ${it.value})" + } val verb = k.verb.name.lowercase() lines.add(".$verb(") if (k.externalEndpointURL != null) { @@ -64,10 +68,17 @@ object AuthWriter { if (contentType != null) { when (contentType) { ContentType.X_WWW_FORM_URLENCODED -> { - lines.add("data: \"${k.payload}\",") + if (replacements == null) { + lines.add("data: \"${k.payload}\",") + } else { + lines.add("data: \"${k.payload}\"") + // Apply string replacements for placeholders + replacements.forEach { lines.append(it) } + lines.append(",") + } } ContentType.JSON -> { - testCaseWriter.printSendJsonBody(k.payload!!, lines) + testCaseWriter.printSendJsonBody(k.payload!!, lines, functionsOnString = replacements) // printSendJsonBody for Playwright appends a trailing comma } else -> throw IllegalStateException("Currently not supporting yet ${k.contentType} in login") diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt index 9efc536170..396b960e2a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt @@ -106,7 +106,7 @@ object CreateUsersWriter { lines.add(".then()") lines.add(".statusCode(both(greaterThanOrEqualTo(200)).and(Matchers.lessThan(400)))") } - if(format.isJavaScript()){ + if(format.isJavaScript() && !format.isPlaywright()){ lines.add(".ok(res => res.status >= 200 && res.status < 400)") } lines.appendSemicolon() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index f9d3f1159e..a95a4d5154 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -666,13 +666,13 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return ".body(containsString(\"$content\"))" } - if (format.isJavaScript()) { - return "expect($responseVariableName.text).toContain(\"$content\");" } - if (format.isPlaywright()) { - return "await $responseVariableName.text();" + return "expect(await $responseVariableName.text()).toContain(\"$content\");" } + if (format.isJavaScript()) { + return "expect($responseVariableName.text).toContain(\"$content\");" } + if (format.isCsharp()) { val k = when { content.startsWith("\"") -> content.substring(1, content.length - 1) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 9d99bae508..ed505043d4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -73,7 +73,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } } - protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && hasTimeoutFault(res) + protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && !format.isPlaywright() && hasTimeoutFault(res) protected fun expectsAssertRaises(res: ActionResult) = format.isPython() && hasTimeoutFault(res) @@ -133,7 +133,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") // for a call expected to reject, wrap it in await expect(...).rejects.toThrow() - format.isJavaScript() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") + // For Playwright, the verb handler will write `await request.(...)` + format.isJavaScript() && !format.isPlaywright() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -211,6 +212,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" + // For Playwright, headers are expressed as entries in an object literal + format.isPlaywright() -> "'Accept': " format.isJavaScript() -> ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " @@ -220,7 +223,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun closeAcceptHeader(openedHeader: String): String { var result = openedHeader - if (!config.outputFormat.isPython()) { + // Do not append a closing parenthesis for Playwright as we are not using a method call + if (!config.outputFormat.isPython() && !format.isPlaywright()) { result += ")" } if (format.isCsharp()){ @@ -749,7 +753,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = \"$text\"") } } - + format.isPlaywright() -> { + // In Playwright, body data must be inside the request options object + lines.add("data: \"$text\",") + } else -> lines.add(".$send(\"$text\")") } } else { @@ -758,6 +765,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = \"\"") } //TODO isn't this valid just for Kotlin??? + format.isPlaywright() -> { + // Empty body for Playwright should still be provided in options when needed + lines.add("data: \"\",") + } else -> lines.add(".$send(\"${"""\"\""""}\")") } } @@ -781,6 +792,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return } + // For Playwright, if a DTO variable is available and allowed, send it directly in the options object + if (format.isPlaywright() && shouldUseDtoForPayload(dtoVar)) { + lines.add("data: ${dtoVar},") + return + } + if(dtoVar != null && functionsOnString != null) { throw IllegalArgumentException("Cannot use extra functions on string JSON when using DTOs") } @@ -796,7 +813,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { functionsOnString?.forEach { lines.append(it) } } format.isPlaywright() -> { - writePlaywrightPayload(lines, bodyLines, false) + writePlaywrightPayload(lines, bodyLines, false, functionsOnString) } format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString) else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, functionsOnString) @@ -820,8 +837,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString - //format.isPlaywright() -> writePlaywrightPayload(lines, bodyLines, true) + format.isPlaywright() -> writePlaywrightPayload(lines, bodyLines, true, functionsOnString) + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString) else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, functionsOnString) } } @@ -892,8 +909,16 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(")") } - private fun writePlaywrightPayload(lines: Lines, bodyLines: List, isMultiLine: Boolean) { - lines.add("data: ${bodyLines.first()}") + private fun writePlaywrightPayload( + lines: Lines, + bodyLines: List, + isMultiLine: Boolean, + functionsOnString: List? = null + ) { + // Build the JSON string expression first + lines.add("data: JSON.parse(${bodyLines.first()}") + + // If multi-line, concatenate subsequent lines if (isMultiLine) { lines.append(" + ") lines.indented { @@ -903,6 +928,26 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("${bodyLines.last()}") } } + + // Apply any string transformation functions (e.g., .replace(...)) + functionsOnString?.let { funcs -> + if (funcs.isNotEmpty()) { + // If we built a multi-line concatenation, ensure method calls are appended correctly + if (!isMultiLine) { + // single-line: can simply append functions inline + funcs.forEach { lines.append(it) } + } else { + // multi-line: add functions each on its own line for readability + lines.indented { + funcs.forEach { lines.add(it) } + } + } + } + } + + // Close JSON.parse(...), then the options object entry + lines.append(")", + ) lines.append(",") } @@ -968,6 +1013,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escaped = GeneUtils.applyEscapes(value, GeneUtils.EscapeMode.ASSERTION, format) when { format.isJavaOrKotlin() -> ".header(\"$name\", \"$escaped\")" + format.isPlaywright() -> + "expect($responseVariableName.headers()[\"$name\"]?.startsWith(\"$escaped\")).toBe(true);" format.isJavaScript() -> "expect($responseVariableName.header[\"$name\"].startsWith(\"$escaped\")).toBe(true);" format.isPython() -> "assert \"$escaped\" in $responseVariableName.headers[\"$name\"]" @@ -976,6 +1023,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else { when { format.isJavaOrKotlin() -> ".header(\"$name\", isEmptyOrNullString())" + format.isPlaywright() -> + "expect($responseVariableName.headers()[\"$name\"]).toBeUndefined();" format.isJavaScript() -> "expect($responseVariableName.header[\"$name\"]).toBeUndefined();" format.isPython() -> "assert \"$name\" not in $responseVariableName.headers" @@ -1098,9 +1147,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - if (format.isPlaywright()) { - // No extra processing for Playwright - if (format.isJavaScript()) { + if (format.isJavaScript() && !format.isPlaywright()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... From 24873dfb89dd474839771c95418540be734c9990 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Thu, 9 Jul 2026 16:16:54 +0200 Subject: [PATCH 22/24] Adjust request/body and suite scaffolding, update options.md and merge master: HttpWsTestCaseWriter: emit form bodies as data: inside Playwright request options (instead of chained calls). TestSuiteWriter: remove Playwright test.beforeAll and wrap Playwright imports/tests in a Jest guard with a placeholder test to prevent Jest from executing PW suites. --- .../output/service/HttpWsTestCaseWriter.kt | 4 ++++ .../core/output/service/TestSuiteWriter.kt | 24 +++++++++++++++---- docs/options.md | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index ed505043d4..0fd66e82fa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -707,6 +707,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$body\"") } + format.isPlaywright() -> { + // In Playwright, use the options object field instead of chained calls + lines.add("data: \"$body\",") + } else -> lines.add(".$send(\"$body\")") } } else if (bodyParam.isXml()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 9fc454b9d4..f8d577e86c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -542,6 +542,9 @@ class TestSuiteWriter { if (format.isJavaScript()) { lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';") if (format.isPlaywright()) { + // Guard: avoid executing Playwright tests when Jest is running in the same folder + lines.add("const isJest = typeof process !== 'undefined' && process.env && process.env.JEST_WORKER_ID;") + lines.add("if (!isJest) {") lines.add("const { test, expect } = require(\"@playwright/test\");") } else { lines.add("const superagent = require(\"superagent\");") @@ -792,6 +795,13 @@ class TestSuiteWriter { val format = config.outputFormat + // For Playwright, avoid emitting a top-level test.beforeAll hook to prevent + // Playwright runner errors when files are imported indirectly. Also, for + // black-box scenarios the hook would be empty anyway. + if (format.isJavaScript() && format.isPlaywright()) { + return + } + when { format.isJUnit4() -> lines.add("@BeforeClass") format.isJUnit5() -> lines.add("@BeforeAll") @@ -803,11 +813,7 @@ class TestSuiteWriter { lines.add("fun initClass()") } format.isJavaScript() -> { - if (format.isPlaywright()) { - lines.add("test.beforeAll( async () =>") - } else { - lines.add("beforeAll( async () =>") - } + lines.add("beforeAll( async () =>") } } @@ -1137,6 +1143,14 @@ class TestSuiteWriter { private fun footer(lines: Lines) { + // Close Playwright guard if it was opened; add a Jest placeholder test in the else branch + if (config.outputFormat.isPlaywright()) { + lines.add("} else {") + lines.indented { + lines.add("test('placeholder - Playwright suite skipped under Jest', () => { expect(true).toBe(true); });") + } + lines.add("}") + } if (config.outputFormat.isJavaOrKotlin() || config.outputFormat.isCsharp()) { //due to opening of class lines.addEmpty(2) diff --git a/docs/options.md b/docs/options.md index 1029d38305..1d0c8d8bde 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_JEST_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.| From 5da0f4adc4939e252d7b3327827097138afaeff9 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Fri, 10 Jul 2026 13:13:40 +0200 Subject: [PATCH 23/24] Remove commented out code and unnecessary isJavaScript condition. --- .../org/evomaster/core/output/service/TestSuiteWriter.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index f8d577e86c..42deba1947 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -798,7 +798,7 @@ class TestSuiteWriter { // For Playwright, avoid emitting a top-level test.beforeAll hook to prevent // Playwright runner errors when files are imported indirectly. Also, for // black-box scenarios the hook would be empty anyway. - if (format.isJavaScript() && format.isPlaywright()) { + if (format.isPlaywright()) { return } @@ -870,10 +870,6 @@ class TestSuiteWriter { addStatement("RestAssured.urlEncodingEnabled = false", lines) } - //if (format.isJavaScript() && !format.isPlaywright()) { - // addStatement("process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'", lines) - //} - if (format.isJavaOrKotlin()) { // global HTTP client config. The socket timeout MUST match the one used during // fuzzing (tcpTimeoutMs), so that timeout faults are reproduced consistently From 1291cd28df0751fde7810ba471212b7f4d9da626 Mon Sep 17 00:00:00 2001 From: May Kristin Ugelstad <“may.kristin.ugelstad@aize.io”> Date: Wed, 19 Aug 2026 17:25:25 +0200 Subject: [PATCH 24/24] Streamline Playwright handling and header setup across writers: - GraphQLTestCaseWriter: simplify JS header logic, Playwright headers/body are handled centrally - HttpWsTestCaseWriter, RestTestCaseWriter, ApiTestCaseWriter, TestCaseWriter: align Playwright request construction and payload handling for consistency - TokenWriter, CookieWriter: minor adjustments for header propagation with Playwright - Add Playwright-specific options where applicable - Update e2e Spring test bases and JS package metadata to match Playwright path --- .../spring/graphql/bb/SpringTestBase.kt | 8 +- .../spring-rest-bb/javascript/package.json | 3 +- .../e2etests/spring/rest/bb/SpringTestBase.kt | 6 +- .../evomaster/core/output/auth/AuthWriter.kt | 183 +++++++++++------- .../core/output/auth/CookieWriter.kt | 27 +-- .../evomaster/core/output/auth/TokenWriter.kt | 39 ++-- .../core/output/service/ApiTestCaseWriter.kt | 19 +- .../output/service/GraphQLTestCaseWriter.kt | 8 +- .../output/service/HttpWsTestCaseWriter.kt | 75 +++---- .../core/output/service/RestTestCaseWriter.kt | 14 +- .../core/output/service/TestCaseWriter.kt | 13 +- .../core/output/service/TestSuiteWriter.kt | 11 -- 12 files changed, 241 insertions(+), 165 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index 399d12cd72..d99e3079e5 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -100,8 +100,12 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) - outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + outputFormat.isJavaScript() -> + if (outputFormat.isPlaywright()) { + BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) + } else { + BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + } outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") } diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index 21e7571566..153d9a34f2 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -18,6 +18,7 @@ }, "jest": { "testEnvironment": "node", - "testRegex": "generated/.*/.*[tT]est\\.js$" + "testRegex": "generated/.*/.*[tT]est\\.js$", + "testPathIgnorePatterns": [".*_pw\\.js$"] } } diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index 2434bbc8fb..8b3975fa01 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -171,8 +171,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) - outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), outputFormat.isPlaywright()) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) outputFormat.isKotlin() -> BlackBoxUtils.runKotlinTests(outputFolderName) @@ -218,6 +217,9 @@ abstract class SpringTestBase : RestTestBase() { if(outputFormat.isKotlin()){ setOption(args,"outputFilePrefix",BlackBoxUtils.getOutputFilePrefixKotlin(outputFolderName)) } + if (outputFormat.isPlaywright()) { + setOption(args, "outputFilePrefix", "EvoMaster_pw") + } defaultSeed++ lambda.accept(ArrayList(args)) diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt index 54ff8dc97c..69fda3eefa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt @@ -31,65 +31,9 @@ object AuthWriter { placeHolderResolver: PlaceHolderResolver? ) { - // Compute placeholder replacements for PLaywright (used for dynamic user data) + // Playwright specific handling if (format.isPlaywright()) { - val replacements: List? = placeHolderResolver?.placeHolders?.entries?.map { - val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) - ".replace(\"${placeholder}\", ${it.value})" - } - val verb = k.verb.name.lowercase() - lines.add(".$verb(") - if (k.externalEndpointURL != null) { - lines.append("\"${k.externalEndpointURL}\"") - } else { - lines.append("$baseUrlOfSut + \"") - lines.append("${k.endpoint}\"") - } - lines.append(", {") - lines.addEmpty() - lines.indented { - // headers block - lines.add("headers: {") - lines.indented { - // content-type header if present - val contentType = k.contentType - if (contentType != null) { - lines.add("'content-type': \"${contentType.defaultValue}\",") - } - // any additional headers - for (header in k.headers) { - lines.add("'${header.name}': \"${header.value}\",") - } - } - lines.add("},") - - // body/payload - val contentType = k.contentType - if (contentType != null) { - when (contentType) { - ContentType.X_WWW_FORM_URLENCODED -> { - if (replacements == null) { - lines.add("data: \"${k.payload}\",") - } else { - lines.add("data: \"${k.payload}\"") - // Apply string replacements for placeholders - replacements.forEach { lines.append(it) } - lines.append(",") - } - } - ContentType.JSON -> { - testCaseWriter.printSendJsonBody(k.payload!!, lines, functionsOnString = replacements) - // printSendJsonBody for Playwright appends a trailing comma - } - else -> throw IllegalStateException("Currently not supporting yet ${k.contentType} in login") - } - } - - // disable redirections and ignore HTTPS errors - lines.add("maxRedirects: 0,") - lines.add("ignoreHTTPSErrors: true,") - } - lines.add("})") + addPlaywrightAuthCall(lines, k, testCaseWriter, format, baseUrlOfSut, placeHolderResolver) return } @@ -111,10 +55,7 @@ object AuthWriter { } } - val replacements = placeHolderResolver?.placeHolders?.entries?.map { - val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) - ".replace(\"${placeholder}\", ${it.value})" - } + val replacements = buildReplacements(format, placeHolderResolver) when (contentType) { ContentType.X_WWW_FORM_URLENCODED -> { @@ -188,11 +129,101 @@ object AuthWriter { } } - private fun callEndpoint( + /** + * Emit Playwright request call for authentication endpoints, including headers, body and options. + */ + private fun addPlaywrightAuthCall( lines: Lines, k: CallToEndpoint, + testCaseWriter: HttpWsTestCaseWriter, format: OutputFormat, - baseUrlOfSut: String + baseUrlOfSut: String, + placeHolderResolver: PlaceHolderResolver? + ) { + // Compute placeholder replacements for Playwright (used for dynamic user data) + val replacements: List? = buildReplacements(format, placeHolderResolver) + + // Hoist contentType once and reuse below (headers + body) + val contentType = k.contentType + + // Emit verb and URL, leave call open to add Playwright options object + emitVerbAndUrl(lines, k, format, baseUrlOfSut, close = false) + lines.append(", {") + lines.addEmpty() + lines.indented { + emitPlaywrightHeaders(lines, k, contentType) + emitPlaywrightBody(lines, k, testCaseWriter, replacements, contentType) + emitPlaywrightRequestOptions(lines) + } + lines.add("})") + } + + /** + * Emit the Playwright headers block, including optional content-type and any custom headers. + */ + private fun emitPlaywrightHeaders( + lines: Lines, + k: CallToEndpoint, + contentType: ContentType? + ) { + lines.add("headers: {") + lines.indented { + if (contentType != null) { + lines.add("'content-type': \"${contentType.defaultValue}\",") + } + for (header in k.headers) { + lines.add("'${header.name}': \"${header.value}\",") + } + } + lines.add("},") + } + + /** + * Emit the Playwright body/payload block according to the content type. + */ + private fun emitPlaywrightBody( + lines: Lines, + k: CallToEndpoint, + testCaseWriter: HttpWsTestCaseWriter, + replacements: List?, + contentType: ContentType? + ) { + if (contentType == null) return + when (contentType) { + ContentType.X_WWW_FORM_URLENCODED -> { + if (replacements == null) { + lines.add("data: \"${k.payload}\",") + } else { + lines.add("data: \"${k.payload}\"") + replacements.forEach { lines.append(it) } + lines.append(",") + } + } + ContentType.JSON -> { + testCaseWriter.printSendJsonBody(k.payload!!, lines, functionsOnString = replacements) + } + else -> throw IllegalStateException("Currently not supporting yet ${k.contentType} in login") + } + } + + /** + * Emit standard Playwright request options for auth requests. + */ + private fun emitPlaywrightRequestOptions(lines: Lines) { + lines.add("maxRedirects: 0,") + lines.add("ignoreHTTPSErrors: true,") + } + + /** + * Emit the HTTP verb and URL for the call, with an option to close the parenthesis. + * Example (Java/JS): `.post(baseUrl + "/login"` and closes with `)` if `close=true`. + */ + private fun emitVerbAndUrl( + lines: Lines, + k: CallToEndpoint, + format: OutputFormat, + baseUrlOfSut: String, + close: Boolean = false ) { val verb = k.verb.name.lowercase() lines.add(".$verb(") @@ -206,8 +237,30 @@ object AuthWriter { } lines.append("${k.endpoint}\"") } - if (!format.isPython()) { + if (close) { lines.append(")") } } + + /** + * Build a list of string replacement functions to apply to payload strings. + * Each entry is a snippet like: `.replace("", value)`; returns null if no placeholders. + */ + private fun buildReplacements( + format: OutputFormat, + placeHolderResolver: PlaceHolderResolver? + ): List? = placeHolderResolver?.placeHolders?.entries?.map { + val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) + ".replace(\"${placeholder}\", ${it.value})" + } + + private fun callEndpoint( + lines: Lines, + k: CallToEndpoint, + format: OutputFormat, + baseUrlOfSut: String + ) { + // Delegate to the shared emitter; Python keeps it open, others close it + emitVerbAndUrl(lines, k, format, baseUrlOfSut, close = !format.isPython()) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index 5dca24321c..27eaaf47b1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -87,22 +87,25 @@ object CookieWriter { // SuperAgent/Playwright cookie extraction if (format.isPlaywright()) { lines.add(".then((res) => {") - lines.add("const setCookie = res.headers()['set-cookie'];") - lines.add("if (setCookie) {") - lines.add("return setCookie.split('\\n')[0].split(';')[0];") - lines.add("}") - lines.add("return null;") + lines.indented { + lines.add("const setCookie = res.headers()['set-cookie'];") + lines.add("if (setCookie) {") + lines.indented { + lines.add("return setCookie.split('\\n')[0].split(';')[0];") + } + lines.add("}") + lines.add("return null;") + } lines.add("})") - } else { - lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") - lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") - } - - if (format.isPlaywright()) { // Playwright cookie extraction must NOT assume res is a response object lines.add(".then(async (cookie) => {") - lines.add("${cookiesName(k)} = cookie;") + lines.indented { + lines.add("${cookiesName(k)} = cookie;") + } lines.add("})") + } else { + lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") + lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") } lines.appendSemicolon() } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index bb73a7f8d5..1d8d00caf2 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -54,18 +54,19 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") - format.isPlaywright() -> lines.add("let ${tokenName(k)};") - format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") + format.isJavaScript() && format.isPlaywright() -> lines.add("let ${tokenName(k)};") + format.isJavaScript() && !format.isPlaywright()-> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") - format.isPlaywright() -> testCaseWriter.startRequest(lines) //handled in HttpWsTestCaseWriter with startRequest(lines) and await request format.isJavaScript() -> { - lines.append("\"\"") - lines.appendSemicolon() - lines.addEmpty() - lines.append("await superagent") + if (format.isPlaywright()) { + testCaseWriter.startRequest(lines) //handled in HttpWsTestCaseWriter with startRequest(lines) and await request + } else {lines.append("\"\"") + lines.appendSemicolon() + lines.addEmpty() + lines.append("await superagent")} } } @@ -89,11 +90,11 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { - if (format.isPlaywright()) { - lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;})") - } else if (format.isJavaScript()) { - lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") - lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } + if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;})") + } else {lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") + lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") }} } else if (format.isPython()) { lines.add("${tokenName(k)} = ${responseName(k)}.json()$path") }else if (format.isJavaOrKotlin()) { @@ -106,11 +107,15 @@ object TokenWriter { } TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector - if (format.isPlaywright()) { - lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];})") - } else if (format.isJavaScript()) { - lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") - lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } + if (format.isJavaScript()) { + if (format.isPlaywright()) { + // In Playwright, use a single-callback then; do not append a separate error handler + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];})") + } else { + // In SuperAgent, support success and error callbacks in the same then(...) + lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") + lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } + } } else if (format.isPython()) { lines.add("${tokenName(k)} = ${responseName(k)}.headers[\"$header\"]") }else if (format.isJavaOrKotlin()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index a95a4d5154..148afcf56f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -349,7 +349,19 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { private fun handleDollarSign(text: String): String{ return text.replace("\$", "\\\$") } - + /** + * Formats a field path according to the active output format. + * + * Rules: + * - Empty input returns an empty string. + * - Java/Kotlin: wrap the path in single quotes and append a dot, + * unless it already starts with a quote. + * - Other formats: return unchanged if starting with '[' or '.'; + * otherwise prefix with a dot. + * + * @param fieldPath raw field path + * @return formatted field path for the target generator + */ private fun formatFieldPath(fieldPath: String): String { if (fieldPath.isEmpty()) { return "" @@ -380,8 +392,9 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isPlaywright() -> "expect(($field)$fieldWithDot).toBe(null);" - format.isJavaScript() -> "expect($field$fieldPath).toBe(null);" // ($field$)fieldPath + format.isJavaScript() -> + if (format.isPlaywright()) "expect(($field)$fieldWithDot).toBe(null);" + else "expect($field$fieldPath).toBe(null);" format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 090947a5b5..49293395c4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -52,10 +52,12 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"application/json\")") - format.isPlaywright() -> { - // Handled in callEndpoint for Playwright + format.isJavaScript() -> { + // Playwright headers/body handled in callEndpoint; set header only for non-Playwright JS + if (!format.isPlaywright()) { + lines.add(".set('Content-Type','application/json')") + } } - format.isJavaScript() -> lines.add(".set('Content-Type','application/json')") format.isPython() -> lines.add("headers[\"content-type\"] = \"application/json\"") // format.isCsharp() -> lines.add("Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(\"application/json\"));") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 1ca68822ca..0371427717 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -80,8 +80,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isPlaywright() -> lines.append("await request") - format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") + format.isJavaScript() -> + if (format.isPlaywright()) { lines.append("await request") + } + else { lines.append("await superagent") + } format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -124,7 +127,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") - format.isJavaScript() || format.isPlaywright() -> lines.append("const $resVarName = ") + format.isJavaScript() -> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") } @@ -214,9 +217,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" - // For Playwright, headers are expressed as entries in an object literal - format.isPlaywright() -> "'Accept': " - format.isJavaScript() -> ".set('Accept', " + format.isJavaScript() -> + if (format.isPlaywright()) "'Accept': " + else ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " else -> throw IllegalArgumentException("Invalid format: $format") @@ -260,8 +263,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" - format.isJavaScript() && !format.isPlaywright()-> "set" - format.isPlaywright() -> "" // headers are handled in a map in the options object + format.isJavaScript() -> "set" format.isPython() -> "headers = {}" else -> throw IllegalArgumentException("Not supported format: $format") } @@ -504,11 +506,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $varName = System.currentTimeMillis()") } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") - } else if(format.isPlaywright()) { - lines.addStatement("const $varName = performance.now()") } else if(format.isJavaScript()) { - lines.addStatement("$varName = performance.now()") - } + if (format.isPlaywright()) { + lines.addStatement("const $varName = performance.now()") + } else { + lines.addStatement("$varName = performance.now()") + } + } lines.addEmpty(1) return varName @@ -525,12 +529,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $finalVarName = System.currentTimeMillis() - $varName") } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") - } else if(format.isPlaywright()) { - lines.addStatement("const $finalVarName = performance.now() - $varName") } else if(format.isJavaScript()) { - lines.addStatement("$finalVarName = performance.now() - $varName") + if (format.isPlaywright()) { lines.addStatement("const $finalVarName = performance.now() - $varName") } + else + { lines.addStatement("$finalVarName = performance.now() - $varName") } } - lines.addEmpty(1) if(isVulnerable) @@ -538,7 +541,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: SQL Injection vulnerability detected in this call. Expected response time (sqliInjectedSleepDurationMs) should be greater than ${config.sqliInjectedSleepDurationMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName > ${config.sqliInjectedSleepDurationMs})") - format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isPython() -> lines.addStatement("assert $finalVarName > ${config.sqliInjectedSleepDurationMs}") else -> {} @@ -547,7 +549,6 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isJavaScript()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") else -> {} @@ -590,6 +591,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.indent(2) if (format.isPlaywright()) { handleVerbEndpoint(baseUrlOfSut, call, lines) + // required to turn await request. (url) into await request. (url, { ... }) lines.replaceInCurrent(Regex("\\)$"), "") lines.append(", {") lines.addEmpty() @@ -648,8 +650,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() && !format.isPlaywright() -> "send" - format.isPlaywright() -> "data" + format.isJavaScript() -> + if (format.isPlaywright()) "data" + else "send" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -843,8 +846,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } } - format.isPlaywright() -> writePlaywrightPayload(lines, bodyLines, true, functionsOnString) - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString) + format.isJavaScript() -> { + if (format.isPlaywright()) { + writePlaywrightPayload(lines, bodyLines, true, functionsOnString) + } else { + writeStringifiedPayload(lines, send, bodyLines, functionsOnString) + } + } else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, functionsOnString) } } @@ -931,7 +939,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { (1 until bodyLines.lastIndex).forEach { i -> lines.add("${bodyLines[i]} + ") } - lines.add("${bodyLines.last()}") + lines.add(bodyLines.last()) } } @@ -1019,20 +1027,20 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escaped = GeneUtils.applyEscapes(value, GeneUtils.EscapeMode.ASSERTION, format) when { format.isJavaOrKotlin() -> ".header(\"$name\", \"$escaped\")" - format.isPlaywright() -> - "expect($responseVariableName.headers()[\"$name\"]?.startsWith(\"$escaped\")).toBe(true);" format.isJavaScript() -> - "expect($responseVariableName.header[\"$name\"].startsWith(\"$escaped\")).toBe(true);" + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"$name\"]?.startsWith(\"$escaped\")).toBe(true);" } + else + { "expect($responseVariableName.header[\"$name\"].startsWith(\"$escaped\")).toBe(true);" } format.isPython() -> "assert \"$escaped\" in $responseVariableName.headers[\"$name\"]" else -> throw IllegalStateException("Unsupported format $format") } } else { when { format.isJavaOrKotlin() -> ".header(\"$name\", isEmptyOrNullString())" - format.isPlaywright() -> - "expect($responseVariableName.headers()[\"$name\"]).toBeUndefined();" format.isJavaScript() -> - "expect($responseVariableName.header[\"$name\"]).toBeUndefined();" + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"$name\"]).toBeUndefined();" } + else + { "expect($responseVariableName.header[\"$name\"]).toBeUndefined();" } format.isPython() -> "assert \"$name\" not in $responseVariableName.headers" else -> throw IllegalStateException("Unsupported format $format") } @@ -1087,10 +1095,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" - format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" format.isJavaScript() -> + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" } + else "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" - format.isCsharp() -> "Assert.Contains(\"$bodyTypeSimplified\", $responseVariableName.Content.Headers.GetValues(\"Content-Type\").First());" format.isPython() -> "assert \"$bodyTypeSimplified\" in $responseVariableName.headers[\"content-type\"]" else -> throw IllegalStateException("Unsupported format $format") @@ -1215,8 +1223,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isPlaywright() -> " ((await $resVarName.json())$dictAccess)?.toString()" - format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" + format.isJavaScript() -> + if (format.isPlaywright()) { "((await $resVarName.json())$dictAccess)?.toString()" } + else "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index fa911ff3c6..fa2d7655f4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -35,10 +35,6 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { companion object { private val log = LoggerFactory.getLogger(RestTestCaseWriter::class.java) } - - // Local extension/helper to avoid depending on OutputFormat.isJsBased from elsewhere - private fun OutputFormat.isJsBasedLocal(): Boolean = this.isJavaScript() || this.isPlaywright() - @Inject private lateinit var partialOracles: PartialOracles @@ -205,7 +201,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJsBasedLocal() -> lines.add("const $name = ") + format.isJavaScript() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -421,13 +417,13 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { + lines.add("const $location = $resVarName") if (format.isPlaywright()) { - lines.add("const $location = $resVarName") lines.append(".headers()['location'];") } else { - lines.add("const $location = $resVarName.header['location'];") - } + lines.append(".header['location'];") + } val validCheck = "${TestSuiteWriter.jsImport}.isValidURIorEmpty($location)" lines.add("expect($validCheck).toBe(true);") } @@ -459,7 +455,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJsBasedLocal() -> lines.add("const ") + format.isJavaScript() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 72b58ad48c..b903c7da80 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -82,9 +82,9 @@ abstract class TestCaseWriter { protected abstract fun addTestCommentBlock(lines: Lines, test: TestCase) /** - * Compute the list of Playwright fixtures to destructure in the test signature. + * Compute Playwright fixtures to destructure in the test signature. * For REST-only generation we currently need only the `request` fixture. - * This helper makes it easy to extend later (e.g., include `page`) without touching call sites. + * This helper can be extended later (e.g., include `page`) by refactoring it to a List. */ private fun playwrightFixturesFor(test: TestCase): String = "request" @@ -132,11 +132,10 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isPlaywright() -> { - val fixtures = playwrightFixturesFor(test) - lines.add("test(\"${test.name}\", async ({ $fixtures }) => {") - } - format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() -> + if (format.isPlaywright()) {val fixtures = playwrightFixturesFor(test) + lines.add("test(\"${test.name}\", async ({ $fixtures }) => {")} + else lines.add("test(\"${test.name}\", async () => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 42deba1947..55ef78ae27 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -542,9 +542,6 @@ class TestSuiteWriter { if (format.isJavaScript()) { lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';") if (format.isPlaywright()) { - // Guard: avoid executing Playwright tests when Jest is running in the same folder - lines.add("const isJest = typeof process !== 'undefined' && process.env && process.env.JEST_WORKER_ID;") - lines.add("if (!isJest) {") lines.add("const { test, expect } = require(\"@playwright/test\");") } else { lines.add("const superagent = require(\"superagent\");") @@ -1139,14 +1136,6 @@ class TestSuiteWriter { private fun footer(lines: Lines) { - // Close Playwright guard if it was opened; add a Jest placeholder test in the else branch - if (config.outputFormat.isPlaywright()) { - lines.add("} else {") - lines.indented { - lines.add("test('placeholder - Playwright suite skipped under Jest', () => { expect(true).toBe(true); });") - } - lines.add("}") - } if (config.outputFormat.isJavaOrKotlin() || config.outputFormat.isCsharp()) { //due to opening of class lines.addEmpty(2)