diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt index a3577df23b..31a5e277cb 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt @@ -36,7 +36,6 @@ object BlackBoxUtils { private fun mvn() = if (isWindows()) "mvn.cmd" else "mvn" - private fun runNpmInstall() { val command = listOf(npm(), "ci") @@ -106,7 +105,7 @@ object BlackBoxUtils { } } - fun runNpmTests(folderRelativePath: String) { + fun runNpmTests(folderRelativePath: String, isPlaywright: Boolean = false) { runNpmInstall() val path = if(folderRelativePath.endsWith("/")){ @@ -116,8 +115,12 @@ object BlackBoxUtils { "$folderRelativePath/" } - val command = listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") - runTestsCommand(command, JS_BASE_PATH, "NPM") + val command = if (isPlaywright) { + listOf(npm(), "run", "test:playwright", "--", path) + } else { + listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") + } + runTestsCommand(command, JS_BASE_PATH, if (isPlaywright) "Playwright" else "NPM") } fun runPythonTests(folderRelativePath: String) { @@ -151,4 +154,4 @@ object BlackBoxUtils { fun getOutputFilePrefixKotlin(outputFolderName: String) = "com.kotlin.$outputFolderName.EM" -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index 30291759dc..d99e3079e5 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -100,7 +100,12 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + outputFormat.isJavaScript() -> + if (outputFormat.isPlaywright()) { + BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) + } else { + BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + } outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") } diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json index c85a0aca99..4333fffe1d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json @@ -7,6 +7,7 @@ "name": "evomaster-client-js-e2e-tests", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", @@ -933,6 +934,21 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -3248,6 +3264,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -4567,6 +4627,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "requires": { + "playwright": "1.59.1" + } + }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -6292,6 +6361,31 @@ "find-up": "^4.0.0" } }, + "playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.59.1" + }, + "dependencies": { + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + } + } + }, + "playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true + }, "pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index 5c69b06726..153d9a34f2 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -6,16 +6,19 @@ "author": "EvoMaster Team", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", "urijs": "1.19.6" }, "scripts": { - "test": "jest" + "test": "jest", + "test:playwright": "playwright test" }, "jest": { "testEnvironment": "node", - "testRegex": "generated/.*/.*[tT]est\\.js$" + "testRegex": "generated/.*/.*[tT]est\\.js$", + "testPathIgnorePatterns": [".*_pw\\.js$"] } } diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js new file mode 100644 index 0000000000..ce35c56d60 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js @@ -0,0 +1,6 @@ +const { defineConfig } = require('@playwright/test'); + +module.exports = defineConfig({ + testDir: './generated', + testMatch: /.*[tT]est\.js/, +}); diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml index 86e7552fb2..c590281551 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml @@ -11,7 +11,7 @@ --> org.evomaster - evomaster-e2e-tests + evomaster-e2e-tests-spring 6.1.2-SNAPSHOT ../../pom.xml diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index 2ed918c6bb..8b3975fa01 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -150,7 +150,6 @@ abstract class SpringTestBase : RestTestBase() { } } - fun runBlackBoxEM( outputFormat: OutputFormat, outputFolderName: String, @@ -172,7 +171,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ - outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) + outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), outputFormat.isPlaywright()) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) outputFormat.isKotlin() -> BlackBoxUtils.runKotlinTests(outputFolderName) @@ -218,6 +217,9 @@ abstract class SpringTestBase : RestTestBase() { if(outputFormat.isKotlin()){ setOption(args,"outputFilePrefix",BlackBoxUtils.getOutputFilePrefixKotlin(outputFolderName)) } + if (outputFormat.isPlaywright()) { + setOption(args, "outputFilePrefix", "EvoMaster_pw") + } defaultSeed++ lambda.accept(ArrayList(args)) @@ -230,4 +232,4 @@ abstract class SpringTestBase : RestTestBase() { } } -} +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index c5aeb96248..d46f77115a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,6 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, + JS_PLAYWRIGHT, //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST ; @@ -28,6 +29,8 @@ enum class OutputFormat { fun isJavaScript() = this.name.startsWith("js_", true) + fun isPlaywright() = this.name.endsWith("_playwright", true) + fun isJavaOrKotlin() = isJava() || isKotlin() fun isJUnit5() = this.name.endsWith("junit_5", true) @@ -40,5 +43,4 @@ enum class OutputFormat { fun isCsharp() = this.name.startsWith("csharp",ignoreCase = true) fun isPython() = this.name.startsWith("python_", true) - } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt index a01fe19178..69fda3eefa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/AuthWriter.kt @@ -31,6 +31,12 @@ object AuthWriter { placeHolderResolver: PlaceHolderResolver? ) { + // Playwright specific handling + if (format.isPlaywright()) { + addPlaywrightAuthCall(lines, k, testCaseWriter, format, baseUrlOfSut, placeHolderResolver) + return + } + if(format.isJavaScript()) { callEndpoint(lines, k, format, baseUrlOfSut) } @@ -49,10 +55,7 @@ object AuthWriter { } } - val replacements = placeHolderResolver?.placeHolders?.entries?.map { - val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) - ".replace(\"${placeholder}\", ${it.value})" - } + val replacements = buildReplacements(format, placeHolderResolver) when (contentType) { ContentType.X_WWW_FORM_URLENCODED -> { @@ -97,8 +100,8 @@ object AuthWriter { } } - if (format.isJavaScript()){ - // disable redirections + // Disable redirections where supported (SuperAgent/Frisby style only) + if (format.isJavaScript() && !format.isPlaywright()){ lines.add(".redirects(0)") } @@ -126,11 +129,101 @@ object AuthWriter { } } - private fun callEndpoint( + /** + * Emit Playwright request call for authentication endpoints, including headers, body and options. + */ + private fun addPlaywrightAuthCall( lines: Lines, k: CallToEndpoint, + testCaseWriter: HttpWsTestCaseWriter, format: OutputFormat, - baseUrlOfSut: String + baseUrlOfSut: String, + placeHolderResolver: PlaceHolderResolver? + ) { + // Compute placeholder replacements for Playwright (used for dynamic user data) + val replacements: List? = buildReplacements(format, placeHolderResolver) + + // Hoist contentType once and reuse below (headers + body) + val contentType = k.contentType + + // Emit verb and URL, leave call open to add Playwright options object + emitVerbAndUrl(lines, k, format, baseUrlOfSut, close = false) + lines.append(", {") + lines.addEmpty() + lines.indented { + emitPlaywrightHeaders(lines, k, contentType) + emitPlaywrightBody(lines, k, testCaseWriter, replacements, contentType) + emitPlaywrightRequestOptions(lines) + } + lines.add("})") + } + + /** + * Emit the Playwright headers block, including optional content-type and any custom headers. + */ + private fun emitPlaywrightHeaders( + lines: Lines, + k: CallToEndpoint, + contentType: ContentType? + ) { + lines.add("headers: {") + lines.indented { + if (contentType != null) { + lines.add("'content-type': \"${contentType.defaultValue}\",") + } + for (header in k.headers) { + lines.add("'${header.name}': \"${header.value}\",") + } + } + lines.add("},") + } + + /** + * Emit the Playwright body/payload block according to the content type. + */ + private fun emitPlaywrightBody( + lines: Lines, + k: CallToEndpoint, + testCaseWriter: HttpWsTestCaseWriter, + replacements: List?, + contentType: ContentType? + ) { + if (contentType == null) return + when (contentType) { + ContentType.X_WWW_FORM_URLENCODED -> { + if (replacements == null) { + lines.add("data: \"${k.payload}\",") + } else { + lines.add("data: \"${k.payload}\"") + replacements.forEach { lines.append(it) } + lines.append(",") + } + } + ContentType.JSON -> { + testCaseWriter.printSendJsonBody(k.payload!!, lines, functionsOnString = replacements) + } + else -> throw IllegalStateException("Currently not supporting yet ${k.contentType} in login") + } + } + + /** + * Emit standard Playwright request options for auth requests. + */ + private fun emitPlaywrightRequestOptions(lines: Lines) { + lines.add("maxRedirects: 0,") + lines.add("ignoreHTTPSErrors: true,") + } + + /** + * Emit the HTTP verb and URL for the call, with an option to close the parenthesis. + * Example (Java/JS): `.post(baseUrl + "/login"` and closes with `)` if `close=true`. + */ + private fun emitVerbAndUrl( + lines: Lines, + k: CallToEndpoint, + format: OutputFormat, + baseUrlOfSut: String, + close: Boolean = false ) { val verb = k.verb.name.lowercase() lines.add(".$verb(") @@ -144,8 +237,30 @@ object AuthWriter { } lines.append("${k.endpoint}\"") } - if (!format.isPython()) { + if (close) { lines.append(")") } } + + /** + * Build a list of string replacement functions to apply to payload strings. + * Each entry is a snippet like: `.replace("", value)`; returns null if no placeholders. + */ + private fun buildReplacements( + format: OutputFormat, + placeHolderResolver: PlaceHolderResolver? + ): List? = placeHolderResolver?.placeHolders?.entries?.map { + val placeholder = GeneUtils.applyEscapes(it.key, mode = GeneUtils.EscapeMode.BODY, format) + ".replace(\"${placeholder}\", ${it.value})" + } + + private fun callEndpoint( + lines: Lines, + k: CallToEndpoint, + format: OutputFormat, + baseUrlOfSut: String + ) { + // Delegate to the shared emitter; Python keeps it open, others close it + emitVerbAndUrl(lines, k, format, baseUrlOfSut, close = !format.isPython()) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index 8648fe2481..27eaaf47b1 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -51,7 +51,12 @@ object CookieWriter { when { format.isJava() -> lines.add("final Map ${cookiesName(k)} = ") format.isKotlin() -> lines.add("val ${cookiesName(k)} : Map = ") - format.isJavaScript() -> lines.add("const ${cookiesName(k)} = ") + format.isJavaScript() -> when { + format.isPlaywright() -> + lines.add("let ${cookiesName(k)};") + else -> + lines.add("const ${cookiesName(k)} = ") + } } if (!format.isPython()) { @@ -77,9 +82,31 @@ object CookieWriter { format.isPython() -> lines.append(".cookies") } - if(format.isJavaScript()){ - lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") - lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") + if (format.isJavaScript()) { + + // SuperAgent/Playwright cookie extraction + if (format.isPlaywright()) { + lines.add(".then((res) => {") + lines.indented { + lines.add("const setCookie = res.headers()['set-cookie'];") + lines.add("if (setCookie) {") + lines.indented { + lines.add("return setCookie.split('\\n')[0].split(';')[0];") + } + lines.add("}") + lines.add("return null;") + } + lines.add("})") + // Playwright cookie extraction must NOT assume res is a response object + lines.add(".then(async (cookie) => {") + lines.indented { + lines.add("${cookiesName(k)} = cookie;") + } + lines.add("})") + } else { + lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") + lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") + } lines.appendSemicolon() } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt index 9efc536170..396b960e2a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CreateUsersWriter.kt @@ -106,7 +106,7 @@ object CreateUsersWriter { lines.add(".then()") lines.add(".statusCode(both(greaterThanOrEqualTo(200)).and(Matchers.lessThan(400)))") } - if(format.isJavaScript()){ + if(format.isJavaScript() && !format.isPlaywright()){ lines.add(".ok(res => res.status >= 200 && res.status < 400)") } lines.appendSemicolon() diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index 5192549ca3..1d8d00caf2 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -54,16 +54,19 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") - format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") + format.isJavaScript() && format.isPlaywright() -> lines.add("let ${tokenName(k)};") + format.isJavaScript() && !format.isPlaywright()-> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") format.isJavaScript() -> { - lines.append("\"\"") - lines.appendSemicolon() - lines.addEmpty() - lines.append("await superagent") + if (format.isPlaywright()) { + testCaseWriter.startRequest(lines) //handled in HttpWsTestCaseWriter with startRequest(lines) and await request + } else {lines.append("\"\"") + lines.appendSemicolon() + lines.addEmpty() + lines.append("await superagent")} } } @@ -88,8 +91,10 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { if (format.isJavaScript()) { - lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") - lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;})") + } else {lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") + lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") }} } else if (format.isPython()) { lines.add("${tokenName(k)} = ${responseName(k)}.json()$path") }else if (format.isJavaOrKotlin()) { @@ -103,8 +108,14 @@ object TokenWriter { TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector if (format.isJavaScript()) { - lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") - lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } + if (format.isPlaywright()) { + // In Playwright, use a single-callback then; do not append a separate error handler + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];})") + } else { + // In SuperAgent, support success and error callbacks in the same then(...) + lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") + lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } + } } else if (format.isPython()) { lines.add("${tokenName(k)} = ${responseName(k)}.headers[\"$header\"]") }else if (format.isJavaOrKotlin()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index 85287f4f40..148afcf56f 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -349,13 +349,52 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { private fun handleDollarSign(text: String): String{ return text.replace("\$", "\\\$") } + /** + * Formats a field path according to the active output format. + * + * Rules: + * - Empty input returns an empty string. + * - Java/Kotlin: wrap the path in single quotes and append a dot, + * unless it already starts with a quote. + * - Other formats: return unchanged if starting with '[' or '.'; + * otherwise prefix with a dot. + * + * @param fieldPath raw field path + * @return formatted field path for the target generator + */ + private fun formatFieldPath(fieldPath: String): String { + if (fieldPath.isEmpty()) { + return "" + } + if (format.isJavaOrKotlin()) { + return if (fieldPath.startsWith("'")) "$fieldPath." else "'$fieldPath'." + } + return if (fieldPath.startsWith("[") || fieldPath.startsWith(".")) { + fieldPath + } else { + ".$fieldPath" + } + } private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { + val field = when { + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } + else -> "" + } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isJavaScript() -> "expect($responseVariableName.body$fieldPath).toBe(null);" + format.isJavaScript() -> + if (format.isPlaywright()) "expect(($field)$fieldWithDot).toBe(null);" + else "expect($field$fieldPath).toBe(null);" format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") @@ -409,10 +448,24 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { + val field = when { + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } + else -> "" + } + + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" + }else if (format.isPlaywright()){ // playwright + "expect($field$fieldWithDot).toBe($toPrint);" }else { // javascript - "expect($responseVariableName.body$fieldPath).toBe($toPrint);" + "expect($field$fieldPath).toBe($toPrint);" // ($field$)fieldPath } if (flakyValue == null || flakyValue == value){ @@ -560,6 +613,9 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { + if (format.isPlaywright()) { + return "expect(await $responseVariableName.text()).toBe(\"\");" + } /* This is super ugly... but there is no clean solution for this in Jest nor SuperAgent... :( @@ -595,8 +651,15 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val path = if (fieldPath.isEmpty()) "" else "$fieldPath." ".body(\"${path}size()\", equalTo($expectedSize))" } - format.isJavaScript() -> - "expect($responseVariableName.body$fieldPath.length).toBe($expectedSize);" + format.isJavaScript() -> { + if (format.isPlaywright()) { + val field = formatFieldPath(fieldPath) + "expect((await $responseVariableName.json())$field).toHaveLength($expectedSize);" + } else { + val field = "$responseVariableName.body" + "expect($field$fieldPath.length).toBe($expectedSize);" // ($field$)fieldPath + } + } format.isCsharp() -> "Assert.True($responseVariableName$fieldPath.Count == $expectedSize);" format.isPython() -> @@ -616,11 +679,13 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return ".body(containsString(\"$content\"))" } - if (format.isJavaScript()) { - return "expect($responseVariableName.text).toContain(\"$content\");" - + if (format.isPlaywright()) { + return "expect(await $responseVariableName.text()).toContain(\"$content\");" } + if (format.isJavaScript()) { + return "expect($responseVariableName.text).toContain(\"$content\");" } + if (format.isCsharp()) { val k = when { content.startsWith("\"") -> content.substring(1, content.length - 1) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 30962d25e7..49293395c4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -52,7 +52,12 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"application/json\")") - format.isJavaScript() -> lines.add(".set('Content-Type','application/json')") + format.isJavaScript() -> { + // Playwright headers/body handled in callEndpoint; set header only for non-Playwright JS + if (!format.isPlaywright()) { + lines.add(".set('Content-Type','application/json')") + } + } format.isPython() -> lines.add("headers[\"content-type\"] = \"application/json\"") // format.isCsharp() -> lines.add("Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(\"application/json\"));") } @@ -95,10 +100,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } override fun handleVerbEndpoint(baseUrlOfSut: String, _call: HttpWsAction, lines: Lines) { - // TODO maybe in future might want to have GET for QUERY types val verb = "post" - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("await request.$verb(") + } else { + lines.add(".$verb(") + } if(config.blackBox){ /* @@ -131,6 +140,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } } + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + lines.add("maxRedirects: 0,") + } + lines.add("}") + } + lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index 8da5baaefd..0371427717 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -73,14 +73,18 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT } } - protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && hasTimeoutFault(res) + protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && !format.isPlaywright() && hasTimeoutFault(res) protected fun expectsAssertRaises(res: ActionResult) = format.isPython() && hasTimeoutFault(res) fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + format.isJavaScript() -> + if (format.isPlaywright()) { lines.append("await request") + } + else { lines.append("await superagent") + } format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -132,7 +136,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") // for a call expected to reject, wrap it in await expect(...).rejects.toThrow() - format.isJavaScript() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") + // For Playwright, the verb handler will write `await request.(...)` + format.isJavaScript() && !format.isPlaywright() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -212,7 +217,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" - format.isJavaScript() -> ".set('Accept', " + format.isJavaScript() -> + if (format.isPlaywright()) "'Accept': " + else ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " else -> throw IllegalArgumentException("Invalid format: $format") @@ -221,12 +228,15 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun closeAcceptHeader(openedHeader: String): String { var result = openedHeader - if (!config.outputFormat.isPython()) { + // Do not append a closing parenthesis for Playwright as we are not using a method call + if (!config.outputFormat.isPython() && !format.isPlaywright()) { result += ")" } if (format.isCsharp()){ result = "$result;" } + if (format.isPlaywright()) { result += "," + } return result } @@ -237,7 +247,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { but that is not the case for the other libraries used for example in JS and C# */ return config.enableBasicAssertions && - (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.PYTHON_UNITTEST) + (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.JS_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) } protected fun handleHeaders(call: HttpWsAction, lines: Lines) { @@ -262,10 +272,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add(set) } + val bodyParam = call.parameters.find { p -> p is BodyParam } as BodyParam? + + if (format.isPlaywright() && bodyParam != null) { + val contentType = bodyParam.contentType() + if (contentType != null) { + lines.add("'Content-Type': '$contentType',") + } + } + //headers in specified auth info call.auth.headers.forEach { if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${it.value}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${it.value}', // ${call.auth.name}") } else { lines.add(".$set(\"${it.name}\", \"${it.value}\") // ${call.auth.name}") } @@ -283,6 +304,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escapedHeader = GeneUtils.applyEscapes(x, GeneUtils.EscapeMode.BODY, format) if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${escapedHeader}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${escapedHeader}',") } else { lines.add(".$set(\"${it.name}\", \"${escapedHeader}\")") @@ -296,13 +319,22 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val tokenHeader = elc.token!!.sendName if (format.isPython()) { lines.add("headers[\"$tokenHeader\"] = ${TokenWriter.authPayloadName(elc)} # ${call.auth.name}") + } else if (format.isPlaywright()) { + lines.add("'$tokenHeader': ${TokenWriter.authPayloadName(elc)}, // ${call.auth.name}") } else { lines.add(".$set(\"$tokenHeader\", ${TokenWriter.authPayloadName(elc)}) // ${call.auth.name}") } } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isJavaScript() -> when { + format.isPlaywright() -> { + val cookieVar = CookieWriter.cookiesName(elc) + lines.add("'Cookie': $cookieVar,") + } + else -> + lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + } // Python cookies are set alongside the headers and body when performing the request } } @@ -329,7 +361,19 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val code = res.getStatusCode() when { - format.isJavaScript() -> { + + format.isJavaScript() && format.isPlaywright() -> { + val statusAssert = "expect($responseVariableName.status()).toBe($code);" + if (res.getFlakyStatusCode() == null){ + lines.add(statusAssert) + }else{ + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(statusAssert) + } + lines.addEmpty() + } + + format.isJavaScript() && !format.isPlaywright() -> { val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -463,8 +507,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") } else if(format.isJavaScript()) { - lines.addStatement("$varName = performance.now()") - } + if (format.isPlaywright()) { + lines.addStatement("const $varName = performance.now()") + } else { + lines.addStatement("$varName = performance.now()") + } + } lines.addEmpty(1) return varName @@ -482,9 +530,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") } else if(format.isJavaScript()) { - lines.addStatement("$finalVarName = performance.now() - $varName") + if (format.isPlaywright()) { lines.addStatement("const $finalVarName = performance.now() - $varName") } + else + { lines.addStatement("$finalVarName = performance.now() - $varName") } } - lines.addEmpty(1) if(isVulnerable) @@ -500,7 +549,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") else -> {} } @@ -540,6 +589,28 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isJavaScript() -> { lines.indent(2) + if (format.isPlaywright()) { + handleVerbEndpoint(baseUrlOfSut, call, lines) + // required to turn await request. (url) into await request. (url, { ... }) + lines.replaceInCurrent(Regex("\\)$"), "") + lines.append(", {") + lines.addEmpty() + lines.indented { + if (call is org.evomaster.core.problem.rest.data.RestCallAction) { + lines.add("method: \"${call.verb.name.uppercase()}\",") + } + lines.add("headers: {") + lines.indented { + lines.add(getAcceptHeader(call, res)) + handleHeaders(call, lines) + } + lines.add("},") + handleBody(call, lines, dtoVar) + lines.add("maxRedirects: 0,") + lines.add("ignoreHTTPSErrors: true,") + } + lines.add("})") + } else { //in SuperAgent, verb must be first handleVerbEndpoint(baseUrlOfSut, call, lines) //client timeout, same source as fuzzing tcpTimeoutMs @@ -548,6 +619,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { handleHeaders(call, lines) handleBody(call, lines) } + } format.isCsharp() -> { handleVerbEndpoint(baseUrlOfSut, call, lines) @@ -578,7 +650,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() -> "send" + format.isJavaScript() -> + if (format.isPlaywright()) "data" + else "send" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -608,7 +682,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isPlaywright() -> { + // handled in makeHttpCall through options object + } format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -635,6 +712,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$body\"") } + format.isPlaywright() -> { + // In Playwright, use the options object field instead of chained calls + lines.add("data: \"$body\",") + } else -> lines.add(".$send(\"$body\")") } } else if (bodyParam.isXml()) { @@ -647,7 +728,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$escapedXml\"") } - + format.isPlaywright() -> { + lines.add("data: \"$escapedXml\",") + } else -> lines.add(".$send(\"$escapedXml\")") } } else { @@ -679,7 +762,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = \"$text\"") } } - + format.isPlaywright() -> { + // In Playwright, body data must be inside the request options object + lines.add("data: \"$text\",") + } else -> lines.add(".$send(\"$text\")") } } else { @@ -688,6 +774,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = \"\"") } //TODO isn't this valid just for Kotlin??? + format.isPlaywright() -> { + // Empty body for Playwright should still be provided in options when needed + lines.add("data: \"\",") + } else -> lines.add(".$send(\"${"""\"\""""}\")") } } @@ -711,6 +801,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { return } + // For Playwright, if a DTO variable is available and allowed, send it directly in the options object + if (format.isPlaywright() && shouldUseDtoForPayload(dtoVar)) { + lines.add("data: ${dtoVar},") + return + } + if(dtoVar != null && functionsOnString != null) { throw IllegalArgumentException("Cannot use extra functions on string JSON when using DTOs") } @@ -725,6 +821,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = ${bodyLines.first()}") functionsOnString?.forEach { lines.append(it) } } + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, false, functionsOnString) + } format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString) else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, functionsOnString) } @@ -747,7 +846,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, functionsOnString) + format.isJavaScript() -> { + if (format.isPlaywright()) { + writePlaywrightPayload(lines, bodyLines, true, functionsOnString) + } else { + writeStringifiedPayload(lines, send, bodyLines, functionsOnString) + } + } else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, functionsOnString) } } @@ -818,6 +923,48 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(")") } + private fun writePlaywrightPayload( + lines: Lines, + bodyLines: List, + isMultiLine: Boolean, + functionsOnString: List? = null + ) { + // Build the JSON string expression first + lines.add("data: JSON.parse(${bodyLines.first()}") + + // If multi-line, concatenate subsequent lines + if (isMultiLine) { + lines.append(" + ") + lines.indented { + (1 until bodyLines.lastIndex).forEach { i -> + lines.add("${bodyLines[i]} + ") + } + lines.add(bodyLines.last()) + } + } + + // Apply any string transformation functions (e.g., .replace(...)) + functionsOnString?.let { funcs -> + if (funcs.isNotEmpty()) { + // If we built a multi-line concatenation, ensure method calls are appended correctly + if (!isMultiLine) { + // single-line: can simply append functions inline + funcs.forEach { lines.append(it) } + } else { + // multi-line: add functions each on its own line for readability + lines.indented { + funcs.forEach { lines.add(it) } + } + } + } + } + + // Close JSON.parse(...), then the options object entry + lines.append(")", + ) + lines.append(",") + } + /** * This is done mainly for RestAssured */ @@ -837,6 +984,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } + format.isPlaywright() -> { + // assertions for Playwright are handled in handleResponseAfterTheCall, + // as they cannot be chained directly in the request call + } + else -> throw IllegalStateException("No assertion in calls for format: $format") } @@ -876,7 +1028,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> ".header(\"$name\", \"$escaped\")" format.isJavaScript() -> - "expect($responseVariableName.header[\"$name\"].startsWith(\"$escaped\")).toBe(true);" + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"$name\"]?.startsWith(\"$escaped\")).toBe(true);" } + else + { "expect($responseVariableName.header[\"$name\"].startsWith(\"$escaped\")).toBe(true);" } format.isPython() -> "assert \"$escaped\" in $responseVariableName.headers[\"$name\"]" else -> throw IllegalStateException("Unsupported format $format") } @@ -884,7 +1038,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> ".header(\"$name\", isEmptyOrNullString())" format.isJavaScript() -> - "expect($responseVariableName.header[\"$name\"]).toBeUndefined();" + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"$name\"]).toBeUndefined();" } + else + { "expect($responseVariableName.header[\"$name\"]).toBeUndefined();" } format.isPython() -> "assert \"$name\" not in $responseVariableName.headers" else -> throw IllegalStateException("Unsupported format $format") } @@ -899,7 +1055,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleResponseAssertions(lines: Lines, res: HttpWsCallResult, responseVariableName: String?) { - assert(responseVariableName != null || format.isJavaOrKotlin()) + assert(responseVariableName != null || format.isJavaOrKotlin() || format.isPlaywright()) /* there are 2 cases: @@ -940,8 +1096,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" format.isJavaScript() -> + if (format.isPlaywright()) { "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" } + else "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" - format.isCsharp() -> "Assert.Contains(\"$bodyTypeSimplified\", $responseVariableName.Content.Headers.GetValues(\"Content-Type\").First());" format.isPython() -> "assert \"$bodyTypeSimplified\" in $responseVariableName.headers[\"content-type\"]" else -> throw IllegalStateException("Unsupported format $format") @@ -1004,8 +1161,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - - if (format.isJavaScript()) { + if (format.isJavaScript() && !format.isPlaywright()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... @@ -1063,10 +1219,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } val jsonPath = JsonUtils.fromPointerToPath(jsonPointer) + val dictAccess = JsonUtils.fromPointerToDictionaryAccess(jsonPointer) return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" - format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" + format.isJavaScript() -> + if (format.isPlaywright()) { "((await $resVarName.json())$dictAccess)?.toString()" } + else "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") } @@ -1134,4 +1293,4 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } -} +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 0c6e8ee7a0..fa2d7655f4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines import org.evomaster.core.output.SqlWriter import org.evomaster.core.output.TestCase import org.evomaster.core.output.TestWriterUtils +import org.evomaster.core.output.OutputFormat import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.enterprise.EnterpriseActionResult import org.evomaster.core.problem.httpws.HttpWsAction @@ -34,7 +35,6 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { companion object { private val log = LoggerFactory.getLogger(RestTestCaseWriter::class.java) } - @Inject private lateinit var partialOracles: PartialOracles @@ -217,14 +217,26 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val call = _call as RestCallAction val verb = call.verb.name.lowercase() - if (format.isCsharp()) { - lines.append(".${StringUtils.capitalization(verb)}Async(") - } else { - if (verb == "trace" && format.isJavaOrKotlin()) { - //currently, RestAssured does not have a trace() method - lines.add(".request(io.restassured.http.Method.TRACE, ") - } else { - lines.add(".$verb(") + when { + format.isPlaywright() -> { + val verbToUse = call.verb.name.lowercase() + // Playwright provides direct helpers for common verbs. For the rest, use fetch with explicit method. + val hasDirectHelper = setOf("get", "post", "put", "patch", "delete", "head").contains(verbToUse) + if (!hasDirectHelper) { + lines.add("await request.fetch(") + } else { + lines.add("await request.$verbToUse(") + } + // Note: the options object is appended in HttpWsTestCaseWriter.makeHttpCall. + } + format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") + else -> { + if (verb == "trace" && format.isJavaOrKotlin()) { + //currently, RestAssured does not have a trace() method + lines.add(".request(io.restassured.http.Method.TRACE, ") + } else { + lines.add(".$verb(") + } } } @@ -405,7 +417,13 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { - lines.add("const $location = $resVarName.header['location'];") + lines.add("const $location = $resVarName") + + if (format.isPlaywright()) { + lines.append(".headers()['location'];") + } else { + lines.append(".header['location'];") + } val validCheck = "${TestSuiteWriter.jsImport}.isValidURIorEmpty($location)" lines.add("expect($validCheck).toBe(true);") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7f856335f1..b903c7da80 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -81,6 +81,13 @@ abstract class TestCaseWriter { protected abstract fun addTestCommentBlock(lines: Lines, test: TestCase) + /** + * Compute Playwright fixtures to destructure in the test signature. + * For REST-only generation we currently need only the `request` fixture. + * This helper can be extended later (e.g., include `page`) by refactoring it to a List. + */ + private fun playwrightFixturesFor(test: TestCase): String = "request" + fun convertToCompilableTestCode( test: TestCase, baseUrlOfSut: String, @@ -125,7 +132,10 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() -> + if (format.isPlaywright()) {val fixtures = playwrightFixturesFor(test) + lines.add("test(\"${test.name}\", async ({ $fixtures }) => {")} + else lines.add("test(\"${test.name}\", async () => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } @@ -318,6 +328,12 @@ abstract class TestCaseWriter { testSuitePath: Path?, baseUrlOfSut: String ) { + val playwrightExpectException = format.isPlaywright() && shouldFailIfExceptionNotThrown(res) + val hasThrownVar = if (playwrightExpectException) "hasThrown_${counter++}" else "" + + if (playwrightExpectException) { + lines.add("let $hasThrownVar = false;") + } when { /* TODO do we need to handle differently in JS due to Promises? @@ -389,6 +405,12 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } + if (playwrightExpectException) { + lines.indented { + lines.add("$hasThrownVar = true;") + } + } + lines.indented { res.getErrorMessage()?.let { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") @@ -401,6 +423,9 @@ abstract class TestCaseWriter { if (!format.isPython()) { lines.add("}") } + if (playwrightExpectException) { + lines.add("expect($hasThrownVar).toBe(true);") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 71ccd4c999..55ef78ae27 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -541,9 +541,13 @@ class TestSuiteWriter { if (format.isJavaScript()) { lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';") + if (format.isPlaywright()) { + lines.add("const { test, expect } = require(\"@playwright/test\");") + } else { lines.add("const superagent = require(\"superagent\");") // HTTP client timeout (ms) lines.add("const $httpTimeoutVarMs = ${config.tcpTimeoutMs};") + } val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -552,7 +556,8 @@ class TestSuiteWriter { if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") } - if (config.testTimeout > 0) { + // Playwright has its own test runner and does not use Jest + if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") } } @@ -787,6 +792,13 @@ class TestSuiteWriter { val format = config.outputFormat + // For Playwright, avoid emitting a top-level test.beforeAll hook to prevent + // Playwright runner errors when files are imported indirectly. Also, for + // black-box scenarios the hook would be empty anyway. + if (format.isPlaywright()) { + return + } + when { format.isJUnit4() -> lines.add("@BeforeClass") format.isJUnit5() -> lines.add("@BeforeAll") @@ -797,7 +809,9 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() -> lines.add("beforeAll( async () =>") + format.isJavaScript() -> { + lines.add("beforeAll( async () =>") + } } lines.block { @@ -964,7 +978,13 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") + format.isJavaScript() -> { + if (format.isPlaywright()) { + lines.add("test.afterAll( async () =>") + } else { + lines.add("afterAll( async () =>") + } + } } if (!format.isCsharp()) { @@ -1017,7 +1037,13 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") + format.isJavaScript() -> { + if (format.isPlaywright()) { + lines.add("test.beforeEach(async () => ") + } else { + lines.add("beforeEach(async () => ") + } + } //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } diff --git a/docs/options.md b/docs/options.md index f51c573d3f..ad6a6ff083 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.|