From 3a1eec0b4ab0d9cb719b760ae3d881c55d4f1b05 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 21 Jul 2026 09:31:08 +0200 Subject: [PATCH 1/2] fix: avoid retrying untracked karate scenarios --- .../KarateExecutionInstrumentation.java | 8 + .../karate/KarateTracingHook.java | 2 +- .../src/test/groovy/KarateTest.groovy | 13 + .../TestSucceedCalledFeatureKarate.java | 11 + .../org/example/test_called_feature.feature | 4 + .../test_called_feature_nested.feature | 4 + .../test-efd-called-feature/coverages.ftl | 1 + .../test-efd-called-feature/events.ftl | 192 +++++++ .../resources/test-efd-setup/coverages.ftl | 1 + .../test/resources/test-efd-setup/events.ftl | 527 ++++++++++++++++++ .../karate2/KarateScenarioAdvice.java | 8 + .../karate2/KarateTracingListener.java | 5 +- .../src/test/groovy/KarateV2Test.groovy | 2 + .../java/KarateStandaloneScenarioTest.java | 23 + .../TestSucceedCalledFeatureKarate.java | 16 + .../org/example/test_called_feature.feature | 4 + .../test_called_feature_nested.feature | 4 + .../test-efd-called-feature/coverages.ftl | 1 + .../test-efd-called-feature/events.ftl | 192 +++++++ .../resources/test-efd-setup/coverages.ftl | 1 + .../test/resources/test-efd-setup/events.ftl | 521 +++++++++++++++++ 21 files changed, 1537 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature.feature create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature_nested.feature create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/events.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/events.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/KarateStandaloneScenarioTest.java create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature.feature create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature_nested.feature create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/events.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/events.ftl diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java index e218b5b60cc..1a1b030fac1 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java @@ -76,6 +76,10 @@ public void methodAdvice(MethodTransformer transformer) { public static class RetryAdvice { @Advice.OnMethodEnter public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) { + if (KarateTracingHook.skipTracking(scenarioRuntime)) { + return; + } + ExecutionContext executionContext = InstrumentationContext.get(Scenario.class, ExecutionContext.class) .computeIfAbsent(scenarioRuntime.scenario, ExecutionContext::create); @@ -91,6 +95,10 @@ public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) { @Advice.OnMethodExit public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) { + if (KarateTracingHook.skipTracking(scenarioRuntime)) { + return; + } + if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) { // nested call return; diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java index 542d82ef641..4542e816c92 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java @@ -233,7 +233,7 @@ private static boolean skipTracking(FeatureRuntime fr) { return !fr.caller.isNone(); } - private static boolean skipTracking(ScenarioRuntime sr) { + public static boolean skipTracking(ScenarioRuntime sr) { // do not track nested scenario calls and setup scenarios return !sr.caller.isNone() || sr.tags.getTagKeys().contains("setup"); } diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy index 442222a9963..ea4285a8331 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy @@ -103,6 +103,19 @@ class KarateTest extends CiVisibilityInstrumentationTest { "test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] // is executed only twice "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] "test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | [] + "test-efd-called-feature" | [TestSucceedCalledFeatureKarate] | [new TestFQN("[org/example/test_called_feature] test called feature", "caller")] + } + + def "test early flakiness detection ignores setup scenarios"() { + given: + Assumptions.assumeTrue(KarateUtils.isSetupTagSupported(KarateUtils.getKarateVersion())) + + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + runTests([TestWithSetupKarate]) + + assertSpansData("test-efd-setup") } def "test quarantined #testcaseName"() { diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java new file mode 100644 index 00000000000..f0db64756aa --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java @@ -0,0 +1,11 @@ +package org.example; + +import com.intuit.karate.junit5.Karate; + +public class TestSucceedCalledFeatureKarate { + + @Karate.Test + public Karate testCalledFeature() { + return Karate.run("classpath:org/example/test_called_feature.feature"); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature.feature b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature.feature new file mode 100644 index 00000000000..397dfdcf103 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature.feature @@ -0,0 +1,4 @@ +Feature: test called feature + + Scenario: caller + * call read('test_called_feature_nested.feature') diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature_nested.feature b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature_nested.feature new file mode 100644 index 00000000000..4dcb62008a4 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature_nested.feature @@ -0,0 +1,4 @@ +Feature: test called feature nested + + Scenario: called + * def value = true diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/coverages.ftl b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/events.ftl b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/events.ftl new file mode 100644 index 00000000000..60071b5ef53 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/events.ftl @@ -0,0 +1,192 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* call read('test_called_feature_nested.feature')" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* call read('test_called_feature_nested.feature')", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* def value = true" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_span_id}, + "resource" : "* def value = true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.suite" : "[org/example/test_called_feature] test called feature", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_called_feature] test called feature", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.name" : "caller", + "test.status" : "pass", + "test.suite" : "[org/example/test_called_feature] test called feature", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_2}, + "resource" : "[org/example/test_called_feature] test called feature.caller", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate-1.0", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "karate.test_module", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/coverages.ftl b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/events.ftl b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/events.ftl new file mode 100644 index 00000000000..6f8a76bef9b --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/events.ftl @@ -0,0 +1,527 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 15, + "step.startLine" : 15 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 15, + "step.startLine" : 15 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 15, + "step.startLine" : 15 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_3}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 21, + "step.startLine" : 21 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_4}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_4}, + "start" : ${content_start_4}, + "trace_id" : ${content_trace_id_4} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 21, + "step.startLine" : 21 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_5}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_5}, + "trace_id" : ${content_trace_id_5} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print foo" + }, + "metrics" : { + "step.endLine" : 21, + "step.startLine" : 21 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_6}, + "resource" : "* print foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_6}, + "start" : ${content_start_6}, + "trace_id" : ${content_trace_id_6} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_with_setup] test with setup", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_8}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "karate-1.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetup\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_8}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_9}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetup\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_2}, + "start" : ${content_start_9}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_10}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetup\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_3}, + "start" : ${content_start_10}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_11}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "karate-1.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetupOnce\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_4}, + "start" : ${content_start_11}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_4} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_12}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetupOnce\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_5}, + "start" : ${content_start_12}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_5} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_13}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-1.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.traits" : "{\"category\":[\"withSetupOnce\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_6}, + "start" : ${content_start_13}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_6} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_14}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate-1.0", + "test.early_flake.abort_reason" : "faulty", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_8}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_14}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_15}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.abort_reason" : "faulty", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-1.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_9} + }, + "name" : "karate.test_module", + "resource" : "karate-1.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_15}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java index 6b647b73c87..dbeb18a63ab 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java @@ -17,6 +17,10 @@ public class KarateScenarioAdvice { public static class RetryAdvice { @Advice.OnMethodEnter public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) { + if (KarateTracingListener.skipTracking(scenarioRuntime)) { + return; + } + ExecutionContext executionContext = InstrumentationContext.get(Scenario.class, ExecutionContext.class) .computeIfAbsent(scenarioRuntime.getScenario(), ExecutionContext::create); @@ -31,6 +35,10 @@ public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) { public static void afterExecute( @Advice.This ScenarioRuntime scenarioRuntime, @Advice.Return(readOnly = false) ScenarioResult result) { + if (KarateTracingListener.skipTracking(scenarioRuntime)) { + return; + } + if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) { // nested call (a retry invoked below, or a called scenario) return; diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java index ce37bdc0ffa..a9ea059ca26 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java @@ -243,9 +243,10 @@ private static boolean skipTracking(FeatureRuntime fr) { return fr.getCaller() != null; } - private static boolean skipTracking(ScenarioRuntime sr) { + public static boolean skipTracking(ScenarioRuntime sr) { // do not track nested (called) scenario runs and setup scenarios - return sr.getFeatureRuntime().getCaller() != null || sr.getScenario().isSetup(); + FeatureRuntime fr = sr.getFeatureRuntime(); + return fr == null || fr.getCaller() != null || sr.getScenario().isSetup(); } private static boolean skipTracking(Step step) { diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy index 84d52bbf87f..6adce09eeac 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy @@ -91,6 +91,8 @@ class KarateV2Test extends CiVisibilityInstrumentationTest { "test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] // is executed only twice "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] "test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | [] + "test-efd-setup" | [TestWithSetupKarate] | [] + "test-efd-called-feature" | [TestSucceedCalledFeatureKarate] | [new TestFQN("[org/example/test_called_feature] test called feature", "caller")] } def "test quarantined #testcaseName"() { diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/KarateStandaloneScenarioTest.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/KarateStandaloneScenarioTest.java new file mode 100644 index 00000000000..15f2c4f2db9 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/KarateStandaloneScenarioTest.java @@ -0,0 +1,23 @@ +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.karatelabs.common.Resource; +import io.karatelabs.core.KarateJs; +import io.karatelabs.core.ScenarioResult; +import io.karatelabs.core.ScenarioRuntime; +import io.karatelabs.gherkin.Feature; +import io.karatelabs.gherkin.Scenario; +import org.junit.jupiter.api.Test; + +public class KarateStandaloneScenarioTest { + + @Test + public void testStandaloneScenarioRuntime() { + Resource resource = Resource.path("classpath:org/example/test_succeed_one_case.feature"); + Feature feature = Feature.read(resource); + Scenario scenario = feature.getSections().getFirst().getScenario(); + + ScenarioResult result = new ScenarioRuntime(new KarateJs(resource), scenario).call(); + + assertTrue(result.isPassed()); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java new file mode 100644 index 00000000000..df87c42a593 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java @@ -0,0 +1,16 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.intuit.karate.Results; +import com.intuit.karate.Runner; +import org.junit.jupiter.api.Test; + +public class TestSucceedCalledFeatureKarate { + + @Test + public void test() { + Results results = Runner.path("classpath:org/example/test_called_feature.feature").parallel(1); + assertEquals(0, results.getFailCount(), results.getErrorMessages()); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature.feature b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature.feature new file mode 100644 index 00000000000..397dfdcf103 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature.feature @@ -0,0 +1,4 @@ +Feature: test called feature + + Scenario: caller + * call read('test_called_feature_nested.feature') diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature_nested.feature b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature_nested.feature new file mode 100644 index 00000000000..4dcb62008a4 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_called_feature_nested.feature @@ -0,0 +1,4 @@ +Feature: test called feature nested + + Scenario: called + * def value = true diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/coverages.ftl b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/events.ftl b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/events.ftl new file mode 100644 index 00000000000..dc7845a9902 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-called-feature/events.ftl @@ -0,0 +1,192 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* read('test_called_feature_nested.feature')" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* read('test_called_feature_nested.feature')", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* value = true" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_span_id}, + "resource" : "* value = true", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-2.0", + "test.status" : "pass", + "test.suite" : "[org/example/test_called_feature] test called feature", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_called_feature] test called feature", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-2.0", + "test.name" : "caller", + "test.status" : "pass", + "test.suite" : "[org/example/test_called_feature] test called feature", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_2}, + "resource" : "[org/example/test_called_feature] test called feature.caller", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate-2.0", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate-2.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-2.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "karate.test_module", + "resource" : "karate-2.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/coverages.ftl b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/events.ftl b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/events.ftl new file mode 100644 index 00000000000..e3ad6366b05 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-efd-setup/events.ftl @@ -0,0 +1,521 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 13, + "step.startLine" : 13 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 13, + "step.startLine" : 13 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_2}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_2}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 13, + "step.startLine" : 13 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_3}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_3}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 18, + "step.startLine" : 18 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_4}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_4}, + "start" : ${content_start_4}, + "trace_id" : ${content_trace_id_4} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 18, + "step.startLine" : 18 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_5}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_5}, + "trace_id" : ${content_trace_id_5} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* foo" + }, + "metrics" : { + "step.endLine" : 18, + "step.startLine" : 18 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id_6}, + "resource" : "* foo", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_6}, + "start" : ${content_start_6}, + "trace_id" : ${content_trace_id_6} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-2.0", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_with_setup] test with setup", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_8}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "karate-2.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_8}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_9}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-2.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_2}, + "start" : ${content_start_9}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_10}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-2.0", + "test.name" : "first scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_3}, + "start" : ${content_start_10}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_3} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_11}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.module" : "karate-2.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_4}, + "start" : ${content_start_11}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_4} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_12}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-2.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_6}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_5}, + "start" : ${content_start_12}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_5} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_13}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.final_status" : "pass", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_new" : "true", + "test.is_retry" : "true", + "test.module" : "karate-2.0", + "test.name" : "second scenario", + "test.parameters" : "{\"foo\":\"bar\"}", + "test.retry_reason" : "early_flake_detection", + "test.status" : "pass", + "test.suite" : "[org/example/test_with_setup] test with setup", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_7}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_7}, + "resource" : "[org/example/test_with_setup] test with setup.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id_6}, + "start" : ${content_start_13}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_6} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_14}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate-2.0", + "test.early_flake.abort_reason" : "faulty", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_8}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate-2.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_14}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_15}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.early_flake.abort_reason" : "faulty", + "test.early_flake.enabled" : "true", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate-2.0", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_9} + }, + "name" : "karate.test_module", + "resource" : "karate-2.0", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_15}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file From 42b936881f4cf2e09f3bcb62616e7faa4787d784 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Tue, 21 Jul 2026 11:30:34 +0200 Subject: [PATCH 2/2] fix: merge test scenarios --- .../src/test/groovy/KarateTest.groovy | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy index ea4285a8331..4cafd545c09 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy @@ -85,6 +85,8 @@ class KarateTest extends CiVisibilityInstrumentationTest { } def "test early flakiness detection #testcaseName"() { + Assumptions.assumeTrue(assumption) + givenEarlyFlakinessDetectionEnabled(true) givenKnownTests(knownTestsList) @@ -93,29 +95,18 @@ class KarateTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | knownTestsList - "test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestFQN("[org/example/test_succeed_one_case] test succeed", "first scenario")] + testcaseName | tests | knownTestsList | assumption + "test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestFQN("[org/example/test_succeed_one_case] test succeed", "first scenario")] | true "test-efd-known-parameterized-test" | [TestParameterizedKarate] | [ new TestFQN("[org/example/test_parameterized] test parameterized", "first scenario as an outline") - ] - "test-efd-new-test" | [TestSucceedOneCaseKarate] | [] - "test-efd-new-parameterized-test" | [TestParameterizedKarate] | [] - "test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] // is executed only twice - "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] - "test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | [] - "test-efd-called-feature" | [TestSucceedCalledFeatureKarate] | [new TestFQN("[org/example/test_called_feature] test called feature", "caller")] - } - - def "test early flakiness detection ignores setup scenarios"() { - given: - Assumptions.assumeTrue(KarateUtils.isSetupTagSupported(KarateUtils.getKarateVersion())) - - givenEarlyFlakinessDetectionEnabled(true) - givenKnownTests([]) - - runTests([TestWithSetupKarate]) - - assertSpansData("test-efd-setup") + ] | true + "test-efd-new-test" | [TestSucceedOneCaseKarate] | [] | true + "test-efd-new-parameterized-test" | [TestParameterizedKarate] | [] | true + "test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] | true // is executed only twice + "test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] | true + "test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | [] | true + "test-efd-setup" | [TestWithSetupKarate] | [] | KarateUtils.isSetupTagSupported(KarateUtils.getKarateVersion()) + "test-efd-called-feature" | [TestSucceedCalledFeatureKarate] | [new TestFQN("[org/example/test_called_feature] test called feature", "caller")] | true } def "test quarantined #testcaseName"() {