From a2dbd3da2e9eb575fe8db2875852ac614e508ff5 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 30 Sep 2026 16:45:05 +0200 Subject: [PATCH 1/2] feat(ci-visibility): add experimental namespace-isolated observer agent Build a relocated copy of the stock agent that can trace this repository's own test runs with CI Visibility while the tracer under test runs in the same JVMs. Also fix three stock issues found while doing so: TypeFactory resolving a stale schema for the transform target, missing git data in linked worktrees, and per-test coverage recursing through covered defineClass hooks. --- buildSrc/build.gradle.kts | 14 + .../observer/ObserverAgentRewriter.java | 682 ++++++++++++++++ .../observer/ObserverAgentSelection.java | 50 ++ .../plugin/observer/ObserverAgentTask.java | 27 + .../observer/bootstrap/ObserverBootstrap.java | 26 + .../observer/bootstrap/ObserverRuntime.java | 446 +++++++++++ .../dd-trace-java.configure-tests.gradle.kts | 19 + ...dd-trace-java.modifiable-config.gradle.kts | 7 +- .../observer/ObserverConfigSourcesTest.java | 727 ++++++++++++++++++ .../bootstrap/ObserverRuntimeTest.java | 202 +++++ .../observer/ObserverAgentRewriterTest.kt | 212 +++++ .../trace/civisibility/ci/UnknownCIInfo.java | 20 +- .../coverage/line/LineCoverageStore.java | 4 + .../civisibility/ci/UnknownCIInfoTest.groovy | 21 + .../bytebuddy/outline/TypeFactory.java | 19 +- .../outline/TypeFactoryTransformTest.java | 126 +++ dd-java-agent/build.gradle | 6 + docs/tracing_the_tracer.md | 186 +++++ 18 files changed, 2786 insertions(+), 8 deletions(-) create mode 100644 buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentRewriter.java create mode 100644 buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentSelection.java create mode 100644 buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentTask.java create mode 100644 buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java create mode 100644 buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java create mode 100644 buildSrc/src/test/java/datadog/gradle/plugin/observer/ObserverConfigSourcesTest.java create mode 100644 buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java create mode 100644 buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentRewriterTest.kt create mode 100644 dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java create mode 100644 docs/tracing_the_tracer.md diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index b31a2fde05c..f1447b1a18f 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -5,6 +5,15 @@ plugins { alias(libs.plugins.spotless) } +spotless { + java { + target("src/**/*.java") + removeUnusedImports() + forbidWildcardImports() + googleJavaFormat(libs.versions.google.java.format.get()) + } +} + // The buildSrc still needs to target Java 8 as build time instrumentation and muzzle plugin // allow to schedule workers on different JDK version. java { @@ -102,6 +111,7 @@ dependencies { implementation(libs.asm) implementation(libs.asm.tree) + implementation(libs.asm.commons) implementation(platform("com.fasterxml.jackson:jackson-bom:2.17.2")) implementation("com.fasterxml.jackson.core:jackson-databind") @@ -128,6 +138,10 @@ testing { targets.configureEach { testTask.configure { enabled = providers.gradleProperty("runBuildSrcTests").isPresent or providers.systemProperty("idea.active").isPresent + providers.gradleProperty("observerTestArtifact").orNull?.let { + systemProperty("observer.test.artifact", it) + systemProperty("observer.test.stock", providers.gradleProperty("observerTestStock").get()) + } } } } diff --git a/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentRewriter.java b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentRewriter.java new file mode 100644 index 00000000000..8e1ff6277ec --- /dev/null +++ b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentRewriter.java @@ -0,0 +1,682 @@ +package datadog.gradle.plugin.observer; + +import static java.util.Arrays.asList; +import static java.util.Collections.sort; + +import datadog.trace.observer.bootstrap.ObserverBootstrap; +import datadog.trace.observer.bootstrap.ObserverRuntime; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.FieldInsnNode; +import org.objectweb.asm.tree.FrameNode; +import org.objectweb.asm.tree.InsnList; +import org.objectweb.asm.tree.InsnNode; +import org.objectweb.asm.tree.JumpInsnNode; +import org.objectweb.asm.tree.LabelNode; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodInsnNode; +import org.objectweb.asm.tree.MethodNode; +import org.objectweb.asm.tree.VarInsnNode; + +/** Offline, layout-checked namespace isolation of the stock agent. */ +public final class ObserverAgentRewriter { + private static final String RUNTIME = "datadog/trace/observer/bootstrap/ObserverRuntime"; + private static final String PROVIDER = "datadog/trace/bootstrap/config/provider/ConfigProvider"; + private static final String JUNIT = "datadog/trace/instrumentation/junit5/"; + private static final String GRADLE = "datadog/trace/instrumentation/gradle/"; + private static final Set SYSTEM_METHODS = + new HashSet<>( + asList("getProperty", "getProperties", "setProperty", "clearProperty", "getenv")); + + /** Host paths, config key fragments and wire names that only look like relocatable packages. */ + private static final Set EXTERNAL_LITERALS = + new HashSet<>( + asList( + "/var/run/datadog/apm.socket", + "/var/run/datadog/dsd.socket", + ".inject.datadog.attribute.enabled", + "datadog.product:", + "datadog.tracer.stats.collapsed_spans", + "datadog.jvm.runtime", + "datadog.operation.name", + "datadog.span.type", + "datadog.span.top_level", + "datadog.is_trace_root", + "datadog.svc_src", + "datadog.origin", + "datadog.peer_tags", + "datadog.sdk.semantics", + "datadog.runtime_id", + "datadog.process_tags")); + + private static final String DOGSTATSD_METRICS = "datadog.dogstatsd.client."; + + private final Set patches = new HashSet<>(); + private final Set configAliases = new TreeSet<>(); + private final Set sensitiveConfig = new TreeSet<>(); + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + throw new IllegalArgumentException("Expected stock-agent.jar observer-agent.jar"); + } + new ObserverAgentRewriter().rewrite(new File(args[0]), new File(args[1])); + } + + public void rewrite(File input, File output) throws Exception { + if (input.getCanonicalFile().equals(output.getCanonicalFile())) { + throw new IllegalArgumentException("Never overwrite the stock agent"); + } + patches.clear(); + configAliases.clear(); + sensitiveConfig.clear(); + Map entries = new LinkedHashMap<>(); + Manifest manifest; + try (JarFile jar = new JarFile(input)) { + manifest = new Manifest(jar.getManifest()); + Attributes attributes = manifest.getMainAttributes(); + require( + "datadog.trace.bootstrap.AgentPreCheck".equals(attributes.getValue("Premain-Class")), + "Expected a stock agent manifest"); + for (String key : asList("Premain-Class", "Agent-Class", "Main-Class")) { + String value = attributes.getValue(key); + if (value != null) { + attributes.putValue(key, relocate(value)); + } + } + attributes.putValue("Tracing-The-Tracer-Observer", "2"); + attributes.putValue("Premain-Class", ObserverBootstrap.class.getName()); + attributes.remove(new Attributes.Name("Agent-Class")); + for (String required : + asList( + "datadog/trace/bootstrap/AgentBootstrap.class", + "datadog/trace/bootstrap/AgentPreCheck.class", + PROVIDER + ".class", + "datadog/trace/bootstrap/config/provider/PropertiesConfigSource.class", + "dd-java-agent.index", + "inst/instrumenter.index", + "inst/known-types.index")) { + require(jar.getJarEntry(required) != null, "Missing expected stock entry: " + required); + } + Enumeration source = jar.entries(); + while (source.hasMoreElements()) { + JarEntry entry = source.nextElement(); + String name = entry.getName(); + // Rebuild known types from the relocated full instrumenter index at startup. + if (entry.isDirectory() + || name.equals("META-INF/MANIFEST.MF") + || name.equals("dd-java-agent.index") + || name.equals("inst/known-types.index")) { + continue; + } + byte[] bytes; + try (InputStream stream = jar.getInputStream(entry)) { + bytes = readAll(stream); + } + if (name.endsWith(".class") || name.endsWith(".classdata")) { + bytes = rewriteClass(bytes); + } else if (name.equals("inst/instrumenter.index")) { + bytes = rewriteInstrumenterIndex(bytes); + patches.add("instrumenter.index"); + } else if (name.contains("META-INF/services/")) { + bytes = + relocate(new String(bytes, StandardCharsets.UTF_8)).getBytes(StandardCharsets.UTF_8); + } + String targetName = relocate(name); + if (name.equals("simplelogger.properties") || name.endsWith("/simplelogger.properties")) { + targetName = + targetName.substring(0, targetName.length() - "simplelogger.properties".length()) + + "observer-simplelogger.properties"; + } + require(entries.put(targetName, bytes) == null, "Duplicate relocated entry: " + name); + } + } + Set expected = + new HashSet<>( + asList( + "instrumenter.index", + "createDefault", + "withoutCollector", + "withPropertiesOverride", + "stable-source", + "child-arguments", + "generated-properties", + "private-carrier", + "numeric-ipc-host", + "generated-file-logger", + "private-logger-resource", + "logger-resource-keys", + "JUnit5Instrumentation$JUnit5Advice", + "JUnit5SpockInstrumentation$SpockAdvice")); + require( + patches.equals(expected), + "Input agent layout changed: expected patches " + expected + ", got " + patches); + try (InputStream stream = ObserverRuntime.class.getResourceAsStream("ObserverRuntime.class")) { + entries.put(RUNTIME + ".class", readAll(stream)); + } + try (InputStream stream = + ObserverBootstrap.class.getResourceAsStream("ObserverBootstrap.class")) { + entries.put("datadog/trace/observer/bootstrap/ObserverBootstrap.class", readAll(stream)); + } + require(!configAliases.isEmpty(), "Missing generated configuration metadata"); + entries.put( + "observer-config-aliases.txt", + String.join("\n", configAliases).getBytes(StandardCharsets.UTF_8)); + require(!sensitiveConfig.isEmpty(), "Missing sensitive configuration metadata"); + entries.put( + "observer-sensitive-config.txt", + String.join("\n", sensitiveConfig).getBytes(StandardCharsets.UTF_8)); + entries.put("dd-observer-agent.index", buildJarIndex(input, entries)); + Path target = output.getAbsoluteFile().toPath(); + Files.createDirectories(target.getParent()); + // A JVM may still be running from the previous jar, so never truncate it in place. + Path temporary = target.resolveSibling(target.getFileName() + ".tmp"); + try { + try (JarOutputStream jar = new JarOutputStream(Files.newOutputStream(temporary), manifest)) { + for (Map.Entry entry : entries.entrySet()) { + JarEntry jarEntry = new JarEntry(entry.getKey()); + jarEntry.setTime(0); + jar.putNextEntry(jarEntry); + jar.write(entry.getValue()); + jar.closeEntry(); + } + } + Files.move( + temporary, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } finally { + Files.deleteIfExists(temporary); + } + } + + static String relocate(String value) { + if (EXTERNAL_LITERALS.contains(value) || value.startsWith(DOGSTATSD_METRICS)) { + return value; + } + return value + .replace("com.datadog.", "__OBSERVER_COM_DOT__") + .replace("com/datadog/", "__OBSERVER_COM_SLASH__") + .replace("datadog.", "datadog.trace.observer.") + .replace("datadog/", "datadog/trace/observer/") + .replace("__OBSERVER_COM_DOT__", "datadog.trace.observer.com.datadog.") + .replace("__OBSERVER_COM_SLASH__", "datadog/trace/observer/com/datadog/") + .replace("net.bytebuddy.", "datadog.trace.observer.net.bytebuddy.") + .replace("net/bytebuddy/", "datadog/trace/observer/net/bytebuddy/") + .replace("dd-java-agent.index", "dd-observer-agent.index") + .replace("dd-java-agent.version", "dd-observer-agent.version") + .replace("instrumenter.index", "observer-instrumenter.index") + .replace("known-types.index", "observer-known-types.index") + .replace("__datadogContext$", "__datadogObserverContext$"); + } + + byte[] rewriteClass(byte[] bytes) { + ClassNode node = new ClassNode(); + new ClassReader(bytes).accept(node, 0); + if (node.name.equals(PROVIDER)) { + require( + node.methods.stream() + .anyMatch( + method -> + method.name.equals("") + && method.desc.equals("([L" + PROVIDER + "$Source;)V")), + "ConfigProvider constructor changed"); + } + for (MethodNode method : node.methods) { + patchGradleBoundary(node, method); + if (node.name.equals("datadog/trace/logging/simplelogger/SLCompatSettings") + && method.name.equals("loadProperties")) { + require( + method.desc.equals("(Ljava/lang/String;)Ljava/util/Properties;"), + "Logger resource parser changed"); + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction.getOpcode() == Opcodes.ARETURN) { + method.instructions.insertBefore( + instruction, + new MethodInsnNode( + Opcodes.INVOKESTATIC, + RUNTIME, + "loggerProperties", + "(Ljava/util/Properties;)Ljava/util/Properties;", + false)); + } + } + patches.add("logger-resource-keys"); + } + if (node.name.equals("datadog/trace/bootstrap/config/provider/StableConfigSource") + && method.name.equals("")) { + require( + method.desc.equals("(Ljava/lang/String;Ldatadog/trace/api/ConfigOrigin;)V"), + "Stable source changed"); + clearBody(method); + InsnList code = method.instructions; + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add( + new MethodInsnNode( + Opcodes.INVOKESPECIAL, PROVIDER + "$Source", "", "()V", false)); + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add(new VarInsnNode(Opcodes.ALOAD, 2)); + code.add( + new FieldInsnNode( + Opcodes.PUTFIELD, node.name, "fileOrigin", "Ldatadog/trace/api/ConfigOrigin;")); + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add( + new FieldInsnNode( + Opcodes.GETSTATIC, + node.name + "$StableConfig", + "EMPTY", + "L" + node.name + "$StableConfig;")); + code.add( + new FieldInsnNode( + Opcodes.PUTFIELD, node.name, "config", "L" + node.name + "$StableConfig;")); + code.add(new InsnNode(Opcodes.RETURN)); + patches.add("stable-source"); + } + if (node.name.equals(PROVIDER) + && asList("createDefault", "withoutCollector", "withPropertiesOverride") + .contains(method.name)) { + String expected = + method.name.equals("withPropertiesOverride") + ? "(Ljava/util/Properties;)L" + PROVIDER + ";" + : "()L" + PROVIDER + ";"; + require(method.desc.equals(expected), "ConfigProvider signature changed: " + method.name); + // Preserve the complete stock source chain, factory flags and caller overrides. + patches.add(method.name); + } + if ((node.name.equals(JUNIT + "JUnit5Instrumentation$JUnit5Advice") + || node.name.equals(JUNIT + "JUnit5SpockInstrumentation$SpockAdvice")) + && method.name.equals("addTracingListener")) { + require( + method.desc.equals( + "(Lorg/junit/platform/engine/TestEngine;Lorg/junit/platform/engine/ExecutionRequest;)V"), + "Advice signature changed"); + InsnList guard = new InsnList(); + LabelNode accepted = new LabelNode(); + // Already relocated: the remapper below deliberately leaves this bridge name unchanged. + guard.add( + new MethodInsnNode( + Opcodes.INVOKESTATIC, RUNTIME, "isOuterGradleExecution", "()Z", false)); + guard.add(new JumpInsnNode(Opcodes.IFNE, accepted)); + guard.add(new InsnNode(Opcodes.RETURN)); + guard.add(accepted); + guard.add(new FrameNode(Opcodes.F_SAME, 0, null, 0, null)); + method.instructions.insert(guard); + patches.add(node.name.substring(JUNIT.length())); + } + for (AbstractInsnNode instruction : method.instructions) { + if (node.name.endsWith("/GeneratedSupportedConfigurations") + && method.name.startsWith("initSensitiveKeys") + && instruction instanceof LdcInsnNode + && ((LdcInsnNode) instruction).cst instanceof String) { + sensitiveConfig.add((String) ((LdcInsnNode) instruction).cst); + } + + if (node.name.endsWith("/GeneratedSupportedConfigurations") + && (method.name.startsWith("initAliasMapping") + || method.name.startsWith("initDeprecated")) + && instruction instanceof LdcInsnNode) { + Object value = ((LdcInsnNode) instruction).cst; + if (value instanceof String && !((String) value).isEmpty()) { + configAliases.add((String) value); + } + } + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if ((call.owner.equals("java/lang/System") && SYSTEM_METHODS.contains(call.name)) + || (call.owner.equals("java/lang/Boolean") && call.name.equals("getBoolean"))) { + call.owner = RUNTIME; + } + } + } + } + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + node.accept( + new ClassRemapper( + writer, + new Remapper(Opcodes.ASM9) { + @Override + public String map(String name) { + return name.equals(RUNTIME) ? name : relocate(name); + } + + @Override + public String mapFieldName(String owner, String name, String descriptor) { + return name.replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public String mapMethodName(String owner, String name, String descriptor) { + return name.replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public String mapInvokeDynamicMethodName( + String name, String descriptor, Handle bootstrapMethod, Object... arguments) { + return super.mapInvokeDynamicMethodName( + name, descriptor, bootstrapMethod, arguments) + .replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public Object mapValue(Object value) { + if (value instanceof String) { + String text = (String) value; + if (node.name.startsWith("datadog/trace/logging/simplelogger/SLCompatSettings") + && text.equals("simplelogger.properties")) { + patches.add("private-logger-resource"); + return "observer-simplelogger.properties"; + } + if (node.name.startsWith(GRADLE)) { + if (text.equals("ciVisibilityService")) { + return "observerCiVisibilityService"; + } + if (text.equals("dd-ci-visibility")) { + return "observer-ci-visibility"; + } + if (text.equals("moduleLayout")) { + return "observerModuleLayout"; + } + } + return relocate(text); + } + if (value instanceof Handle) { + Handle handle = (Handle) value; + if ((handle.getOwner().equals("java/lang/System") + && SYSTEM_METHODS.contains(handle.getName())) + || (handle.getOwner().equals("java/lang/Boolean") + && handle.getName().equals("getBoolean"))) { + return new Handle( + handle.getTag(), RUNTIME, handle.getName(), handle.getDesc(), false); + } + } + return super.mapValue(value); + } + })); + return writer.toByteArray(); + } + + private static void clearBody(MethodNode method) { + method.instructions.clear(); + method.tryCatchBlocks.clear(); + if (method.localVariables != null) { + method.localVariables.clear(); + } + } + + private void patchGradleBoundary(ClassNode node, MethodNode method) { + if (node.name.equals(GRADLE + "GradleDaemonLoggingInstrumentation$ReinitialiseLogging") + && method.name.equals("reinitialiseTracerLogging")) { + require( + method.desc.equals("()V") && (method.access & Opcodes.ACC_STATIC) != 0, + "Gradle logging reset seam changed"); + InsnList guard = new InsnList(); + LabelNode reset = new LabelNode(); + guard.add( + new MethodInsnNode( + Opcodes.INVOKESTATIC, RUNTIME, "usingGeneratedFileLogger", "()Z", false)); + guard.add(new JumpInsnNode(Opcodes.IFEQ, reset)); + guard.add(new InsnNode(Opcodes.RETURN)); + guard.add(reset); + guard.add(new FrameNode(Opcodes.F_SAME, 0, null, 0, null)); + method.instructions.insert(guard); + patches.add("generated-file-logger"); + } + if (node.name.equals(GRADLE + "CiVisibilityService") + && method.name.equals("getTracerJvmArgs")) { + require( + method.desc.equals("(Ljava/lang/String;)Ljava/util/Collection;"), + "Gradle worker transport changed"); + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("datadog/trace/api/civisibility/domain/BuildModuleSettings") + && call.name.equals("getSystemProperties")) { + method.instructions.insert( + call, + new MethodInsnNode( + Opcodes.INVOKESTATIC, + RUNTIME, + "generatedProperties", + "(Ljava/util/Map;)Ljava/util/Map;", + false)); + patches.add("generated-properties"); + } + } + } + } + if (node.name.equals(GRADLE + "TracerArgumentsProvider") && method.name.equals("asArguments")) { + require(method.desc.equals("()Ljava/lang/Iterable;"), "Gradle argument provider changed"); + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction.getOpcode() == Opcodes.ARETURN) { + method.instructions.insertBefore( + instruction, + new MethodInsnNode( + Opcodes.INVOKESTATIC, + RUNTIME, + "childArguments", + "(Ljava/lang/Iterable;)Ljava/lang/Iterable;", + false)); + patches.add("child-arguments"); + } + } + } + if (node.name.equals("datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl") + && method.name.equals("getPropertiesPropagatedToChildProcess")) { + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("java/net/InetSocketAddress") && call.name.equals("getHostName")) { + call.name = "getAddress"; + call.desc = "()Ljava/net/InetAddress;"; + method.instructions.insert( + call, + new MethodInsnNode( + Opcodes.INVOKEVIRTUAL, + "java/net/InetAddress", + "getHostAddress", + "()Ljava/lang/String;", + false)); + patches.add("numeric-ipc-host"); + } + } + } + } + if (node.name.equals("datadog/trace/civisibility/ProcessHierarchy") + && method.name.equals("")) { + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("datadog/environment/SystemProperties") + && call.name.equals("asStringMap")) { + require(call.desc.equals("()Ljava/util/Map;"), "Process carrier changed"); + call.owner = RUNTIME; + call.name = "propagationProperties"; + patches.add("private-carrier"); + } + } + } + } + } + + /** Parse the current packed-name format, including member-level target-system overrides. */ + static byte[] rewriteInstrumenterIndex(byte[] bytes) throws IOException { + DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes)); + int modules = in.readInt(); + int transformations = in.readInt(); + require( + modules > 0 && modules < 10000 && transformations > 0, "Invalid instrumenter index header"); + require(in.readInt() == in.available(), "Invalid packed-name length"); + ByteArrayOutputStream packed = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(packed); + Set found = new HashSet<>(); + int actualTransformations = 0; + for (int i = 0; i < modules; i++) { + String module = readName(in); + ByteArrayOutputStream record = new ByteArrayOutputStream(); + DataOutputStream target = new DataOutputStream(record); + writeName(target, relocate(module)); + target.writeShort(in.readUnsignedShort()); + int flags = in.readUnsignedByte(); + require((flags & ~3) == 0, "Unknown module flags"); + target.writeByte(flags); + int members = in.readUnsignedByte(); + target.writeByte(members); + for (int member = 0; member < (members == 255 ? 1 : members); member++) { + if (members != 255) { + writeName(target, readName(in)); + } + if ((flags & 1) != 0) { + int overrides = in.readUnsignedByte(); + target.writeByte(overrides); + for (int o = 0; o < overrides; o++) { + writeName(target, readName(in)); + target.writeShort(in.readUnsignedShort()); + } + } + } + require(found.add(module), "Duplicate instrumenter module"); + out.write(record.toByteArray()); + actualTransformations += members == 255 ? 1 : members; + } + require( + in.available() == 0 && actualTransformations == transformations, + "Instrumenter index layout changed"); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + DataOutputStream header = new DataOutputStream(result); + header.writeInt(found.size()); + header.writeInt(transformations); + header.writeInt(packed.size()); + packed.writeTo(header); + return result.toByteArray(); + } + + private static String readName(DataInputStream in) throws IOException { + byte[] bytes = new byte[in.readUnsignedByte()]; + in.readFully(bytes); + return new String(bytes, StandardCharsets.ISO_8859_1); + } + + private static void writeName(DataOutputStream out, String name) throws IOException { + require(name.length() < 256, "Indexed name exceeds one-byte length"); + out.writeByte(name.length()); + out.writeBytes(name); + } + + /** Rebuild the jar-local trie from final entries, never byte-patch a serialized trie. */ + private static byte[] buildJarIndex(File original, Map entries) throws Exception { + try (URLClassLoader loader = new URLClassLoader(new URL[] {original.toURI().toURL()}, null)) { + Class builderClass = loader.loadClass("datadog.instrument.utils.ClassNameTrie$Builder"); + Object builder = builderClass.getConstructor().newInstance(); + List prefixes = new ArrayList<>(); + List names = new ArrayList<>(entries.keySet()); + Map classKeys = new LinkedHashMap<>(); + sort(names); + for (String name : names) { + int prefixId = 0; + String key = name; + // Stock root entries are bootstrap classes/resources; classdata belongs to a feature root. + if (!name.startsWith("datadog/") && !name.startsWith("META-INF/") && name.contains("/")) { + String prefix = name.substring(0, name.indexOf('/') + 1); + if (!prefixes.contains(prefix)) { + prefixes.add(prefix); + } + prefixId = prefixes.indexOf(prefix) + 1; + key = name.substring(prefix.length()); + } + if (key.endsWith(".classdata")) { + key = key.substring(0, key.length() - ".classdata".length()); + } else if (key.endsWith(".class")) { + key = key.substring(0, key.length() - ".class".length()); + } + if (name.endsWith(".class") || name.endsWith(".classdata")) { + require( + key.equals(new ClassReader(entries.get(name)).getClassName()), + "Entry/class identity mismatch: " + name); + require( + classKeys.put(key.replace('/', '.'), prefixId) == null, + "Duplicate class across feature roots: " + name); + } + builderClass + .getMethod("put", String.class, int.class) + .invoke(builder, key.replace('/', '.'), prefixId); + // Resource lookup also needs the class suffix. + String resourceKey = + (prefixId == 0 ? name : name.substring(prefixes.get(prefixId - 1).length())) + .replace(".classdata", ".class"); + builderClass + .getMethod("put", String.class, int.class) + .invoke(builder, resourceKey.replace('/', '.'), prefixId); + } + for (Map.Entry entry : classKeys.entrySet()) { + require( + entry + .getValue() + .equals( + builderClass.getMethod("apply", String.class).invoke(builder, entry.getKey())), + "Class index does not resolve " + entry.getKey()); + require( + entry + .getValue() + .equals( + builderClass + .getMethod("apply", String.class) + .invoke(builder, entry.getKey() + ".class")), + "Resource index does not resolve " + entry.getKey()); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeInt(prefixes.size()); + for (String prefix : prefixes) { + out.writeUTF(prefix); + } + builderClass.getMethod("writeTo", DataOutput.class).invoke(builder, out); + return bytes.toByteArray(); + } + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = stream.read(buffer)) != -1) { + bytes.write(buffer, 0, read); + } + return bytes.toByteArray(); + } + + private static void require(boolean condition, String message) { + if (!condition) { + throw new IllegalArgumentException(message); + } + } +} diff --git a/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentSelection.java b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentSelection.java new file mode 100644 index 00000000000..f5ec892bbd4 --- /dev/null +++ b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentSelection.java @@ -0,0 +1,50 @@ +package datadog.gradle.plugin.observer; + +import java.io.File; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.jar.JarFile; + +/** Resolve the observer actually attached to this JVM, without caller artifact metadata. */ +public final class ObserverAgentSelection { + private ObserverAgentSelection() {} + + public static String configurationFingerprint() throws Exception { + return (String) + Class.forName("datadog.trace.observer.bootstrap.ObserverRuntime", false, null) + .getMethod("configurationFingerprint") + .invoke(null); + } + + public static File attached() throws Exception { + Class runtime; + try { + runtime = Class.forName("datadog.trace.observer.bootstrap.ObserverRuntime", false, null); + } catch (ClassNotFoundException absent) { + return null; + } + URL resource = runtime.getResource("ObserverRuntime.class"); + if (resource == null || !resource.getProtocol().equals("jar")) { + throw new IllegalStateException("Attached observer has no jar resource"); + } + File jar = + new File(((JarURLConnection) resource.openConnection()).getJarFileURL().toURI()) + .getCanonicalFile(); + try (JarFile file = new JarFile(jar)) { + if (!"2" + .equals(file.getManifest().getMainAttributes().getValue("Tracing-The-Tracer-Observer"))) { + throw new IllegalArgumentException("Not an observer artifact"); + } + } + return jar; + } + + public static File validate() throws Exception { + File jar = attached(); + if (jar == null) { + throw new IllegalArgumentException( + "traceTracer requires an observer attached to the Gradle daemon with -javaagent"); + } + return jar; + } +} diff --git a/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentTask.java b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentTask.java new file mode 100644 index 00000000000..a2392bad19a --- /dev/null +++ b/buildSrc/src/main/java/datadog/gradle/plugin/observer/ObserverAgentTask.java @@ -0,0 +1,27 @@ +package datadog.gradle.plugin.observer; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.TaskAction; +import org.gradle.work.DisableCachingByDefault; + +/** Explicit developer artifact: never included in the published distribution. */ +@DisableCachingByDefault(because = "Experimental offline transformation of the locally built agent") +public abstract class ObserverAgentTask extends DefaultTask { + @InputFile + @PathSensitive(PathSensitivity.NONE) + public abstract RegularFileProperty getStockAgent(); + + @OutputFile + public abstract RegularFileProperty getObserverAgent(); + + @TaskAction + public void rewrite() throws Exception { + new ObserverAgentRewriter() + .rewrite(getStockAgent().get().getAsFile(), getObserverAgent().get().getAsFile()); + } +} diff --git a/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java new file mode 100644 index 00000000000..8c9360cb2f0 --- /dev/null +++ b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java @@ -0,0 +1,26 @@ +package datadog.trace.observer.bootstrap; + +import java.io.File; +import java.lang.instrument.Instrumentation; +import java.util.jar.JarFile; + +/** Install the JDK-only bridge in bootstrap before any relocated early configuration reads. */ +public final class ObserverBootstrap { + private ObserverBootstrap() {} + + public static void premain(String arguments, Instrumentation instrumentation) throws Exception { + File jar = + new File( + ObserverBootstrap.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + instrumentation.appendToBootstrapClassLoaderSearch(new JarFile(jar)); + Class.forName("datadog.trace.observer.bootstrap.ObserverRuntime", true, null) + .getMethod("initializePremain", String.class) + .invoke(null, arguments); + Class.forName("datadog.trace.observer.trace.bootstrap.AgentPreCheck", true, null) + .getMethod("premain", String.class, Instrumentation.class) + .invoke( + null, + arguments != null && arguments.startsWith("v1:") ? null : arguments, + instrumentation); + } +} diff --git a/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java new file mode 100644 index 00000000000..e15a27b8437 --- /dev/null +++ b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java @@ -0,0 +1,446 @@ +package datadog.trace.observer.bootstrap; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; + +/** Private stock source views and process transport; no subject configuration is imported. */ +public final class ObserverRuntime { + public static final String PROPERTY_PREFIX = "tracing.observer.config."; + public static final String ENV_PREFIX = "TRACING_OBSERVER_CONFIG_"; + private static final String CHILD = "tracing.observer.child.v1"; + private static final int LIMIT = 1024 * 1024; + private static final Set ALIASES = metadata("observer-config-aliases.txt"); + private static final Set SENSITIVE = metadata("observer-sensitive-config.txt"); + private static final String GENERATED = "tracing.observer.generated."; + private static final Properties CONFIG = new Properties(); + private static final Map ENVIRONMENT = new HashMap<>(); + private static final Map CARRIER = new HashMap<>(); + private static final Map LAUNCH = new HashMap<>(); + + private static volatile boolean initialized; + private static String generatedLogFile; + private static String configurationFingerprint; + + /** Initialize the private source view before the stock bootstrap reads configuration. */ + public static synchronized void initializePremain(String arguments) { + initialize(arguments); + } + + /** Private source initialization, also usable by standalone offline configuration tests. */ + public static synchronized void initialize(String arguments) { + if (initialized) { + throw new IllegalStateException("Observer runtime already initialized"); + } + String child = System.getProperty(CHILD); + boolean envelope = arguments != null && arguments.startsWith("v1:"); + if (envelope && child != null) { + throw new IllegalArgumentException("Conflicting observer launch channels"); + } + for (String key : System.getProperties().stringPropertyNames()) { + if (key.startsWith(PROPERTY_PREFIX)) { + CONFIG.setProperty( + privateSpelling(key.substring(PROPERTY_PREFIX.length())), System.getProperty(key)); + } else if (key.startsWith("tracing.observer.") && !key.equals(CHILD)) { + LAUNCH.put(key, System.getProperty(key)); + } + } + for (Map.Entry e : System.getenv().entrySet()) { + if (e.getKey().startsWith(ENV_PREFIX)) { + ENVIRONMENT.put(e.getKey().substring(ENV_PREFIX.length()), e.getValue()); + } + } + Properties childOverrides = new Properties(); + if (child != null) { + childOverrides.putAll(CONFIG); + } + if (child != null || envelope) { + List> maps = decode(child != null ? child : arguments.substring(3)); + if (child != null) { + CONFIG.clear(); + CONFIG.putAll(maps.get(0)); + } else { + for (Map.Entry e : maps.get(0).entrySet()) { + CONFIG.setProperty(privateSpelling(e.getKey()), e.getValue()); + } + } + ENVIRONMENT.putAll(maps.get(1)); + LAUNCH.clear(); + LAUNCH.putAll(maps.get(2)); + CARRIER.putAll(maps.get(3)); + } + configurationFingerprint = fingerprint(); + CONFIG.putAll(CARRIER); + CONFIG.putAll(childOverrides); + configureFileLogger(); + initialized = true; + } + + private ObserverRuntime() {} + + private static void ensureInitialized() { + if (!initialized) { + synchronized (ObserverRuntime.class) { + if (!initialized) { + initialize(null); + } + } + } + } + + /** Stable task input for the captured caller configuration, not later mutable IPC state. */ + public static String configurationFingerprint() { + ensureInitialized(); + return configurationFingerprint; + } + + private static String fingerprint() { + Map properties = new HashMap<>(); + for (String key : CONFIG.stringPropertyNames()) { + properties.put(key, CONFIG.getProperty(key)); + } + String encoded = encode(asList(properties, ENVIRONMENT, LAUNCH, CARRIER)); + try { + byte[] hash = + MessageDigest.getInstance("SHA-256").digest(encoded.getBytes(StandardCharsets.UTF_8)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(hash); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + public static boolean usingGeneratedFileLogger() { + ensureInitialized(); + return generatedLogFile != null + && generatedLogFile.equals( + CONFIG.getProperty("datadog.trace.observer.slf4j.simpleLogger.logFile")); + } + + private static void configureFileLogger() { + String logs = LAUNCH.get("tracing.observer.log.directory"); + if (logs != null) { + String key = "datadog.trace.observer.slf4j.simpleLogger.logFile"; + if (!CONFIG.containsKey(key)) { + generatedLogFile = + new File(logs, "observer-" + Long.toHexString(System.nanoTime()) + ".log").getPath(); + CONFIG.setProperty(key, generatedLogFile); + } + } + } + + /** Resource files keep ordinary logger key spellings; values are never relocated. */ + public static Properties loggerProperties(Properties properties) { + if (properties == null) { + return null; + } + Properties result = new Properties(); + for (String key : properties.stringPropertyNames()) { + result.setProperty(privateSpelling(key), properties.getProperty(key)); + } + return result; + } + + private static String privateSpelling(String key) { + // Internal snapshots already use private keys; never relocate them recursively. + if (key.startsWith("datadog.trace.observer.")) { + return key; + } + if (key.startsWith("datadog.") || key.startsWith("net.bytebuddy.")) { + // Per-class logger keys also contain the target class name in their suffix. + return key.replace("com.datadog.", "__OBSERVER_COM_DOT__") + .replace("datadog.", "datadog.trace.observer.") + .replace("__OBSERVER_COM_DOT__", "datadog.trace.observer.com.datadog.") + .replace("net.bytebuddy.", "datadog.trace.observer.net.bytebuddy."); + } + return key; + } + + private static Set metadata(String resource) { + Set result = new HashSet<>(); + try (InputStream in = ObserverRuntime.class.getResourceAsStream("/" + resource)) { + if (in != null) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int n; + while ((n = in.read(buffer)) != -1) { + out.write(buffer, 0, n); + } + for (String key : new String(out.toByteArray(), StandardCharsets.UTF_8).split("\n")) { + result.add(key); + } + } + } catch (IOException e) { + throw new IllegalStateException("Cannot load observer configuration metadata", e); + } + return result; + } + + private static boolean privateKey(String key) { + return key.startsWith("dd.") + || key.startsWith("datadog.") + || key.startsWith("otel.") + || key.startsWith("net.bytebuddy.") + || key.startsWith("tracing.observer.") + || CONFIG.containsKey(key) + || ALIASES.contains(key); + } + + public static String getProperty(String key) { + return getProperty(key, null); + } + + public static String getProperty(String key, String fallback) { + ensureInitialized(); + if (key == null) { + throw new NullPointerException("key"); + } + if (key.isEmpty()) { + throw new IllegalArgumentException("key is empty"); + } + if (!privateKey(key)) { + return System.getProperty(key, fallback); + } + String value = CONFIG.getProperty(key); + return value == null ? fallback : value; + } + + public static boolean getBoolean(String key) { + try { + return Boolean.parseBoolean(getProperty(key)); + } catch (IllegalArgumentException | NullPointerException ignored) { + return false; + } + } + + public static String setProperty(String key, String value) { + getProperty(key); + return privateKey(key) + ? (String) CONFIG.setProperty(key, value) + : System.setProperty(key, value); + } + + public static String clearProperty(String key) { + getProperty(key); + return privateKey(key) ? (String) CONFIG.remove(key) : System.clearProperty(key); + } + + public static Properties getProperties() { + ensureInitialized(); + Properties result = new Properties(); + Properties real = System.getProperties(); + for (String key : real.stringPropertyNames()) { + if (!privateKey(key)) { + result.setProperty(key, real.getProperty(key)); + } + } + result.putAll(CONFIG); + return result; + } + + public static String getenv(String key) { + ensureInitialized(); + if (key == null) { + throw new NullPointerException("key"); + } + return key.startsWith("DD_") + || key.startsWith("OTEL_") + || key.startsWith("TRACING_OBSERVER_") + || ALIASES.contains(key) + || ENVIRONMENT.containsKey(key) + ? ENVIRONMENT.get(key) + : System.getenv(key); + } + + public static Map getenv() { + ensureInitialized(); + Map result = new HashMap<>(); + for (String key : System.getenv().keySet()) { + String value = getenv(key); + if (value != null) { + result.put(key, value); + } + } + result.putAll(ENVIRONMENT); + return unmodifiableMap(result); + } + + /** + * Only the parent-generated map can classify a worker; ambient HTTP headers are not a carrier. + */ + public static Map propagationProperties() { + ensureInitialized(); + return unmodifiableMap(CARRIER); + } + + /** Mark only generated settings; stock debug/additional arguments remain untouched. */ + public static Map generatedProperties(Map generated) { + ensureInitialized(); + Map properties = new HashMap<>(); + for (String key : CONFIG.stringPropertyNames()) { + properties.put(key, CONFIG.getProperty(key)); + } + for (Map.Entry entry : generated.entrySet()) { + String environmentKey = + entry.getKey().toUpperCase(Locale.ROOT).replace('.', '_').replace('-', '_'); + // A resolved environment/file credential must never be promoted into a JVM argument. + // Explicit caller JVM properties keep their original source role. + if (!SENSITIVE.contains(environmentKey) || CONFIG.containsKey(entry.getKey())) { + properties.put(entry.getKey(), entry.getValue()); + } + } + if (generatedLogFile != null) { + properties.remove("datadog.trace.observer.slf4j.simpleLogger.logFile"); + } + Map marked = new HashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getValue() != null) { + marked.put(GENERATED + entry.getKey(), entry.getValue()); + } + } + return marked; + } + + /** Encode after stock Gradle project-property substitution, preserving non-generated args. */ + public static Iterable childArguments(Iterable arguments) { + ensureInitialized(); + Map carrier = new HashMap<>(); + List result = new ArrayList<>(); + for (String argument : arguments) { + if (argument.startsWith("-D" + GENERATED)) { + int separator = argument.indexOf('='); + carrier.put( + argument.substring(2 + GENERATED.length(), separator), + argument.substring(separator + 1)); + } else { + result.add(argument); + } + } + // Environment is inherited in its normal namespaced role, never Base64-encoded in argv. + Map none = emptyMap(); + result.add("-D" + CHILD + "=" + encode(asList(none, none, LAUNCH, carrier))); + return result; + } + + static String encode(List> maps) { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(bytes); + out.writeInt(1); + for (Map map : maps) { + Map sorted = new TreeMap<>(map); + sorted.values().removeIf(value -> value == null); + out.writeInt(sorted.size()); + for (Map.Entry e : sorted.entrySet()) { + writeString(out, e.getKey()); + writeString(out, e.getValue()); + } + } + if (bytes.size() > LIMIT) { + throw new IllegalArgumentException("Observer envelope too large"); + } + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes.toByteArray()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static void writeString(DataOutputStream out, String value) throws IOException { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + out.writeInt(bytes.length); + out.write(bytes); + } + + static List> decode(String encoded) { + if (encoded.length() > LIMIT * 2) { + throw new IllegalArgumentException("Observer envelope too large"); + } + try { + byte[] decoded = Base64.getUrlDecoder().decode(encoded); + if (decoded.length > LIMIT) { + throw new IllegalArgumentException("Observer envelope too large"); + } + DataInputStream in = new DataInputStream(new ByteArrayInputStream(decoded)); + if (in.readInt() != 1) { + throw new IOException("version"); + } + List> maps = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + int count = in.readInt(); + if (count < 0 || count > 10000) { + throw new IOException("count"); + } + Map map = new HashMap<>(); + for (int j = 0; j < count; j++) { + if (map.put(readString(in), readString(in)) != null) { + throw new IOException("duplicate key"); + } + } + maps.add(map); + } + if (in.available() != 0) { + throw new IOException("trailing data"); + } + return maps; + } catch (IOException | IllegalArgumentException e) { + throw new IllegalArgumentException("Malformed observer child envelope", e); + } + } + + private static String readString(DataInputStream in) throws IOException { + int size = in.readInt(); + if (size < 0 || size > LIMIT || size > in.available()) { + throw new IOException("length"); + } + byte[] bytes = new byte[size]; + in.readFully(bytes); + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes)).toString(); + } + + public static boolean isOuterGradleExecution() { + return isOuterGradleExecution(Thread.currentThread().getStackTrace()); + } + + /** Fail closed: only the synchronous Gradle-owned outer engine launch is supported. */ + public static boolean isOuterGradleExecution(StackTraceElement[] stack) { + boolean gradle = false; + int engines = 0; + for (StackTraceElement frame : stack) { + if (frame + .getClassName() + .equals("org.junit.platform.launcher.core.EngineExecutionOrchestrator") + && frame.getMethodName().equals("executeEngine")) { + engines++; + } + if (frame + .getClassName() + .equals( + "org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestDefinitionProcessor$CollectThenExecuteTestDefinitionConsumer") + && frame.getMethodName().equals("processAllTestDefinitions")) { + gradle = true; + } + } + return gradle && engines == 1; + } +} diff --git a/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts index 04862485528..b71e68ce31b 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts @@ -80,6 +80,25 @@ tasks.withType().configureEach { timeout.set(Duration.of(20, ChronoUnit.MINUTES)) } +// Experimental namespace-isolated observer. No observer dependencies/configuration enter ordinary tests. +if (providers.gradleProperty("traceTracer").map { it.toBoolean() }.orElse(false).get()) { + require(!gradle.startParameter.isConfigurationCacheRequested) { + "The experimental Gradle observer does not support configuration cache" + } + require(!providers.gradleProperty("traceTracerIntakePort").isPresent && + !providers.gradleProperty("traceTracerDebug").isPresent) { + "Use ordinary namespaced tracer configuration instead of traceTracerIntakePort/traceTracerDebug" + } + val observerJar = datadog.gradle.plugin.observer.ObserverAgentSelection.validate() + tasks.withType().configureEach { + inputs.file(observerJar).withPathSensitivity(org.gradle.api.tasks.PathSensitivity.NONE) + inputs.property("observerConfiguration", provider { + datadog.gradle.plugin.observer.ObserverAgentSelection.configurationFingerprint() + }) + // The relocated stock Gradle service is the sole worker injector and module owner. + } +} + // Register a task "allTests" that depends on all non-latest and non-traceAgentTest Test tasks. // This is used when we only want to run the 'main' test sets. tasks.register("allTests") { diff --git a/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts index 047b64d326d..3b4cb8b1dc2 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts @@ -21,13 +21,18 @@ val agentJar = rootProject.layout.projectDirectory .file("buildSrc/modifiable-config-agent/build/libs/modifiable-config-agent.jar") .asFile +val observerArgument = datadog.gradle.plugin.observer.ObserverAgentSelection.attached()?.let { + "-javaagent:" + it.absolutePath +} + tasks.withType().configureEach { inputs.file(agentJar).withPathSensitivity(org.gradle.api.tasks.PathSensitivity.NONE) // Attach lazily so we can skip when the Test JVM already has another -javaagent. // doFirst prepends, so this runs after any -javaagent registered later via doFirst. doFirst { val foreignAgent = allJvmArgs.firstOrNull { - it.startsWith("-javaagent:") && !it.contains("modifiable-config-agent") + it.startsWith("-javaagent:") && !it.contains("modifiable-config-agent") && + it != observerArgument } if (foreignAgent != null) { logger.info( diff --git a/buildSrc/src/test/java/datadog/gradle/plugin/observer/ObserverConfigSourcesTest.java b/buildSrc/src/test/java/datadog/gradle/plugin/observer/ObserverConfigSourcesTest.java new file mode 100644 index 00000000000..826e25da978 --- /dev/null +++ b/buildSrc/src/test/java/datadog/gradle/plugin/observer/ObserverConfigSourcesTest.java @@ -0,0 +1,727 @@ +package datadog.gradle.plugin.observer; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.io.TempDir; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.MethodInsnNode; +import org.objectweb.asm.tree.MethodNode; + +/** Differential source tests without installing either tracer or reading host stable files. */ +@EnabledIfSystemProperty(named = "observer.test.artifact", matches = ".+") +class ObserverConfigSourcesTest { + private static final String STOCK = "datadog.trace."; + private static final String OBSERVER = "datadog.trace.observer.trace."; + + /** Only test environment transport is substituted, below the stock config helper. */ + public static class Environment { + public static Map values = new HashMap<>(); + + public static String getenv(String key) { + return values.get(key); + } + + public static Map getenv() { + return new HashMap<>(values); + } + } + + @Test + void defaultLoggerResourceIsPrivateButExplicitSelectionUsesOrdinaryKeys(@TempDir Path directory) + throws Exception { + Files.write( + directory.resolve("simplelogger.properties"), + "datadog.slf4j.simpleLogger.defaultLogLevel=ERROR\n".getBytes("ISO-8859-1")); + Properties original = (Properties) System.getProperties().clone(); + ClassLoader context = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader resources = + new URLClassLoader(new URL[] {directory.toUri().toURL()}, null)) { + Thread.currentThread().setContextClassLoader(resources); + for (boolean explicit : new boolean[] {false, true}) { + String key = "tracing.observer.config.datadog.slf4j.simpleLogger.configurationFile"; + if (explicit) { + System.setProperty(key, "simplelogger.properties"); + } else { + System.clearProperty(key); + } + try (SourceLoader loader = loader(true)) { + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + Properties properties = (Properties) runtime.getMethod("getProperties").invoke(null); + Class settings = loader.loadClass(OBSERVER + "logging.simplelogger.SLCompatSettings"); + Object instance = settings.getConstructor(Properties.class).newInstance(properties); + Map description = + (Map) settings.getMethod("getSettingsDescription").invoke(instance); + assertEquals(explicit ? "ERROR" : "INFO", description.get("defaultLogLevel")); + assertEquals( + explicit ? "simplelogger.properties" : "observer-simplelogger.properties", + description.get("configurationFile")); + } + } + } finally { + System.setProperties(original); + Thread.currentThread().setContextClassLoader(context); + } + } + + @Test + void stockFactoriesAndPrivateSourcesAgree(@TempDir Path directory) throws Exception { + Path file = directory.resolve("observer.properties"); + Files.write( + file, + ("dd.service=file-service\ndd.trace.sample.rate=0.25\ndd.tags=file:yes,shared:file\ndd.api-key=file-dummy\n") + .getBytes("ISO-8859-1")); + for (boolean withFile : new boolean[] {false, true}) { + for (String factory : + new String[] {"createDefault", "withoutCollector", "withPropertiesOverride"}) { + Map stock = evaluate(false, factory, withFile ? file : null); + Map observer = evaluate(true, factory, withFile ? file : null); + assertEquals(stock, observer, factory + " file=" + withFile); + assertEquals( + factory.equals("withPropertiesOverride") && withFile ? "provided" : "high-alias", + observer.get("service")); + assertEquals(71, observer.get("numeric")); + assertEquals(false, observer.get("boolean")); + assertEquals("env-dummy", observer.get("key")); + assertEquals(!factory.equals("withoutCollector"), observer.get("collect")); + @SuppressWarnings("unchecked") + Map tags = (Map) observer.get("tags"); + assertEquals("property", tags.get("shared")); + assertEquals("yes", tags.get("env")); + if (withFile) { + assertEquals("yes", tags.get("file")); + } + } + } + } + + @Test + void propertyCredentialsRemainExcludedAndExplicitFilesAreReread(@TempDir Path directory) + throws Exception { + Path file = directory.resolve("observer.properties"); + Files.write(file, "dd.api-key=file-dummy\n".getBytes("ISO-8859-1")); + Properties original = (Properties) System.getProperties().clone(); + try (SourceLoader loader = loader(true)) { + System.setProperty("tracing.observer.config.dd.api-key", "ignored-property-dummy"); + System.setProperty("dd.trace.config", "/subject-file-must-not-be-read"); + Class provider = loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object first = provider.getMethod("createDefault").invoke(null); + Class system = + loader.loadClass(OBSERVER + "bootstrap.config.provider.SystemPropertiesConfigSource"); + assertNull( + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(first, "api-key", null, system, new String[0])); + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + runtime + .getMethod("setProperty", String.class, String.class) + .invoke(null, "dd.trace.config", file.toString()); + Object second = provider.getMethod("createDefault").invoke(null); + assertEquals( + "file-dummy", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(second, "api-key", null, system, new String[0])); + Files.write(file, "dd.api-key=file-dummy-2\n".getBytes("ISO-8859-1")); + Object third = provider.getMethod("createDefault").invoke(null); + assertEquals( + "file-dummy-2", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(third, "api-key", null, system, new String[0])); + assertEquals("/subject-file-must-not-be-read", System.getProperty("dd.trace.config")); + Class stable = loader.loadClass(OBSERVER + "bootstrap.config.provider.StableConfigSource"); + assertEquals( + java.util.Collections.emptySet(), + stable.getMethod("getKeys").invoke(stable.getField("LOCAL").get(null))); + assertEquals( + java.util.Collections.emptySet(), + stable.getMethod("getKeys").invoke(stable.getField("FLEET").get(null))); + } finally { + System.setProperties(original); + } + } + + @Test + void ordinaryDestinationsProductsRoleControlsAndFilesAreNotRestricted(@TempDir Path directory) + throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path key = directory.resolve("owned-dummy.key"); + Files.write(key, "ordinary-offline-dummy".getBytes("UTF-8")); + Map values = new LinkedHashMap<>(); + values.put("site", "datad0g.com"); + values.put("civisibility.agentless.url", "https://example.invalid/tests"); + values.put("trace.agent.url", "https://example.invalid/agent"); + values.put("api-key-file", key.toString()); + values.put("application-key-file", key.toString()); + values.put("profiling.apikey.file", key.toString()); + values.put("profiling.enabled", "true"); + values.put("civisibility.auto.configuration.enabled", "false"); + values.put("civisibility.build.instrumentation.enabled", "false"); + values.put("writer.type", "DDAgentWriter"); + try { + for (String source : new String[] {"property", "environment", "file", "override"}) { + System.setProperties((Properties) original.clone()); + Properties fileValues = new Properties(); + Map environment = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + String full = "dd." + entry.getKey(); + if (source.equals("property")) { + System.setProperty("tracing.observer.config." + full, entry.getValue()); + } else if (source.equals("environment")) { + environment.put( + "TRACING_OBSERVER_CONFIG_" + + full.toUpperCase(Locale.ROOT).replace('.', '_').replace('-', '_'), + entry.getValue()); + } else { + fileValues.setProperty(source.equals("file") ? full : entry.getKey(), entry.getValue()); + } + } + Path config = directory.resolve("ordinary.properties"); + if (source.equals("file")) { + try (java.io.OutputStream out = Files.newOutputStream(config)) { + fileValues.store(out, "offline owned inputs"); + } + System.setProperty("tracing.observer.config.dd.trace.config", config.toString()); + } + try (SourceLoader loader = loader(true)) { + loader.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class provider = + loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object instance = + source.equals("override") + ? provider + .getMethod("withPropertiesOverride", Properties.class) + .invoke(null, fileValues) + : provider.getMethod("createDefault").invoke(null); + for (Map.Entry entry : values.entrySet()) { + assertEquals( + entry.getValue(), + configString(provider, instance, entry.getKey()), + source + " " + entry.getKey()); + } + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void childTransportKeepsCarrierAndInheritedEnvironmentWithoutSerializingCredentials( + @TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path file = directory.resolve("roundtrip.properties"); + Files.write(file, "dd.service=file-parent\n".getBytes("ISO-8859-1")); + Map environment = new HashMap<>(); + environment.put("TRACING_OBSERVER_CONFIG_DD_ENV", "private-environment"); + environment.put("TRACING_OBSERVER_CONFIG_DD_API_KEY", "owned-secret-marker"); + try (SourceLoader parent = loader(true)) { + System.clearProperty("tracing.observer.child.v1"); + System.setProperty("tracing.observer.config.dd.trace.config", file.toString()); + parent.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class runtime = parent.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + initializeParent(runtime); + Map generated = new HashMap<>(); + generated.put("dd.civisibility.build.instrumentation.enabled", "false"); + generated.put("traceparent", "private-parent-context"); + generated.put("tracestate", "dd=s:1"); + generated.put("baggage", "a=b"); + generated.put("dd.api-key", "owned-secret-marker"); + @SuppressWarnings("unchecked") + Map marked = + (Map) + runtime.getMethod("generatedProperties", Map.class).invoke(null, generated); + assertTrue(marked.values().stream().noneMatch("owned-secret-marker"::equals)); + List stockArguments = new java.util.ArrayList<>(); + marked.forEach((key, value) -> stockArguments.add("-D" + key + "=" + value)); + stockArguments.add("-javaagent:owned.jar"); + stockArguments.add("-Xmx256m"); + @SuppressWarnings("unchecked") + List arguments = + (List) + runtime.getMethod("childArguments", Iterable.class).invoke(null, stockArguments); + assertEquals( + stockArguments.subList(stockArguments.size() - 2, stockArguments.size()), + arguments.subList(0, 2)); + String child = arguments.get(2).substring("-Dtracing.observer.child.v1=".length()); + assertTrue( + !new String(java.util.Base64.getUrlDecoder().decode(child), "UTF-8") + .contains("owned-secret-marker")); + Files.write(file, "dd.service=file-worker\n".getBytes("ISO-8859-1")); + System.setProperty("tracing.observer.child.v1", child); + try (SourceLoader worker = loader(true)) { + worker.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class provider = worker.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object config = provider.getMethod("createDefault").invoke(null); + assertEquals("private-environment", configString(provider, config, "env")); + assertEquals("file-worker", configString(provider, config, "service")); + assertEquals("owned-secret-marker", configString(provider, config, "api-key")); + assertEquals( + "false", configString(provider, config, "civisibility.build.instrumentation.enabled")); + Class workerRuntime = + worker.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + Map carrier = + (Map) workerRuntime.getMethod("propagationProperties").invoke(null); + assertEquals("private-parent-context", carrier.get("traceparent")); + assertEquals("dd=s:1", carrier.get("tracestate")); + assertEquals("a=b", carrier.get("baggage")); + } + } finally { + System.setProperties(original); + } + } + + @Test + void artifactRetainsCatalogCapabilitiesAndStockGradleArgumentBody() throws Exception { + try (java.util.jar.JarFile stock = + new java.util.jar.JarFile(System.getProperty("observer.test.stock")); + java.util.jar.JarFile observer = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + java.io.DataInputStream stockIndex = + new java.io.DataInputStream( + stock.getInputStream(stock.getJarEntry("inst/instrumenter.index"))); + java.io.DataInputStream observerIndex = + new java.io.DataInputStream( + observer.getInputStream(observer.getJarEntry("inst/observer-instrumenter.index"))); + int modules = stockIndex.readInt(); + assertTrue(modules > 6); + assertEquals(modules, observerIndex.readInt()); + assertEquals(stockIndex.readInt(), observerIndex.readInt()); + for (String[] seam : + new String[][] { + {"junit5/JUnitPlatformUtils", "capabilities"}, + {"gradle/CiVisibilityService", "getTracerJvmArgs"}, + {"gradle/CiVisibilityPluginExtension", "applyJacocoSettings"}, + {"gradle/TracerArgumentsProvider", "replaceProjectProperties"} + }) { + List> bodies = new java.util.ArrayList<>(); + for (boolean relocated : new boolean[] {false, true}) { + java.util.jar.JarFile jar = relocated ? observer : stock; + String resource = + "inst/datadog/" + + (relocated ? "trace/observer/" : "") + + "trace/instrumentation/" + + seam[0] + + ".classdata"; + ClassNode node = new ClassNode(); + new ClassReader(SourceLoader.read(jar.getInputStream(jar.getJarEntry(resource)))) + .accept(node, 0); + MethodNode method = + node.methods.stream().filter(m -> m.name.equals(seam[1])).findFirst().get(); + List body = new java.util.ArrayList<>(); + for (AbstractInsnNode instruction : method.instructions) { + if (!(instruction instanceof MethodInsnNode + && ((MethodInsnNode) instruction).name.equals("generatedProperties"))) { + body.add(instruction.getOpcode()); + } + } + bodies.add(body); + } + assertEquals(bodies.get(0), bodies.get(1), seam[0] + "." + seam[1]); + } + } + } + + @Test + void artifactKeepsExternalLiteralsThatLookLikePackages() throws Exception { + try (java.util.jar.JarFile observer = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + for (String[] expected : + new String[][] { + { + "datadog/trace/observer/trace/api/ConfigDefaults.class", "/var/run/datadog/apm.socket" + }, + {"datadog/trace/observer/trace/api/Config.class", ".inject.datadog.attribute.enabled"}, + { + "trace/datadog/trace/observer/trace/agent/core/otlp/metrics/OtlpStatsMetricWriter.classdata", + "datadog.origin" + }, + { + "logs-intake/datadog/trace/observer/trace/logging/intake/LogsWriterImpl.classdata", + "datadog.product:" + } + }) { + java.util.jar.JarEntry entry = observer.getJarEntry(expected[0]); + assertTrue(entry != null, expected[0]); + List constants = new java.util.ArrayList<>(); + ClassNode node = new ClassNode(); + new ClassReader(SourceLoader.read(observer.getInputStream(entry))).accept(node, 0); + node.fields.forEach(field -> constants.add(field.value)); + for (MethodNode method : node.methods) { + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof org.objectweb.asm.tree.LdcInsnNode) { + constants.add(((org.objectweb.asm.tree.LdcInsnNode) instruction).cst); + } + } + } + assertTrue(constants.contains(expected[1]), expected[0] + " " + expected[1]); + } + } + } + + @Test + void configReadsOwnedCredentialFilesWithStockSemantics(@TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path key = directory.resolve("owned-dummy.key"); + Files.write(key, "offline-credential-dummy\n".getBytes("UTF-8")); + try { + for (String spelling : + new String[] { + "dd.api-key-file", "dd.profiling.api-key-file", "dd.profiling.apikey.file" + }) { + System.setProperties((Properties) original.clone()); + System.setProperty("tracing.observer.config." + spelling, key.toString()); + try (SourceLoader loader = loader(true)) { + Class config = loader.loadClass(OBSERVER + "api.Config"); + Object instance = config.getMethod("get").invoke(null); + assertEquals( + "offline-credential-dummy", config.getMethod("getApiKey").invoke(instance), spelling); + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void stockAgentArgumentsAndOtelFileInputsRemainPrivate(@TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path otel = directory.resolve("owned-otel.properties"); + Files.write( + otel, + "otel.service.name=otel-file\notel.exporter.otlp.endpoint=https://example.invalid\n" + .getBytes("ISO-8859-1")); + try (SourceLoader loader = loader(true)) { + System.setProperty("tracing.observer.config.dd.trace.otel.enabled", "true"); + System.setProperty( + "tracing.observer.config.otel.javaagent.configuration-file", otel.toString()); + System.setProperty("dd.service", "subject"); + Class provider = loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object initial = provider.getMethod("createDefault").invoke(null); + assertEquals("true", configString(provider, initial, "trace.otel.enabled")); + assertEquals("otel-file", configString(provider, initial, "service.name")); + Class injector = + loader.loadClass(OBSERVER + "bootstrap.config.provider.AgentArgsInjector"); + injector + .getMethod("injectAgentArgsConfig", String.class) + .invoke(null, "dd.service=ordinary,dd.site=datad0g.com"); + Object config = provider.getMethod("createDefault").invoke(null); + assertEquals("ordinary", configString(provider, config, "service")); + assertEquals("datad0g.com", configString(provider, config, "site")); + assertEquals("subject", System.getProperty("dd.service")); + } finally { + System.setProperties(original); + } + } + + @Test + void generatedSettingsUseStockPlaceholderSubstitutionBeforeEncoding() throws Exception { + String name = OBSERVER + "instrumentation.gradle.TracerArgumentsProvider"; + ClassNode node = new ClassNode(); + try (java.util.jar.JarFile jar = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + new ClassReader( + SourceLoader.read( + jar.getInputStream( + jar.getJarEntry("inst/" + name.replace('.', '/') + ".classdata")))) + .accept(node, 0); + } + // Make only the abstract Gradle service accessor concrete so the real substitution method + // can be exercised without starting Gradle or either tracer. + node.access &= ~Opcodes.ACC_ABSTRACT; + MethodNode service = + node.methods.stream() + .filter(m -> m.name.equals("getCiVisibilityService")) + .findFirst() + .get(); + service.access &= ~Opcodes.ACC_ABSTRACT; + service.instructions.add(new org.objectweb.asm.tree.InsnNode(Opcodes.ACONST_NULL)); + service.instructions.add(new org.objectweb.asm.tree.InsnNode(Opcodes.ARETURN)); + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + node.accept(writer); + class FixtureLoader extends ClassLoader { + FixtureLoader() { + super(ObserverConfigSourcesTest.class.getClassLoader()); + } + + Class define() { + byte[] bytes = writer.toByteArray(); + return defineClass(name, bytes, 0, bytes.length); + } + } + Class provider = new FixtureLoader().define(); + Object instance = + provider + .getConstructor(String.class, Map.class) + .newInstance(":test", singletonMap("fixture", "resolved $ value")); + Method replace = provider.getDeclaredMethod("replaceProjectProperties", String.class); + replace.setAccessible(true); + try (SourceLoader loader = loader(true)) { + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + @SuppressWarnings("unchecked") + Map marked = + (Map) + runtime + .getMethod("generatedProperties", Map.class) + .invoke(null, singletonMap("dd.tags", "tag:${fixture}")); + List arguments = new java.util.ArrayList<>(); + for (Map.Entry entry : marked.entrySet()) { + arguments.add( + (String) replace.invoke(instance, "-D" + entry.getKey() + "=" + entry.getValue())); + } + arguments.add((String) replace.invoke(instance, "-Dadditional=${fixture}")); + @SuppressWarnings("unchecked") + List result = + (List) + runtime.getMethod("childArguments", Iterable.class).invoke(null, arguments); + assertEquals("-Dadditional=resolved $ value", result.get(0)); + String decoded = + new String( + java.util.Base64.getUrlDecoder() + .decode(result.get(1).substring("-Dtracing.observer.child.v1=".length())), + "UTF-8"); + assertTrue(decoded.contains("tag:resolved $ value")); + assertTrue(!decoded.contains("${fixture}")); + } + MethodNode asArguments = + node.methods.stream().filter(m -> m.name.equals("asArguments")).findFirst().get(); + List calls = new java.util.ArrayList<>(); + for (AbstractInsnNode instruction : asArguments.instructions) { + if (instruction instanceof MethodInsnNode) { + calls.add(((MethodInsnNode) instruction).name); + } + } + assertEquals("collect", calls.get(calls.size() - 2)); + assertEquals("childArguments", calls.get(calls.size() - 1)); + } + + private static void initializeParent(Class runtime) throws Exception { + Method encode = runtime.getDeclaredMethod("encode", List.class); + encode.setAccessible(true); + String envelope = + (String) encode.invoke(null, asList(emptyMap(), emptyMap(), emptyMap(), emptyMap())); + runtime.getMethod("initializePremain", String.class).invoke(null, "v1:" + envelope); + } + + private static Object configString(Class provider, Object config, String key) + throws Exception { + return provider.getMethod("getString", String.class).invoke(config, key); + } + + private Map evaluate(boolean observer, String factory, Path file) + throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try (SourceLoader loader = loader(observer)) { + String prefix = observer ? "tracing.observer.config." : ""; + System.setProperty(prefix + "dd.service.name", "high-alias"); + System.setProperty(prefix + "dd.observer.probe.integer", "invalid-number"); + System.setProperty(prefix + "dd.observer.probe.boolean", "invalid-boolean"); + System.setProperty(prefix + "dd.tags", "prop:yes,shared:property"); + System.setProperty(prefix + "dd.api-key", "ignored-property-dummy"); + if (file != null) { + System.setProperty(prefix + "dd.trace.config", file.toString()); + } + Map env = new HashMap<>(); + String ep = observer ? "TRACING_OBSERVER_CONFIG_" : ""; + env.put(ep + "DD_SERVICE", "env-service"); + env.put(ep + "DD_OBSERVER_PROBE_INTEGER", "71"); + env.put(ep + "DD_OBSERVER_PROBE_BOOLEAN", "true"); + env.put(ep + "DD_TAGS", "env:yes,shared:environment"); + env.put(ep + "DD_API_KEY", "env-dummy"); + env.put(ep + "DD_APP_KEY", "app-dummy"); + loader.loadClass(Environment.class.getName()).getField("values").set(null, env); + String name = observer ? OBSERVER : STOCK; + Class provider = loader.loadClass(name + "bootstrap.config.provider.ConfigProvider"); + Properties provided = new Properties(); + provided.setProperty("service", "provided"); + Object instance = + factory.equals("withPropertiesOverride") + ? provider.getMethod(factory, Properties.class).invoke(null, provided) + : provider.getMethod(factory).invoke(null); + Map result = new LinkedHashMap<>(); + result.put( + "service", + provider + .getMethod("getString", String.class, String.class, String[].class) + .invoke(instance, "service", null, new String[] {"service.name"})); + result.put( + "numeric", + provider + .getMethod("getInteger", String.class, int.class, String[].class) + .invoke(instance, "observer.probe.integer", 9, new String[0])); + result.put( + "boolean", + provider + .getMethod("getBoolean", String.class, boolean.class, String[].class) + .invoke(instance, "observer.probe.boolean", true, new String[0])); + result.put( + "tags", + provider + .getMethod("getMergedMap", String.class, String[].class) + .invoke(instance, "tags", new String[0])); + Class system = + loader.loadClass(name + "bootstrap.config.provider.SystemPropertiesConfigSource"); + result.put( + "key", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(instance, "api-key", null, system, new String[0])); + result.put( + "applicationKey", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(instance, "application-key", null, system, new String[] {"app-key"})); + assertEquals("app-dummy", result.get("applicationKey")); + Field collect = provider.getDeclaredField("collectConfig"); + collect.setAccessible(true); + result.put("collect", collect.get(instance)); + return result; + } finally { + System.setProperties(original); + } + } + + private SourceLoader loader(boolean observer) throws Exception { + Path artifact = Paths.get(System.getProperty("observer.test.artifact")); + Path stock = Paths.get(System.getProperty("observer.test.stock")); + return new SourceLoader(observer, observer ? artifact : stock, artifact); + } + + private static class SourceLoader extends URLClassLoader { + private final boolean observer; + private final Path artifact; + + SourceLoader(boolean observer, Path jar, Path artifact) throws Exception { + super( + new URL[] { + jar.toUri().toURL(), + Environment.class.getProtectionDomain().getCodeSource().getLocation() + }, + null); + this.observer = observer; + this.artifact = artifact; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + String resource = name.replace('.', '/') + ".class"; + try { + byte[] bytes; + if (!observer && name.equals(STOCK + "bootstrap.config.provider.StableConfigSource")) { + // Only the host-file seam is disabled in the stock oracle. All factory/parsing code is + // stock. + try (java.util.jar.JarFile jar = new java.util.jar.JarFile(artifact.toFile())) { + bytes = + read( + jar.getInputStream( + jar.getJarEntry(resource.replace("datadog/", "datadog/trace/observer/")))); + } + ClassWriter writer = new ClassWriter(0); + new ClassReader(bytes) + .accept( + new ClassRemapper( + writer, + new Remapper(Opcodes.ASM9) { + @Override + public String map(String value) { + return value.replace("datadog/trace/observer/", "datadog/"); + } + }), + 0); + bytes = writer.toByteArray(); + } else { + URL url = findResource(resource); + if (url == null) { + throw new ClassNotFoundException(name); + } + try (InputStream in = url.openStream()) { + bytes = read(in); + } + } + ClassNode node = new ClassNode(); + new ClassReader(bytes).accept(node, 0); + if (!name.equals(Environment.class.getName())) { + for (MethodNode method : node.methods) { + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("java/lang/System") && call.name.equals("getenv")) { + call.owner = Environment.class.getName().replace('.', '/'); + } + } + } + } + } + ClassWriter writer = new ClassWriter(0); + node.accept(writer); + bytes = writer.toByteArray(); + return defineClass(name, bytes, 0, bytes.length); + } catch (Exception e) { + throw new ClassNotFoundException(name, e); + } + } + + private static byte[] read(InputStream in) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int n; + while ((n = in.read(buffer)) != -1) { + bytes.write(buffer, 0, n); + } + return bytes.toByteArray(); + } + } +} diff --git a/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java b/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java new file mode 100644 index 00000000000..065a4a63197 --- /dev/null +++ b/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java @@ -0,0 +1,202 @@ +package datadog.trace.observer.bootstrap; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class ObserverRuntimeTest { + @Test + void defaultOffDoesNotSelectAnApplicationClasspathObserver() throws Exception { + assertNull(datadog.gradle.plugin.observer.ObserverAgentSelection.attached()); + assertThrows( + IllegalArgumentException.class, + datadog.gradle.plugin.observer.ObserverAgentSelection::validate); + } + + @Test + void barePremainAndOrdinaryArgumentsNeedNoRoleOrArtifactMetadata() throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try { + System.clearProperty("tracing.observer.child.v1"); + for (String arguments : new String[] {null, "", "dd.service=ordinary"}) { + try (URLClassLoader loader = + new URLClassLoader( + new URL[] { + ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation() + }, + null)) { + Class runtime = loader.loadClass(ObserverRuntime.class.getName()); + runtime.getMethod("initializePremain", String.class).invoke(null, arguments); + assertNull(runtime.getMethod("getProperty", String.class).invoke(null, "dd.writer.type")); + assertNull( + runtime + .getMethod("getProperty", String.class) + .invoke(null, "dd.civisibility.enabled")); + assertTrue( + ((Map) runtime.getMethod("propagationProperties").invoke(null)).isEmpty()); + assertThrows( + java.lang.reflect.InvocationTargetException.class, + () -> runtime.getMethod("initializePremain", String.class).invoke(null, arguments)); + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void onlyEffectiveGeneratedFileLoggerSkipsDaemonStreamReset() throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try { + System.setProperty("tracing.observer.log.directory", "owned-logs"); + for (String destination : + new String[] {"generated", "System.err", "System.out", "custom.log"}) { + String key = "tracing.observer.config.datadog.slf4j.simpleLogger.logFile"; + System.clearProperty(key); + if (!destination.equals("generated")) { + System.setProperty(key, destination); + } + try (URLClassLoader loader = + new URLClassLoader( + new URL[] { + ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation() + }, + null)) { + Class runtime = loader.loadClass(ObserverRuntime.class.getName()); + assertEquals( + destination.equals("generated"), + runtime.getMethod("usingGeneratedFileLogger").invoke(null)); + runtime + .getMethod("setProperty", String.class, String.class) + .invoke(null, "datadog.trace.observer.slf4j.simpleLogger.logFile", "System.err"); + assertEquals(false, runtime.getMethod("usingGeneratedFileLogger").invoke(null)); + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void wholeCarrierRoundTripIsOpaqueAndLossless() { + Map values = new HashMap<>(); + values.put("traceparent", "00-1234-5678-01"); + values.put("tracestate", "dd=s:1"); + values.put("x-datadog-parent-id", "18446744073709551615"); + values.put("baggage", "space=\"quoted\",unicode=\u03b1\n${projectProperty}"); + List> maps = + asList(values, singletonMap("DD_TAGS", "a:b c"), emptyMap(), values); + String encoded = ObserverRuntime.encode(maps); + assertFalse(encoded.contains("${")); + assertEquals(maps, ObserverRuntime.decode(encoded)); + assertThrows(IllegalArgumentException.class, () -> ObserverRuntime.decode("invalid")); + assertThrows(IllegalArgumentException.class, () -> ObserverRuntime.decode(encoded + "AA")); + } + + @Test + void independentSourceSnapshotsAndMutablePrivateOverlay() throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try { + Map config = new HashMap<>(); + config.put("dd.service", "private-service"); + config.put("future.non.dd.key", "future-value"); + Map carrier = new HashMap<>(); + carrier.put("traceparent", "private-carrier"); + carrier.put("dd.civisibility.build.instrumentation.enabled", "false"); + carrier.put("dd.civisibility.auto.configuration.enabled", "false"); + System.setProperty( + "tracing.observer.child.v1", + ObserverRuntime.encode( + asList(config, singletonMap("DD_TAGS", "source:environment"), emptyMap(), carrier))); + System.setProperty("dd.service", "subject-service"); + System.setProperty("traceparent", "subject-carrier"); + try (URLClassLoader loader = + new URLClassLoader( + new URL[] {ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation()}, + null)) { + Class runtime = loader.loadClass(ObserverRuntime.class.getName()); + assertEquals( + "private-service", + runtime.getMethod("getProperty", String.class).invoke(null, "dd.service")); + assertEquals( + "source:environment", + runtime.getMethod("getenv", String.class).invoke(null, "DD_TAGS")); + assertNull(runtime.getMethod("getProperty", String.class).invoke(null, "dd.tags")); + System.setProperty("dd.service", "subject-mutated"); + assertEquals( + "private-service", + runtime.getMethod("getProperty", String.class).invoke(null, "dd.service")); + Properties snapshot = (Properties) runtime.getMethod("getProperties").invoke(null); + assertEquals("future-value", snapshot.getProperty("future.non.dd.key")); + snapshot.setProperty("dd.service", "copy-mutated"); + assertEquals( + "private-service", + runtime.getMethod("getProperty", String.class).invoke(null, "dd.service")); + assertEquals(carrier, runtime.getMethod("propagationProperties").invoke(null)); + assertEquals("subject-carrier", System.getProperty("traceparent")); + assertNull( + runtime + .getMethod("setProperty", String.class, String.class) + .invoke(null, "dd.civisibility.signal.server.port", "12345")); + assertEquals( + "12345", + runtime + .getMethod("getProperty", String.class) + .invoke(null, "dd.civisibility.signal.server.port")); + assertNull(System.getProperty("dd.civisibility.signal.server.port")); + assertEquals( + "12345", + runtime + .getMethod("clearProperty", String.class) + .invoke(null, "dd.civisibility.signal.server.port")); + assertNull( + runtime + .getMethod("getProperty", String.class) + .invoke(null, "dd.civisibility.signal.server.port")); + } + } finally { + System.setProperties(original); + } + } + + @Test + void fullNameNamespaceDoesNotWhitelistKeysOrRewriteValues() throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try { + System.setProperty("tracing.observer.config.dd.future-key", "datadog.value=\u03b1"); + System.setProperty( + "tracing.observer.config.datadog.slf4j.simpleLogger.defaultLogLevel", "WARN"); + try (URLClassLoader loader = + new URLClassLoader( + new URL[] {ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation()}, + null)) { + Class runtime = loader.loadClass(ObserverRuntime.class.getName()); + assertEquals( + "datadog.value=\u03b1", + runtime.getMethod("getProperty", String.class).invoke(null, "dd.future-key")); + assertEquals( + "WARN", + runtime + .getMethod("getProperty", String.class) + .invoke(null, "datadog.trace.observer.slf4j.simpleLogger.defaultLogLevel")); + assertNull(System.getProperty("dd.future-key")); + assertTrue(((Map) runtime.getMethod("propagationProperties").invoke(null)).isEmpty()); + } + } finally { + System.setProperties(original); + } + } +} diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentRewriterTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentRewriterTest.kt new file mode 100644 index 00000000000..330be261928 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentRewriterTest.kt @@ -0,0 +1,212 @@ +package datadog.gradle.plugin.observer + +import datadog.trace.observer.bootstrap.ObserverRuntime +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.ClassNode +import org.objectweb.asm.tree.MethodInsnNode +import java.io.ByteArrayOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream +import java.nio.file.Path +import java.util.Properties +import java.util.jar.Attributes +import java.util.jar.JarOutputStream +import java.util.jar.Manifest + +class ObserverAgentRewriterTest { + @Test + fun `never overwrites the stock artifact`(@TempDir directory: Path) { + val input = directory.resolve("stock.jar").toFile() + input.writeText("unchanged") + assertThrows(IllegalArgumentException::class.java) { + ObserverAgentRewriter().rewrite(input, input) + } + assertEquals("unchanged", input.readText()) + } + + @Test + fun `refuses an incomplete stock layout before writing output`(@TempDir directory: Path) { + val input = directory.resolve("stock.jar").toFile() + val output = directory.resolve("observer.jar").toFile() + val manifest = Manifest() + manifest.mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" + manifest.mainAttributes.putValue("Premain-Class", "datadog.trace.bootstrap.AgentPreCheck") + JarOutputStream(input.outputStream(), manifest).use { } + val failure = assertThrows(IllegalArgumentException::class.java) { + ObserverAgentRewriter().rewrite(input, output) + } + assertTrue(failure.message!!.contains("Missing expected stock entry")) + assertFalse(output.exists()) + } + + @Test + fun `relocates bootstrap dependencies reflection and nested resources`() { + assertEquals("datadog/trace/observer/trace/api/Config", ObserverAgentRewriter.relocate("datadog/trace/api/Config")) + assertEquals("datadog.trace.observer.com.datadog.Foo", ObserverAgentRewriter.relocate("com.datadog.Foo")) + assertEquals("datadog.trace.observer.net.bytebuddy.Foo", ObserverAgentRewriter.relocate("net.bytebuddy.Foo")) + assertEquals("inst/observer-instrumenter.index", ObserverAgentRewriter.relocate("inst/instrumenter.index")) + assertEquals("org.junit.platform.engine.TestEngine", ObserverAgentRewriter.relocate("org.junit.platform.engine.TestEngine")) + } + + @Test + fun `keeps host paths config fragments and wire names unchanged`() { + listOf( + "/var/run/datadog/apm.socket", + "/var/run/datadog/dsd.socket", + ".inject.datadog.attribute.enabled", + "datadog.product:", + "datadog.tracer.stats.collapsed_spans", + "datadog.dogstatsd.client.bytes_sent", + "datadog.origin", + ).forEach { assertEquals(it, ObserverAgentRewriter.relocate(it)) } + assertEquals("datadog.trace.observer.span.dispatch", ObserverAgentRewriter.relocate("datadog.span.dispatch")) + } + + @Test + fun `logger keys relocate target class suffixes once without rewriting values`() { + val properties = Properties() + val keys = listOf( + "datadog.slf4j.simpleLogger.log.datadog.trace.api.Config", + "datadog.slf4j.simpleLogger.log.net.bytebuddy.ByteBuddy", + "datadog.slf4j.simpleLogger.log.com.datadog.Foo", + ) + keys.forEach { properties.setProperty(it, "datadog.value") } + val translated = ObserverRuntime.loggerProperties(properties) + keys.forEach { assertEquals("datadog.value", translated.getProperty(ObserverAgentRewriter.relocate(it))) } + assertEquals(translated, ObserverRuntime.loggerProperties(translated)) + } + + @Test + fun `retains the full catalog and validates packed index`() { + val packed = ByteArrayOutputStream() + val names = listOf("junit5.JUnit5Instrumentation", "junit5.JUnit5SpockInstrumentation", "junit5.JUnit5SkipInstrumentation", + "gradle.GradleBuildScopeServices_8_10_Instrumentation", "gradle.GradlePluginInjectorInstrumentation", + "gradle.GradleServiceValidationInstrumentation", "gradle.GradleDaemonLoggingInstrumentation") + DataOutputStream(packed).use { out -> + names.forEach { name -> + val fullName = "datadog.trace.instrumentation.$name" + out.writeByte(fullName.length) + out.writeBytes(fullName) + out.writeShort(64) + out.writeByte(1) // member target-system overrides + out.writeByte(255) // self membership + out.writeByte(1) + out.writeByte(6) + out.writeBytes("Advice") + out.writeShort(64) + } + } + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { + it.writeInt(names.size) + it.writeInt(names.size) + it.writeInt(packed.size()) + it.write(packed.toByteArray()) + } + val result = ObserverAgentRewriter.rewriteInstrumenterIndex(bytes.toByteArray()) + DataInputStream(result.inputStream()).use { + assertEquals(names.size, it.readInt()) + assertEquals(names.size, it.readInt()) + assertEquals(it.available() - 4, it.readInt()) + } + val text = String(result, Charsets.ISO_8859_1) + assertTrue(text.contains("datadog.trace.observer.trace.instrumentation.junit5.JUnit5Instrumentation")) + assertTrue(text.contains("JUnit5SkipInstrumentation")) + assertThrows(IllegalArgumentException::class.java) { + ObserverAgentRewriter.rewriteInstrumenterIndex(bytes.toByteArray() + 0) + } + } + + @Test + fun `context protocol names are isolated in definitions calls constants and dynamic names`() { + val writer = ClassWriter(0) + val owner = "datadog/trace/bootstrap/FieldBackedContextAccessor" + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, owner, null, "java/lang/Object", null) + writer.visitField(Opcodes.ACC_PUBLIC, "__datadogContext\$0", "Ljava/lang/Object;", null, null).visitEnd() + val method = writer.visitMethod(Opcodes.ACC_PUBLIC, "\$get\$__datadogContext\$", "()V", null, null) + method.visitCode() + method.visitLdcInsn("__datadogContext\$") + method.visitInsn(Opcodes.POP) + method.visitFieldInsn(Opcodes.GETSTATIC, owner, "__datadogContext\$0", "Ljava/lang/Object;") + method.visitInsn(Opcodes.POP) + method.visitMethodInsn(Opcodes.INVOKESTATIC, owner, "\$put\$__datadogContext\$", "()V", false) + method.visitInvokeDynamicInsn("\$get\$__datadogContext\$", "()V", org.objectweb.asm.Handle(Opcodes.H_INVOKESTATIC, owner, "\$put\$__datadogContext\$", "()V", false)) + method.visitInsn(Opcodes.RETURN) + method.visitMaxs(1, 1) + method.visitEnd() + val node = ClassNode() + ClassReader(ObserverAgentRewriter().rewriteClass(writer.toByteArray())).accept(node, 0) + assertEquals("__datadogObserverContext\$0", node.fields.single().name) + assertEquals("\$get\$__datadogObserverContext\$", node.methods.single().name) + val instructions = node.methods.single().instructions.toArray() + assertEquals("__datadogObserverContext\$", (instructions[0] as org.objectweb.asm.tree.LdcInsnNode).cst) + assertEquals("__datadogObserverContext\$0", (instructions[2] as org.objectweb.asm.tree.FieldInsnNode).name) + assertEquals("\$put\$__datadogObserverContext\$", (instructions[4] as MethodInsnNode).name) + val dynamic = instructions[5] as org.objectweb.asm.tree.InvokeDynamicInsnNode + assertEquals("\$get\$__datadogObserverContext\$", dynamic.name) + assertEquals("\$put\$__datadogObserverContext\$", dynamic.bsm.name) + } + + @Test + fun `private config ignores ordinary mutations and preserves JVM facts`() { + val original = System.getProperty("dd.writer.type") + try { + System.setProperty("dd.writer.type", "DDAgentWriter") + assertNull(ObserverRuntime.getProperty("dd.writer.type")) + assertNull(ObserverRuntime.getProperties().getProperty("dd.writer.type")) + assertEquals(System.getProperty("java.version"), ObserverRuntime.getProperty("java.version")) + assertNull(ObserverRuntime.getenv("DD_API_KEY")) + assertNull(ObserverRuntime.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + assertNull(ObserverRuntime.getProperty("dd.civisibility.itr.enabled")) + assertFalse(ObserverRuntime.getBoolean("dd.civisibility.itr.enabled")) + assertFalse(ObserverRuntime.getBoolean("dd.civisibility.enabled")) + assertFalse(ObserverRuntime.getBoolean(null)) + assertFalse(ObserverRuntime.getBoolean("")) + ObserverRuntime.getProperties().setProperty("dd.writer.type", "DDAgentWriter") + assertNull(ObserverRuntime.getProperty("dd.writer.type")) + } finally { + if (original == null) System.clearProperty("dd.writer.type") else System.setProperty("dd.writer.type", original) + } + } + + @Test + fun `private factories retain their bodies and caller overrides`() { + val writer = ClassWriter(0) + val provider = "datadog/trace/bootstrap/config/provider/ConfigProvider" + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, provider, null, "java/lang/Object", null) + writer.visitMethod(Opcodes.ACC_PUBLIC, "", "([L$provider\$Source;)V", null, null).visitEnd() + val method = writer.visitMethod(Opcodes.ACC_PUBLIC or Opcodes.ACC_STATIC, "withPropertiesOverride", "(Ljava/util/Properties;)L$provider;", null, null) + method.visitCode() + method.visitVarInsn(Opcodes.ALOAD, 0) + method.visitInsn(Opcodes.POP) + method.visitInsn(Opcodes.ACONST_NULL) + method.visitInsn(Opcodes.ARETURN) + method.visitMaxs(1, 1) + method.visitEnd() + writer.visitEnd() + val node = ClassNode() + ClassReader(ObserverAgentRewriter().rewriteClass(writer.toByteArray())).accept(node, 0) + val body = node.methods.single { it.name == "withPropertiesOverride" }.instructions.toArray() + assertEquals(listOf(Opcodes.ALOAD, Opcodes.POP, Opcodes.ACONST_NULL, Opcodes.ARETURN), body.map { it.opcode }) + } + + @Test + fun `only the outer Gradle engine launch owns observer listeners`() { + val engine = StackTraceElement("org.junit.platform.launcher.core.EngineExecutionOrchestrator", "executeEngine", "", 1) + val gradle = StackTraceElement("org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestDefinitionProcessor\$CollectThenExecuteTestDefinitionConsumer", "processAllTestDefinitions", "", 1) + assertTrue(ObserverRuntime.isOuterGradleExecution(arrayOf(engine, gradle))) + assertFalse(ObserverRuntime.isOuterGradleExecution(arrayOf(engine, engine, gradle))) + assertFalse(ObserverRuntime.isOuterGradleExecution(arrayOf(engine))) + assertFalse(ObserverRuntime.isOuterGradleExecution(arrayOf(gradle))) + assertFalse(ObserverRuntime.isOuterGradleExecution(emptyArray())) + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java index df6ade9a13e..9a4a9ea85da 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java @@ -1,10 +1,9 @@ package datadog.trace.civisibility.ci; -import static datadog.trace.civisibility.utils.FileUtils.findParentPathBackwards; - import datadog.trace.api.civisibility.telemetry.tag.Provider; import datadog.trace.api.git.GitInfo; import datadog.trace.civisibility.ci.env.CiEnvironment; +import java.nio.file.Files; import java.nio.file.Path; import javax.annotation.Nonnull; import org.slf4j.Logger; @@ -19,7 +18,7 @@ * *

However, we would like to provide git information if the user is using git, so we infer the * workspace path leveraging the `.git` folder, which is usually kept in the root path of the - * repository. + * repository. In linked worktrees and submodules, `.git` is a file pointing to the git directory. * *

The workspace path will be used in the CIProviderInfo constructor to access the `.git` folder * and calculate the git information properly. @@ -47,7 +46,7 @@ public GitInfo buildCIGitInfo() { @Override public CIInfo buildCIInfo() { - Path workspace = findParentPathBackwards(getCurrentPath(), getTargetFolder(), true); + Path workspace = findWorkspace(getCurrentPath(), getTargetFolder()); if (workspace == null) { return CIInfo.NOOP; } @@ -61,6 +60,19 @@ public CIInfo buildCIInfo() { return CIInfo.builder(environment).ciWorkspace(workspace.toAbsolutePath().toString()).build(); } + private static Path findWorkspace(Path current, String gitFolder) { + if (gitFolder == null || gitFolder.isEmpty()) { + return null; + } + for (Path path = current; path != null; path = path.getParent()) { + Path git = path.resolve(gitFolder); + if (Files.isDirectory(git) || Files.isRegularFile(git)) { + return path; + } + } + return null; + } + @Nonnull @Override public PullRequestInfo buildPullRequestInfo() { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java index bde89521dd5..2d5d5a3af09 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java @@ -255,8 +255,12 @@ public static final class Factory implements CoverageStore.Factory { public Factory(CiVisibilityMetricCollector metrics, SourcePathResolver sourcePathResolver) { this.metrics = metrics; this.sourcePathResolver = sourcePathResolver; + // Recording must not load classes: a covered defineClass hook would record again and recurse. + loadRecordingClasses(LineCoverageStore.class, LineProbes.class, ExecutionDataAdapter.class); } + private static void loadRecordingClasses(Class... classes) {} + @Override public CoverageStore create(@Nullable TestIdentifier testIdentifier) { return new LineCoverageStore( diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy index 652fecd470e..cc65de13acf 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy @@ -8,7 +8,10 @@ import datadog.trace.civisibility.ci.env.CiEnvironmentImpl import datadog.trace.civisibility.git.CILocalGitInfoBuilder import datadog.trace.civisibility.git.CIProviderGitInfoBuilder import datadog.trace.civisibility.git.tree.GitClient +import spock.lang.TempDir +import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths class UnknownCIInfoTest extends CITagsProviderTest { @@ -55,6 +58,24 @@ class UnknownCIInfoTest extends CITagsProviderTest { ciTags == expectedTags } + @TempDir + Path temporaryFolder + + def "test workspace is found from a worktree git file"() { + setup: + def worktree = Files.createDirectories(temporaryFolder.resolve("worktree")) + Files.write(worktree.resolve(GIT_FOLDER_FOR_TESTS), "gitdir: /repo/.git/worktrees/worktree\n".bytes) + def module = Files.createDirectories(worktree.resolve("module")) + + when: + def ciInfo = new CIProviderInfoFactory(Config.get(), GIT_FOLDER_FOR_TESTS, new CiEnvironmentImpl(env.getAll())) + .createCIProviderInfo(module) + .buildCIInfo() + + then: + ciInfo.ciWorkspace == worktree.toRealPath().toString() + } + def "test workspace is null if target folder does not exist"() { when: def gitClientFactory = Stub(GitClient.Factory) diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java index 51650b3d2b2..e67c42bbb0c 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java @@ -112,6 +112,9 @@ final class TypeFactory { byte[] targetBytecode; + /** Memoizes the supplied transform schema independently of shared classpath descriptions. */ + private LazyType targetType; + /** Sets the current class-loader context of this type-factory. */ void switchContext(ClassLoader classLoader) { if (currentClassLoader != classLoader || null == classFileLocator) { @@ -153,6 +156,7 @@ static void clear() { void beginTransform(String name, byte[] bytecode) { targetName = name; targetBytecode = bytecode; + targetType = new LazyType(name); if (installing) { originalClassLoader = currentClassLoader; @@ -186,6 +190,7 @@ void endTransform() { targetName = null; targetBytecode = null; + targetType = null; createOutlines = OUTLINING_ENABLED; } @@ -228,7 +233,7 @@ static TypeDescription findType(String name) { } private TypeDescription deferTypeResolution(String name) { - return deferredTypes.computeIfAbsent(name, deferType); + return name.equals(targetName) ? targetType : deferredTypes.computeIfAbsent(name, deferType); } /** Attempts to resolve the named type using the current context. */ @@ -236,7 +241,7 @@ TypeDescription resolveType(LazyType request) { if (null != classFileLocator) { TypeDescription result; if (createOutlines) { - if ("java.lang.Object".equals(request.name)) { + if (request != targetType && "java.lang.Object".equals(request.name)) { return objectOutline; } result = lookupType(request, outlineTypes, outlineTypeParser); @@ -258,6 +263,14 @@ private TypeDescription lookupType( boolean isOutline = typeParser == outlineTypeParser; long fromTick = InstrumenterMetrics.tick(); + // Earlier transformers may have changed this schema since it was cached from the classpath. + // Keep the supplied target bytes local to this transform, not in the shared type caches. + if (request == targetType) { + TypeDescription type = typeParser.parse(targetBytecode); + InstrumenterMetrics.buildTypeDescription(fromTick, isOutline); + return type; + } + // existing type description from same classloader? SharedTypeInfo sharedType = types.find(name); if (null != sharedType @@ -396,7 +409,7 @@ public TypeList.Generic getInterfaces() { @Override public boolean isPublic() { - return isPublicFilter.contains(name) || super.isPublic(); + return (this != targetType && isPublicFilter.contains(name)) || super.isPublic(); } private TypeDescription outline() { diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java new file mode 100644 index 00000000000..465d6e57e56 --- /dev/null +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java @@ -0,0 +1,126 @@ +package datadog.trace.agent.tooling.bytebuddy.outline; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.Serializable; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.Opcodes; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TypeFactoryTransformTest { + private TypeFactory factory; + + public static class Example { + public String originalField; + + public void originalMethod() {} + } + + @BeforeEach + void setUp() { + TypeFactory.clear(); + factory = new TypeFactory(); + TypeFactory.typeFactory.set(factory); + factory.beginInstall(); + factory.switchContext(Example.class.getClassLoader()); + } + + @AfterEach + void tearDown() { + factory.endTransform(); + factory.endInstall(); + TypeFactory.typeFactory.remove(); + TypeFactory.clear(); + } + + @Test + void transformBytesOverrideWarmOutlineAndFullCachesWithoutPoisoningOtherLookups() + throws Exception { + String name = Example.class.getName(); + byte[] originalBytes = ClassFileLocator.ForClassLoader.read(Example.class); + TypeDescription original = TypeFactory.findType(name); + assertTrue(original.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, original.getDeclaredFields().size()); + factory.disableFullDescriptions(); + + factory.beginTransform(name, withInterfaceAndField(originalBytes, "observerField")); + TypeDescription target = TypeFactory.findType(name); + assertAugmented(target, "observerField"); + factory.enableFullDescriptions(); + assertAugmented(target, "observerField"); + assertAugmented(TypeFactory.findType(name), "observerField"); + factory.endTransform(); + + // Original same-loader shared/classpath descriptions are not overwritten by transform bytes. + TypeDescription outsideTransform = TypeFactory.findType(name); + assertTrue(outsideTransform.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, outsideTransform.getDeclaredFields().size()); + factory.disableFullDescriptions(); + + factory.beginTransform(name, withInterfaceAndField(originalBytes, "nextField")); + TypeDescription next = TypeFactory.findType(name); + assertNotSame(target, next); + assertAugmented(next, "nextField"); + factory.enableFullDescriptions(); + assertAugmented(next, "nextField"); + assertTrue(next.getDeclaredFields().filter(named("observerField")).isEmpty()); + factory.endTransform(); + + factory.beginTransform(name, originalBytes); + TypeDescription restored = TypeFactory.findType(name); + assertTrue(restored.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, restored.getDeclaredFields().size()); + } + + private static void assertAugmented(TypeDescription type, String field) { + assertEquals( + Serializable.class.getName(), type.getInterfaces().getOnly().asErasure().getName()); + assertEquals(1, type.getDeclaredFields().filter(named(field)).size()); + assertEquals(1, type.getDeclaredMethods().filter(named("originalMethod")).size()); + } + + private static byte[] withInterfaceAndField(byte[] original, String field) { + ClassWriter writer = new ClassWriter(0); + new ClassReader(original) + .accept( + new ClassVisitor(Opcodes.ASM9, writer) { + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + super.visit( + version, + access, + name, + signature, + superName, + new String[] {"java/io/Serializable"}); + } + + @Override + public void visitEnd() { + super.visitField(Opcodes.ACC_PUBLIC, field, "Ljava/lang/Object;", null, null) + .visitEnd(); + super.visitEnd(); + } + }, + 0); + return writer.toByteArray(); + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 3fe773b73a4..eaebc6716ad 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -701,3 +701,9 @@ tasks.register('updateAgentJarIntegrationsGoldenFile', JavaExec) { tasks.named('check') { dependsOn 'verifyAgentJarContents', 'verifyAgentJarIntegrations' } + +// Developer-only artifact; never part of assemble, publication, or ordinary test execution. +tasks.register('observerJar', datadog.gradle.plugin.observer.ObserverAgentTask) { + stockAgent.set(tasks.named('shadowJar', ShadowJar).flatMap { it.archiveFile }) + observerAgent.set(layout.buildDirectory.file('observer/dd-observer-agent.jar')) +} diff --git a/docs/tracing_the_tracer.md b/docs/tracing_the_tracer.md new file mode 100644 index 00000000000..c103d61efa8 --- /dev/null +++ b/docs/tracing_the_tracer.md @@ -0,0 +1,186 @@ +# Tracing the tracer: experimental namespace-isolated agent + +The observer lets us trace this repository's own test runs with CI Visibility while +the tracer under test runs in the same JVMs. It is the stock Java agent relocated +into a private namespace. It keeps the full instrumenter catalog, capability +reporting and product configuration. Stock configuration decides what runs. + +Having the full catalog does not mean every integration or product can coexist with +the tracer under test. See [Limitations](#limitations). + +## Build + +The observer is a developer-only artifact. It is not part of `assemble`, publication +or ordinary test runs. + +```sh +./gradlew :dd-java-agent:observerJar +# Result: dd-java-agent/build/observer/dd-observer-agent.jar +``` + +The rewrite replaces the output atomically, so a JVM still running from the previous +jar is not affected. Copy the jar to a stable path before a long run anyway. + +Ordinary repository tests have no observer dependency or configuration. +`-PtraceTracer=true` only tracks the attached observer jar and configuration as test +inputs. It does not inject an agent or build the jar. + +## Injection + +Use ordinary premain. No role, hash or custom argument is required: + +```sh +java -javaagent:/absolute/path/dd-observer-agent.jar -jar application.jar +java -javaagent:/absolute/path/dd-observer-agent.jar=dd.service=observer -jar application.jar +``` + +Normal injection uses stock defaults and can start stock transports. Configure +destinations and products for your run. + +## Configuration + +Use full stock configuration names in the private namespace: + +| Input | Observer source | +| --- | --- | +| `-Dtracing.observer.config.dd.service=observer` | JVM property `dd.service` | +| `TRACING_OBSERVER_CONFIG_DD_SERVICE=observer` | Environment `DD_SERVICE` | +| `TRACING_OBSERVER_CONFIG_DD_API_KEY_FILE=/your/key-file` | Environment credential-file selector | +| `-Dtracing.observer.config.dd.trace.config=/your/observer.properties` | Stock properties-file selector | +| `TRACING_OBSERVER_CONFIG_OTEL_SERVICE_NAME=observer` | Stock OTEL environment source | + +Stock precedence, aliases, parsing, defaults, collection flags, caller overrides and +API-key property exclusion all apply. Credential files are reread, not snapshotted. +There is no allowlist for sites, endpoints, credentials, writers, products or OTEL. +Stock `Agent.configureCiVisibility` supplies the agent jar URI and CI defaults. +Ordinary properties files cannot activate early bootstrap product gates, as in stock. + +The observer does not import the subject's DD/OTEL properties, environment, +config-file selection or ambient propagation headers. JVM and CI platform facts are +shared. LOCAL and FLEET stable-config sources stay empty so the observer never opens +host `/etc/datadog-agent` files. + +The implicit logger resource is `observer-simplelogger.properties`. Explicit logger +resources use ordinary logger keys. Logger and Byte Buddy keys are relocated. Their +values are not. + +These are source-isolation boundaries. They do not protect against same-process +reflection. + +## Gradle daemon injection + +Attach the observer to the Gradle daemon, not only the wrapper JVM. Gradle applies +`-D` entries from `org.gradle.jvmargs` after premain, so inherited namespaced +environment variables are the simplest way to configure it. + +Example for running against your own staging account: + +```sh +: "${STAGING_API_KEY_FILE:?Set STAGING_API_KEY_FILE to your staging key file}" +OBSERVER="$PWD/dd-java-agent/build/observer/dd-observer-agent.jar" +TRACING_OBSERVER_CONFIG_DD_CIVISIBILITY_ENABLED=true \ +TRACING_OBSERVER_CONFIG_DD_TRACE_ENABLED=false \ +TRACING_OBSERVER_CONFIG_DD_CIVISIBILITY_AGENTLESS_ENABLED=true \ +TRACING_OBSERVER_CONFIG_DD_SITE=datad0g.com \ +TRACING_OBSERVER_CONFIG_DD_API_KEY_FILE="$STAGING_API_KEY_FILE" \ +TRACING_OBSERVER_CONFIG_DD_SERVICE=tracing-the-tracer \ +./gradlew --no-daemon --no-configuration-cache --no-scan \ + "-Dorg.gradle.jvmargs=-XX:MaxMetaspaceSize=1g -javaagent:$OBSERVER" \ + -PtraceTracer=true \ + :dd-java-agent:instrumentation-testing:test --tests AgentTestRunnerTest +``` + +The stock Gradle integration owns sessions, `Test` task modules and worker injection. +Workers receive the generated module settings privately. The subject's +`traceparent`, `tracestate`, baggage and Datadog headers cannot classify an observer +worker. Debug ports, extra JVM args, project-property substitution and JaCoCo +behavior stay stock. Only generated settings are marked, substituted by the stock +argument provider, and then encoded at the worker boundary. + +The worker carrier is a bounded, versioned, length-delimited UTF-8 envelope in +Base64. It is not encryption. Inherited namespaced environment is never serialized +into child arguments. Sensitive generated keys, identified by stock metadata, are not +promoted from environment or file sources into JVM properties. Explicit caller JVM +properties keep their role, so do not put secrets on command lines. The optional +`tracing.observer.log.directory` launch input writes separate observer log files. + +## How isolation works + +The rewriter runs offline on the stock jar. It fails if any known stock seam changes. + +- Classes, resources, service files and indexes move under `datadog.trace.observer`. + This covers tracer globals, shaded dependencies, Byte Buddy and Gradle + services/resources. +- The field-backed context protocol is renamed: `__datadogObserverContext$` fields and + `$get$__datadogObserverContext$` / `$put$__datadogObserverContext$` methods, + including dynamic names. Field injection stays enabled. +- `System` property, environment and `Boolean.getBoolean` calls go through a private + source view. +- A few literals look like packages but are external names. They keep their stock + spelling: the default agent and DogStatsD socket paths, the + `.inject.datadog.attribute.enabled` config suffix, the logs `datadog.product:` tag, + DogStatsD client and tracer health metric names, and OTLP attribute and scope + names. Internal context and request attribute keys are still relocated so they do + not collide with the subject tracer. +- The JUnit 5 and Spock advice only open observer spans for the synchronous outer + Gradle engine launch. Nested launchers are ignored. + +The modifiable-config convention recognizes the attached observer without path or +hash metadata. + +Three stock fixes are part of this work: + +- `TypeFactory` resolves the current transform target from the supplied bytes, not a + cached classpath description. Without this, the subject transformer removed an + interface the observer had injected. This was reproduced on Mockito's + `DetachedThreadLocal` and aborted a whole retransformation batch. The target + description is reused for the transform and reset afterwards. Other shared caches + are unchanged. This adds one parse per transformed target and has not been + benchmarked. +- `UnknownCIInfo` accepts `.git` as a file when looking for the repository root, as in + linked worktrees and submodules. Before, local runs from a worktree had no + repository, branch or commit tags. +- `LineCoverageStore.Factory` loads the per-test coverage recording classes up front. + JaCoCo probes also land in the tracer under test's `defineClass` hook. Loading a + class lazily while recording ran that hook, which recorded again until a + `StackOverflowError`. The JVM then printed + `java.lang.instrument ASSERTION FAILED ... transform method call failed` and loaded + the class untransformed. + +## Tests + +The rewriter, runtime and configuration sources have `buildSrc` tests. The artifact +tests need a built observer and stock jar: + +```sh +./gradlew :dd-java-agent:observerJar +./gradlew -p buildSrc test \ + --tests 'datadog.gradle.plugin.observer.*' --tests 'datadog.trace.observer.*' \ + -PrunBuildSrcTests \ + -PobserverTestArtifact="$PWD/dd-java-agent/build/observer/dd-observer-agent.jar" \ + -PobserverTestStock="$(ls "$PWD"/dd-java-agent/build/libs/dd-java-agent-*.jar)" +``` + +The `TypeFactory`, `UnknownCIInfo` and `LineCoverageStore` changes are covered by the +regular module tests. End-to-end runs against a local mock intake were done by hand +and are not part of the repository. + +## Limitations + +- Target: the repository's Gradle 9.8 daemon with the stock Gradle 8.10+ hook, the + JUnit Jupiter and Spock outer lifecycle, and the subject instrumentation harness. +- Stock capabilities are reported. Optional capabilities these tests do not select are + not proven compatible. No new product features were added. +- Per-test code coverage and coverage report upload work. Failed Test Replay, test + skipping, Auto Test Retries, Early Flake Detection and Test Management have not been + verified with the observer. +- Every Gradle build opens its own session, including `buildSrc`, `build-logic` and + Kotlin DSL accessor builds. Those sessions run no tests but still fetch settings. +- Unverified: arbitrary asynchronous engines, other runners, Maven, launcher injection + and generic child processes. `JavaExec`, TestKit and smoke-test subprocesses are not + injected automatically. +- Unsupported: configuration cache, daemon reuse, dynamic attach, AOT/CDS and security + manager environments. +- The external-literal list is maintained by hand. A new stock string that looks like + a `datadog.` package but names something external will be relocated until it is + added. From 79ff0a16a7e1545e47c8960326b121f462299e8d Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Thu, 1 Oct 2026 15:57:58 +0200 Subject: [PATCH 2/2] fix(ci-visibility): turn off nested-only test frameworks in the observer JUnit 4, TestNG, Karate, ScalaTest, Weaver and Cucumber only run as fixtures inside this repository's JUnit Platform tests. Observing them reported the fixtures as real tests, and broke the Karate 1.0 fixtures. Callers can still turn them back on. --- .../observer/bootstrap/ObserverRuntime.java | 19 ++++++++++ .../bootstrap/ObserverRuntimeTest.java | 38 +++++++++++++++++++ docs/tracing_the_tracer.md | 11 +++++- 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java index e15a27b8437..06df27ecefa 100644 --- a/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java +++ b/buildSrc/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java @@ -40,6 +40,11 @@ public final class ObserverRuntime { private static final Map CARRIER = new HashMap<>(); private static final Map LAUNCH = new HashMap<>(); + /** Test frameworks that only run as fixtures inside the observed JUnit Platform tests. */ + private static final String[] NESTED_ONLY_INTEGRATIONS = { + "junit-4", "testng", "karate", "scalatest", "weaver", "cucumber" + }; + private static volatile boolean initialized; private static String generatedLogFile; private static String configurationFingerprint; @@ -94,10 +99,24 @@ public static synchronized void initialize(String arguments) { configurationFingerprint = fingerprint(); CONFIG.putAll(CARRIER); CONFIG.putAll(childOverrides); + disableNestedOnlyIntegrations(); configureFileLogger(); initialized = true; } + /** Off unless the caller configured them: observing their fixtures would double-report. */ + private static void disableNestedOnlyIntegrations() { + for (String name : NESTED_ONLY_INTEGRATIONS) { + String env = name.toUpperCase(Locale.ROOT).replace('-', '_'); + if (!CONFIG.containsKey("dd.integration." + name + ".enabled") + && !CONFIG.containsKey("dd.trace.integration." + name + ".enabled") + && !ENVIRONMENT.containsKey("DD_INTEGRATION_" + env + "_ENABLED") + && !ENVIRONMENT.containsKey("DD_TRACE_INTEGRATION_" + env + "_ENABLED")) { + CONFIG.setProperty("dd.integration." + name + ".enabled", "false"); + } + } + } + private ObserverRuntime() {} private static void ensureInitialized() { diff --git a/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java b/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java index 065a4a63197..f54ac52c6bf 100644 --- a/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java +++ b/buildSrc/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.util.HashMap; @@ -172,6 +173,43 @@ void independentSourceSnapshotsAndMutablePrivateOverlay() throws Exception { } } + @Test + void nestedOnlyTestFrameworksAreOffUnlessConfigured() throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try { + System.setProperty( + "tracing.observer.child.v1", + ObserverRuntime.encode( + asList( + singletonMap("dd.integration.karate.enabled", "true"), + singletonMap("DD_TRACE_INTEGRATION_TESTNG_ENABLED", "true"), + emptyMap(), + emptyMap()))); + System.setProperty("dd.integration.junit-4.enabled", "true"); + try (URLClassLoader loader = + new URLClassLoader( + new URL[] {ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation()}, + null)) { + Class runtime = loader.loadClass(ObserverRuntime.class.getName()); + Method property = runtime.getMethod("getProperty", String.class); + for (String name : new String[] {"junit-4", "scalatest", "weaver", "cucumber"}) { + assertEquals("false", property.invoke(null, "dd.integration." + name + ".enabled")); + } + assertEquals("true", property.invoke(null, "dd.integration.karate.enabled")); + assertNull(property.invoke(null, "dd.integration.testng.enabled")); + assertEquals( + "true", + runtime + .getMethod("getenv", String.class) + .invoke(null, "DD_TRACE_INTEGRATION_TESTNG_ENABLED")); + assertNull(property.invoke(null, "dd.integration.junit-5.enabled")); + assertEquals("true", System.getProperty("dd.integration.junit-4.enabled")); + } + } finally { + System.setProperties(original); + } + } + @Test void fullNameNamespaceDoesNotWhitelistKeysOrRewriteValues() throws Exception { Properties original = (Properties) System.getProperties().clone(); diff --git a/docs/tracing_the_tracer.md b/docs/tracing_the_tracer.md index c103d61efa8..9518c5e08da 100644 --- a/docs/tracing_the_tracer.md +++ b/docs/tracing_the_tracer.md @@ -55,6 +55,14 @@ There is no allowlist for sites, endpoints, credentials, writers, products or OT Stock `Agent.configureCiVisibility` supplies the agent jar URI and CI defaults. Ordinary properties files cannot activate early bootstrap product gates, as in stock. +The observer has one default of its own. It turns off the `junit-4`, `testng`, +`karate`, `scalatest`, `weaver` and `cucumber` integrations. In this repository those +frameworks only run as fixtures inside JUnit Platform tests, so observing them would +report the fixtures as our tests. In Karate 1.0 it also broke the fixtures. To turn +one back on, set its `integration..enabled` or `trace.integration..enabled` +key as an observer property or environment variable. A value in a properties file +does not override this default. + The observer does not import the subject's DD/OTEL properties, environment, config-file selection or ambient propagation headers. JVM and CI platform facts are shared. LOCAL and FLEET stable-config sources stay empty so the observer never opens @@ -123,7 +131,8 @@ The rewriter runs offline on the stock jar. It fails if any known stock seam cha names. Internal context and request attribute keys are still relocated so they do not collide with the subject tracer. - The JUnit 5 and Spock advice only open observer spans for the synchronous outer - Gradle engine launch. Nested launchers are ignored. + Gradle engine launch. Nested launchers are ignored. Other test frameworks are off by + default, see [Configuration](#configuration). The modifiable-config convention recognizes the attached observer without path or hash metadata.