diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee8c267c77d..e4d1b15f2d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -279,6 +279,9 @@ # @DataDog/ci-app-libraries /dd-java-agent/agent-ci-visibility/ @DataDog/ci-app-libraries +/dd-java-agent/observer/ @DataDog/ci-app-libraries +/buildSrc/src/*/kotlin/datadog/gradle/plugin/observer/ @DataDog/ci-app-libraries +/docs/tracing_the_tracer.md @DataDog/ci-app-libraries /dd-smoke-tests/backend-mock/ @DataDog/ci-app-libraries /dd-smoke-tests/gradle/ @DataDog/ci-app-libraries /dd-smoke-tests/junit-console/ @DataDog/ci-app-libraries diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelection.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelection.kt new file mode 100644 index 00000000000..cf5e4f337ac --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelection.kt @@ -0,0 +1,49 @@ +package datadog.gradle.plugin.observer + +import java.io.File +import java.net.JarURLConnection +import java.util.jar.JarFile + +/** Finds the observer agent attached to this Gradle JVM, without caller artifact metadata. */ +object ObserverAgentSelection { + private const val RUNTIME = "datadog.trace.observer.bootstrap.ObserverRuntime" + + private val attachedJar: File? by lazy { + val runtime = + try { + Class.forName(RUNTIME, false, null) + } catch (absent: ClassNotFoundException) { + return@lazy null + } + val resource = runtime.getResource("ObserverRuntime.class") + check(resource != null && resource.protocol == "jar") { "Attached observer has no jar resource" } + val jar = File((resource.openConnection() as JarURLConnection).jarFileURL.toURI()).canonicalFile + val attribute = runtime.getField("MANIFEST_ATTRIBUTE").get(null) as String + val version = runtime.getField("MANIFEST_VERSION").get(null) as String + JarFile(jar).use { + require(it.manifest.mainAttributes.getValue(attribute) == version) { "Not an observer artifact: $jar" } + } + jar + } + + @JvmStatic + fun attached(): File? = attachedJar + + @JvmStatic + fun validate(): File = + requireNotNull(attachedJar) { + "traceTracer requires an observer attached to the Gradle daemon with -javaagent" + } + + @JvmStatic + fun configurationFingerprint(): String = + Class.forName(RUNTIME, false, null).getMethod("configurationFingerprint").invoke(null) as String + + /** Whether a `-javaagent:` argument points at the attached observer, whatever path spelling it uses. */ + @JvmStatic + fun isAttachedAgentArgument(argument: String): Boolean = attachedJar?.let { pointsAt(it, argument) } ?: false + + internal fun pointsAt(jar: File, argument: String): Boolean = + argument.startsWith("-javaagent:") && + File(argument.removePrefix("-javaagent:").substringBefore('=')).canonicalFile == jar.canonicalFile +} diff --git a/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts index 04862485528..d6cc7e527e8 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts @@ -80,6 +80,20 @@ tasks.withType().configureEach { timeout.set(Duration.of(20, ChronoUnit.MINUTES)) } +// Experimental namespace-isolated observer. No observer dependencies/configuration enter ordinary tests. +if (providers.gradleProperty("traceTracer").map { it.toBoolean() }.orElse(false).get()) { + require(!gradle.startParameter.isConfigurationCacheRequested) { + "The experimental Gradle observer does not support configuration cache" + } + val observerJar = datadog.gradle.plugin.observer.ObserverAgentSelection.validate() + tasks.withType().configureEach { + inputs.file(observerJar).withPathSensitivity(org.gradle.api.tasks.PathSensitivity.NONE) + inputs.property("observerConfiguration", provider { + datadog.gradle.plugin.observer.ObserverAgentSelection.configurationFingerprint() + }) + } +} + // Register a task "allTests" that depends on all non-latest and non-traceAgentTest Test tasks. // This is used when we only want to run the 'main' test sets. tasks.register("allTests") { diff --git a/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts index 047b64d326d..d1647ea24b4 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.modifiable-config.gradle.kts @@ -27,7 +27,8 @@ tasks.withType().configureEach { // doFirst prepends, so this runs after any -javaagent registered later via doFirst. doFirst { val foreignAgent = allJvmArgs.firstOrNull { - it.startsWith("-javaagent:") && !it.contains("modifiable-config-agent") + it.startsWith("-javaagent:") && !it.contains("modifiable-config-agent") && + !datadog.gradle.plugin.observer.ObserverAgentSelection.isAttachedAgentArgument(it) } if (foreignAgent != null) { logger.info( diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelectionTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelectionTest.kt new file mode 100644 index 00000000000..547c99df159 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/observer/ObserverAgentSelectionTest.kt @@ -0,0 +1,27 @@ +package datadog.gradle.plugin.observer + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path + +class ObserverAgentSelectionTest { + @Test + fun `nothing is selected without an attached observer`() { + assertThat(ObserverAgentSelection.attached()).isNull() + assertThat(ObserverAgentSelection.isAttachedAgentArgument("-javaagent:/any/observer.jar")).isFalse() + assertThatThrownBy { ObserverAgentSelection.validate() }.isInstanceOf(IllegalArgumentException::class.java) + } + + @Test + fun `agent arguments match the jar whatever path spelling they use`(@TempDir directory: Path) { + val jar = Files.createFile(directory.resolve("observer.jar")).toFile() + val link = Files.createSymbolicLink(directory.resolve("link.jar"), jar.toPath()).toFile() + assertThat(ObserverAgentSelection.pointsAt(jar, "-javaagent:${link.path}")).isTrue() + assertThat(ObserverAgentSelection.pointsAt(jar, "-javaagent:${jar.path}=dd.service=x")).isTrue() + assertThat(ObserverAgentSelection.pointsAt(jar, "-javaagent:${directory.resolve("other.jar")}")).isFalse() + assertThat(ObserverAgentSelection.pointsAt(jar, jar.path)).isFalse() + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java index df6ade9a13e..5323a358903 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/UnknownCIInfo.java @@ -5,6 +5,7 @@ import datadog.trace.api.civisibility.telemetry.tag.Provider; import datadog.trace.api.git.GitInfo; import datadog.trace.civisibility.ci.env.CiEnvironment; +import java.nio.file.Files; import java.nio.file.Path; import javax.annotation.Nonnull; import org.slf4j.Logger; @@ -19,7 +20,7 @@ * *

However, we would like to provide git information if the user is using git, so we infer the * workspace path leveraging the `.git` folder, which is usually kept in the root path of the - * repository. + * repository. In linked worktrees and submodules, `.git` is a file pointing to the git directory. * *

The workspace path will be used in the CIProviderInfo constructor to access the `.git` folder * and calculate the git information properly. @@ -47,7 +48,11 @@ public GitInfo buildCIGitInfo() { @Override public CIInfo buildCIInfo() { - Path workspace = findParentPathBackwards(getCurrentPath(), getTargetFolder(), true); + Path workspace = + findParentPathBackwards( + getCurrentPath(), + getTargetFolder(), + git -> Files.isDirectory(git) || Files.isRegularFile(git)); if (workspace == null) { return CIInfo.NOOP; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java index bde89521dd5..2d5d5a3af09 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/line/LineCoverageStore.java @@ -255,8 +255,12 @@ public static final class Factory implements CoverageStore.Factory { public Factory(CiVisibilityMetricCollector metrics, SourcePathResolver sourcePathResolver) { this.metrics = metrics; this.sourcePathResolver = sourcePathResolver; + // Recording must not load classes: a covered defineClass hook would record again and recurse. + loadRecordingClasses(LineCoverageStore.class, LineProbes.class, ExecutionDataAdapter.class); } + private static void loadRecordingClasses(Class... classes) {} + @Override public CoverageStore create(@Nullable TestIdentifier testIdentifier) { return new LineCoverageStore( diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java index eb668ac8b4b..ce694c3db12 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java @@ -9,6 +9,7 @@ import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,22 +50,24 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx */ public static Path findParentPathBackwards( final Path current, final String target, final boolean isTargetDirectory) { - if (current == null || target == null || target.isEmpty()) { + return findParentPathBackwards( + current, + target, + isTargetDirectory ? path -> Files.isDirectory(path) : path -> Files.isRegularFile(path)); + } + + /** Like {@link #findParentPathBackwards(Path, String, boolean)}, for any kind of target. */ + public static Path findParentPathBackwards( + final Path current, final String target, final Predicate isTarget) { + if (target == null || target.isEmpty()) { return null; } - - final Path targetPath = current.resolve(target); - if (Files.exists(targetPath)) { - if (isTargetDirectory && Files.isDirectory(targetPath)) { - return current; - } else if (!isTargetDirectory && Files.isRegularFile(targetPath)) { - return current; - } else { - return findParentPathBackwards(current.getParent(), target, isTargetDirectory); + for (Path path = current; path != null; path = path.getParent()) { + if (isTarget.test(path.resolve(target))) { + return path; } - } else { - return findParentPathBackwards(current.getParent(), target, isTargetDirectory); } + return null; } public static String expandTilde(final String path) { diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy index 652fecd470e..cc65de13acf 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ci/UnknownCIInfoTest.groovy @@ -8,7 +8,10 @@ import datadog.trace.civisibility.ci.env.CiEnvironmentImpl import datadog.trace.civisibility.git.CILocalGitInfoBuilder import datadog.trace.civisibility.git.CIProviderGitInfoBuilder import datadog.trace.civisibility.git.tree.GitClient +import spock.lang.TempDir +import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths class UnknownCIInfoTest extends CITagsProviderTest { @@ -55,6 +58,24 @@ class UnknownCIInfoTest extends CITagsProviderTest { ciTags == expectedTags } + @TempDir + Path temporaryFolder + + def "test workspace is found from a worktree git file"() { + setup: + def worktree = Files.createDirectories(temporaryFolder.resolve("worktree")) + Files.write(worktree.resolve(GIT_FOLDER_FOR_TESTS), "gitdir: /repo/.git/worktrees/worktree\n".bytes) + def module = Files.createDirectories(worktree.resolve("module")) + + when: + def ciInfo = new CIProviderInfoFactory(Config.get(), GIT_FOLDER_FOR_TESTS, new CiEnvironmentImpl(env.getAll())) + .createCIProviderInfo(module) + .buildCIInfo() + + then: + ciInfo.ciWorkspace == worktree.toRealPath().toString() + } + def "test workspace is null if target folder does not exist"() { when: def gitClientFactory = Stub(GitClient.Factory) diff --git a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/coverage/line/LineCoverageStoreTest.java b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/coverage/line/LineCoverageStoreTest.java index fe719cdc78d..8038b0e716f 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/coverage/line/LineCoverageStoreTest.java +++ b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/coverage/line/LineCoverageStoreTest.java @@ -3,7 +3,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.civisibility.coverage.line.LineCoverageStore.AnalysisCacheKey; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import org.junit.jupiter.api.Test; class LineCoverageStoreTest { @@ -35,4 +42,56 @@ void cacheKeyIgnoresTrailingUnsetProbes() { assertEquals(shortKey, padded); assertEquals(shortKey.hashCode(), padded.hashCode()); } + + @Test + void recordingLoadsNoClassesOnceTheFactoryExists() throws Exception { + // A covered defineClass hook records coverage while a class loads. If recording itself loaded + // a class, it would run the hook again and recurse. + try (RecordingLoader loader = new RecordingLoader()) { + Class factoryType = loader.loadClass(LineCoverageStore.Factory.class.getName()); + Object factory = factoryType.getConstructors()[0].newInstance(null, null); + factoryType + .getMethod("setTotalProbeCount", String.class, int.class) + .invoke(factory, "java/lang/String", 1); + Set loadedByFactory = new HashSet<>(loader.defined); + + Object store = + factoryType.getMethod("create", TestIdentifier.class).invoke(factory, (Object) null); + Object probes = store.getClass().getMethod("getProbes").invoke(store); + probes + .getClass() + .getMethod("record", Class.class, long.class, int.class) + .invoke(probes, String.class, 1L, 0); + + assertEquals(loadedByFactory, new HashSet<>(loader.defined)); + } + } + + /** Defines this package's classes itself, so the test sees exactly when each one loads. */ + private static final class RecordingLoader extends URLClassLoader { + private static final String PACKAGE = LineCoverageStore.class.getPackage().getName() + "."; + + final List defined = new CopyOnWriteArrayList<>(); + + RecordingLoader() { + super( + new URL[] {LineCoverageStore.class.getProtectionDomain().getCodeSource().getLocation()}, + LineCoverageStoreTest.class.getClassLoader()); + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (!name.startsWith(PACKAGE)) { + return super.loadClass(name, resolve); + } + synchronized (getClassLoadingLock(name)) { + Class type = findLoadedClass(name); + if (type == null) { + type = findClass(name); + defined.add(name); + } + return type; + } + } + } } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java index 51650b3d2b2..d3de46aa1c4 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactory.java @@ -112,6 +112,9 @@ final class TypeFactory { byte[] targetBytecode; + /** Description of the current transform target, resolved from the supplied bytes. */ + private LazyType targetType; + /** Sets the current class-loader context of this type-factory. */ void switchContext(ClassLoader classLoader) { if (currentClassLoader != classLoader || null == classFileLocator) { @@ -153,6 +156,7 @@ static void clear() { void beginTransform(String name, byte[] bytecode) { targetName = name; targetBytecode = bytecode; + targetType = new LazyType(name); if (installing) { originalClassLoader = currentClassLoader; @@ -186,6 +190,7 @@ void endTransform() { targetName = null; targetBytecode = null; + targetType = null; createOutlines = OUTLINING_ENABLED; } @@ -228,7 +233,7 @@ static TypeDescription findType(String name) { } private TypeDescription deferTypeResolution(String name) { - return deferredTypes.computeIfAbsent(name, deferType); + return name.equals(targetName) ? targetType : deferredTypes.computeIfAbsent(name, deferType); } /** Attempts to resolve the named type using the current context. */ @@ -236,7 +241,7 @@ TypeDescription resolveType(LazyType request) { if (null != classFileLocator) { TypeDescription result; if (createOutlines) { - if ("java.lang.Object".equals(request.name)) { + if (request != targetType && "java.lang.Object".equals(request.name)) { return objectOutline; } result = lookupType(request, outlineTypes, outlineTypeParser); @@ -258,8 +263,17 @@ private TypeDescription lookupType( boolean isOutline = typeParser == outlineTypeParser; long fromTick = InstrumenterMetrics.tick(); - // existing type description from same classloader? SharedTypeInfo sharedType = types.find(name); + + // A cached description of the transform target can predate changes from earlier transformers, + // so parse the supplied bytes instead. Leave the cached entry for other lookups. + if (request == targetType && null != sharedType) { + TypeDescription type = typeParser.parse(targetBytecode); + InstrumenterMetrics.buildTypeDescription(fromTick, isOutline); + return type; + } + + // existing type description from same classloader? if (null != sharedType && (name.startsWith("java.") || sharedType.sameClassLoader(classLoaderId))) { InstrumenterMetrics.reuseTypeDescription(fromTick, isOutline); @@ -396,7 +410,7 @@ public TypeList.Generic getInterfaces() { @Override public boolean isPublic() { - return isPublicFilter.contains(name) || super.isPublic(); + return (this != targetType && isPublicFilter.contains(name)) || super.isPublic(); } private TypeDescription outline() { diff --git a/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java new file mode 100644 index 00000000000..649387a39be --- /dev/null +++ b/dd-java-agent/agent-tooling/src/test/java/datadog/trace/agent/tooling/bytebuddy/outline/TypeFactoryTransformTest.java @@ -0,0 +1,143 @@ +package datadog.trace.agent.tooling.bytebuddy.outline; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.Serializable; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.jar.asm.ClassReader; +import net.bytebuddy.jar.asm.ClassVisitor; +import net.bytebuddy.jar.asm.ClassWriter; +import net.bytebuddy.jar.asm.Opcodes; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TypeFactoryTransformTest { + private TypeFactory factory; + + public static class Example { + public String originalField; + + public void originalMethod() {} + } + + @BeforeEach + void setUp() { + TypeFactory.clear(); + factory = new TypeFactory(); + TypeFactory.typeFactory.set(factory); + factory.beginInstall(); + factory.switchContext(Example.class.getClassLoader()); + } + + @AfterEach + void tearDown() { + factory.endTransform(); + factory.endInstall(); + TypeFactory.typeFactory.remove(); + TypeFactory.clear(); + } + + @Test + void transformBytesOverrideWarmOutlineAndFullCachesWithoutPoisoningOtherLookups() + throws Exception { + String name = Example.class.getName(); + byte[] originalBytes = ClassFileLocator.ForClassLoader.read(Example.class); + TypeDescription original = TypeFactory.findType(name); + assertTrue(original.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, original.getDeclaredFields().size()); + factory.disableFullDescriptions(); + + factory.beginTransform(name, withInterfaceAndField(originalBytes, "observerField")); + TypeDescription target = TypeFactory.findType(name); + assertAugmented(target, "observerField"); + factory.enableFullDescriptions(); + assertAugmented(target, "observerField"); + assertAugmented(TypeFactory.findType(name), "observerField"); + factory.endTransform(); + + // Original same-loader shared/classpath descriptions are not overwritten by transform bytes. + TypeDescription outsideTransform = TypeFactory.findType(name); + assertTrue(outsideTransform.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, outsideTransform.getDeclaredFields().size()); + factory.disableFullDescriptions(); + + factory.beginTransform(name, withInterfaceAndField(originalBytes, "nextField")); + TypeDescription next = TypeFactory.findType(name); + assertNotSame(target, next); + assertAugmented(next, "nextField"); + factory.enableFullDescriptions(); + assertAugmented(next, "nextField"); + assertTrue(next.getDeclaredFields().filter(named("observerField")).isEmpty()); + factory.endTransform(); + + factory.beginTransform(name, originalBytes); + TypeDescription restored = TypeFactory.findType(name); + assertTrue(restored.getInterfaces().isEmpty()); + factory.enableFullDescriptions(); + assertEquals(1, restored.getDeclaredFields().size()); + } + + @Test + void transformBytesOnAColdCacheAreSharedLikeAnyOtherParse() throws Exception { + String name = Example.class.getName(); + byte[] augmented = + withInterfaceAndField(ClassFileLocator.ForClassLoader.read(Example.class), "sharedField"); + + factory.beginTransform(name, augmented); + assertAugmented(TypeFactory.findType(name), "sharedField"); + factory.endTransform(); + + // Later lookups, such as supertype walks from other classes, reuse the shared description. + factory.switchContext(Example.class.getClassLoader()); + assertEquals( + Serializable.class.getName(), + TypeFactory.findType(name).getInterfaces().getOnly().asErasure().getName()); + } + + private static void assertAugmented(TypeDescription type, String field) { + assertEquals( + Serializable.class.getName(), type.getInterfaces().getOnly().asErasure().getName()); + assertEquals(1, type.getDeclaredFields().filter(named(field)).size()); + assertEquals(1, type.getDeclaredMethods().filter(named("originalMethod")).size()); + } + + private static byte[] withInterfaceAndField(byte[] original, String field) { + ClassWriter writer = new ClassWriter(0); + new ClassReader(original) + .accept( + new ClassVisitor(Opcodes.ASM9, writer) { + @Override + public void visit( + int version, + int access, + String name, + String signature, + String superName, + String[] interfaces) { + super.visit( + version, + access, + name, + signature, + superName, + new String[] {"java/io/Serializable"}); + } + + @Override + public void visitEnd() { + super.visitField(Opcodes.ACC_PUBLIC, field, "Ljava/lang/Object;", null, null) + .visitEnd(); + super.visitEnd(); + } + }, + 0); + return writer.toByteArray(); + } +} diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 3fe773b73a4..476b14ff967 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -699,5 +699,5 @@ tasks.register('updateAgentJarIntegrationsGoldenFile', JavaExec) { } tasks.named('check') { - dependsOn 'verifyAgentJarContents', 'verifyAgentJarIntegrations' + dependsOn 'verifyAgentJarContents', 'verifyAgentJarIntegrations', ':dd-java-agent:observer:test' } diff --git a/dd-java-agent/observer/build.gradle b/dd-java-agent/observer/build.gradle new file mode 100644 index 00000000000..6bc73eec254 --- /dev/null +++ b/dd-java-agent/observer/build.gradle @@ -0,0 +1,45 @@ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + id 'dd-trace-java.conventions.java' +} + +description = 'Experimental namespace-isolated copy of the agent that traces this repository\'s own tests' + +minimumBranchCoverage = 0.7 +minimumInstructionCoverage = 0.8 + +dependencies { + implementation libs.asm + implementation libs.asm.tree + implementation libs.asm.commons + + testImplementation libs.bundles.junit5 + // Only to define the relocated Gradle argument provider in a test. + testImplementation gradleApi() +} + +evaluationDependsOn(':dd-java-agent') + +def stockAgent = project(':dd-java-agent').tasks.named('shadowJar', ShadowJar).flatMap { it.archiveFile } +def observerAgent = layout.buildDirectory.file('libs/dd-observer-agent.jar') + +// Developer-only artifact; never part of assemble or publication. +def observerJar = tasks.register('observerJar', JavaExec) { + description = 'Rewrites the stock agent jar into the observer agent jar' + classpath = sourceSets.main.runtimeClasspath + mainClass = 'datadog.trace.observer.rewriter.ObserverAgentRewriter' + inputs.file(stockAgent).withPathSensitivity(PathSensitivity.NONE) + outputs.file(observerAgent) + argumentProviders.add({ [stockAgent.get().asFile.path, observerAgent.get().asFile.path] } as CommandLineArgumentProvider) +} + +tasks.named('test', Test) { + inputs.files(stockAgent, observerJar).withPathSensitivity(PathSensitivity.NONE) + jvmArgumentProviders.add({ + [ + "-Dobserver.test.stock=${stockAgent.get().asFile.path}", + "-Dobserver.test.artifact=${observerAgent.get().asFile.path}" + ] + } as CommandLineArgumentProvider) +} diff --git a/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java b/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java new file mode 100644 index 00000000000..a9fe8518e19 --- /dev/null +++ b/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverBootstrap.java @@ -0,0 +1,23 @@ +package datadog.trace.observer.bootstrap; + +import java.io.File; +import java.lang.instrument.Instrumentation; +import java.util.jar.JarFile; + +/** Install the JDK-only bridge in bootstrap before any relocated early configuration reads. */ +public final class ObserverBootstrap { + private ObserverBootstrap() {} + + public static void premain(String arguments, Instrumentation instrumentation) throws Exception { + File jar = + new File( + ObserverBootstrap.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + instrumentation.appendToBootstrapClassLoaderSearch(new JarFile(jar)); + Class.forName("datadog.trace.observer.bootstrap.ObserverRuntime", true, null) + .getMethod("initializePremain") + .invoke(null); + Class.forName("datadog.trace.observer.trace.bootstrap.AgentPreCheck", true, null) + .getMethod("premain", String.class, Instrumentation.class) + .invoke(null, arguments, instrumentation); + } +} diff --git a/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java b/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java new file mode 100644 index 00000000000..b92e3600030 --- /dev/null +++ b/dd-java-agent/observer/src/main/java/datadog/trace/observer/bootstrap/ObserverRuntime.java @@ -0,0 +1,592 @@ +package datadog.trace.observer.bootstrap; + +import static java.util.Collections.unmodifiableMap; + +import de.thetaphi.forbiddenapis.SuppressForbidden; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** Private stock source views and process transport; no subject configuration is imported. */ +@SuppressForbidden +public final class ObserverRuntime { + public static final String MANIFEST_ATTRIBUTE = "Tracing-The-Tracer-Observer"; + public static final String MANIFEST_VERSION = "3"; + public static final String ROOTS_RESOURCE = "observer-relocated-roots.txt"; + public static final String ALIASES_RESOURCE = "observer-config-aliases.txt"; + public static final String SENSITIVE_RESOURCE = "observer-sensitive-config.txt"; + public static final String PROPERTY_PREFIX = "tracing.observer.config."; + public static final String ENV_PREFIX = "TRACING_OBSERVER_CONFIG_"; + static final String CHILD = "tracing.observer.child.v2"; + private static final String GENERATED = "tracing.observer.generated."; + private static final String LOG_FILE = "datadog.trace.observer.slf4j.simpleLogger.logFile"; + private static final String LOG_DIRECTORY = "tracing.observer.log.directory"; + + /** Test frameworks that only run as fixtures inside the observed JUnit Platform tests. */ + private static final String[] NESTED_ONLY_INTEGRATIONS = { + "junit-4", "testng", "karate", "scalatest", "weaver", "cucumber" + }; + + private static final Set ROOTS = metadata(ROOTS_RESOURCE); + private static final Set ALIASES = metadata(ALIASES_RESOURCE); + private static final Set SENSITIVE = metadata(SENSITIVE_RESOURCE); + private static final Properties CONFIG = new Properties(); + private static final Map ENVIRONMENT = new HashMap<>(); + private static final Map CARRIER = new HashMap<>(); + private static final Map LAUNCH = new HashMap<>(); + + private static final AtomicInteger OUTER_ENGINES = new AtomicInteger(); + private static final AtomicBoolean UNOWNED_ENGINE_WARNING = new AtomicBoolean(); + + private static final Object LOCK = new Object(); + private static volatile boolean initialized; + private static Envelope callerConfiguration; + private static String generatedLogFile; + + private ObserverRuntime() {} + + /** Initialize the private source view before the stock bootstrap reads configuration. */ + public static void initializePremain() { + synchronized (LOCK) { + initialize(); + } + } + + private static void initialize() { + if (initialized) { + throw new IllegalStateException("Observer runtime already initialized"); + } + for (String key : System.getProperties().stringPropertyNames()) { + if (key.startsWith(PROPERTY_PREFIX)) { + CONFIG.setProperty( + privateSpelling(key.substring(PROPERTY_PREFIX.length())), System.getProperty(key)); + } else if (key.startsWith("tracing.observer.") && !key.equals(CHILD)) { + LAUNCH.put(key, System.getProperty(key)); + } + } + for (Map.Entry e : System.getenv().entrySet()) { + if (e.getKey().startsWith(ENV_PREFIX)) { + ENVIRONMENT.put(e.getKey().substring(ENV_PREFIX.length()), e.getValue()); + } + } + // Gradle applies -D daemon arguments after premain, so the environment is the early channel. + String logs = System.getenv("TRACING_OBSERVER_LOG_DIRECTORY"); + if (logs != null) { + LAUNCH.putIfAbsent(LOG_DIRECTORY, logs); + } + String child = System.getProperty(CHILD); + Properties workerOverrides = new Properties(); + if (child != null) { + // Explicit worker JVM properties still win over the parent's generated settings. + workerOverrides.putAll(CONFIG); + Envelope parent = Envelope.decode(child); + CONFIG.clear(); + CONFIG.putAll(parent.config); + ENVIRONMENT.putAll(parent.environment); + LAUNCH.clear(); + LAUNCH.putAll(parent.launch); + CARRIER.putAll(parent.carrier); + } + callerConfiguration = new Envelope(toMap(CONFIG), ENVIRONMENT, LAUNCH, CARRIER); + CONFIG.putAll(CARRIER); + CONFIG.putAll(workerOverrides); + configureFileLogger(); + initialized = true; + } + + private static void ensureInitialized() { + if (!initialized) { + synchronized (LOCK) { + if (!initialized) { + initialize(); + } + } + } + } + + /** Stable task input for the captured caller configuration, not later mutable IPC state. */ + public static String configurationFingerprint() { + ensureInitialized(); + try { + byte[] hash = + MessageDigest.getInstance("SHA-256") + .digest(callerConfiguration.encode().getBytes(StandardCharsets.UTF_8)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(hash); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** Lowest-precedence stock source, so any explicit observer setting overrides these defaults. */ + public static Map stableConfig(String origin) { + Map config = new HashMap<>(); + if (origin.equals("LOCAL_STABLE_CONFIG")) { + for (String name : NESTED_ONLY_INTEGRATIONS) { + config.put( + "DD_TRACE_" + name.toUpperCase(Locale.ROOT).replace('-', '_') + "_ENABLED", "false"); + } + // This repository has too many root packages to infer them, which would cover everything, + // including JDK and test-only classes. Cover the code under test instead. + config.put("DD_CIVISIBILITY_CODE_COVERAGE_INCLUDES", "datadog.*:com.datadog.*"); + } + return config; + } + + /** + * Moves every package reference whose {@code datadog.}, {@code com.datadog.} or + * {@code net.bytebuddy} root is listed into the observer namespace. Anything else, such as host + * paths, wire names or JNDI names, keeps its stock spelling. + */ + public static String relocate(String value, Set roots) { + StringBuilder result = null; + int copied = 0; + for (int i = value.indexOf("datadog"); i >= 0; i = value.indexOf("datadog", i + 1)) { + int start = i; + int prefixEnd = i + "datadog".length(); + if (prefixEnd >= value.length()) { + break; + } + char separator = value.charAt(prefixEnd); + if (separator != '.' && separator != '/') { + continue; + } + if (i >= 4 && value.startsWith("com", i - 4) && value.charAt(i - 1) == separator) { + start = i - 4; + } + if (!startsName(value, start) + || relocated(value, start, separator) + || value.startsWith("trace" + separator + "observer" + separator, prefixEnd + 1)) { + continue; + } + int segmentEnd = prefixEnd + 1; + while (segmentEnd < value.length() + && Character.isJavaIdentifierPart(value.charAt(segmentEnd))) { + segmentEnd++; + } + String root = + (start == i ? "datadog/" : "com/datadog/") + value.substring(prefixEnd + 1, segmentEnd); + if (!roots.contains(root)) { + continue; + } + if (result == null) { + result = new StringBuilder(value.length() + 32); + } + result.append(value, copied, start).append("datadog").append(separator); + result.append("trace").append(separator).append("observer").append(separator); + copied = start == i ? prefixEnd + 1 : start; + } + String bytebuddy = + result == null ? value : result.append(value, copied, value.length()).toString(); + if (roots.contains("net/bytebuddy")) { + bytebuddy = relocateByteBuddy(bytebuddy, '.'); + bytebuddy = relocateByteBuddy(bytebuddy, '/'); + } + return bytebuddy; + } + + private static String relocateByteBuddy(String value, char separator) { + String name = "net" + separator + "bytebuddy" + separator; + StringBuilder result = null; + int copied = 0; + for (int i = value.indexOf(name); i >= 0; i = value.indexOf(name, i + 1)) { + if (!startsName(value, i) || relocated(value, i, separator)) { + continue; + } + if (result == null) { + result = new StringBuilder(value.length() + 32); + } + result.append(value, copied, i).append("datadog").append(separator); + result.append("trace").append(separator).append("observer").append(separator); + copied = i; + } + return result == null ? value : result.append(value, copied, value.length()).toString(); + } + + /** Whether the name at {@code index} already sits under the observer namespace. */ + private static boolean relocated(String value, int index, char separator) { + String observer = "datadog" + separator + "trace" + separator + "observer" + separator; + return index >= observer.length() && value.startsWith(observer, index - observer.length()); + } + + /** + * Package names here are lower case, so a name starts anywhere except after a lower-case letter, + * digit or underscore. That covers descriptors such as {@code ILdatadog/} and {@code -Dnet.}. + */ + private static boolean startsName(String value, int index) { + if (index == 0) { + return true; + } + char previous = value.charAt(index - 1); + return !(previous >= 'a' && previous <= 'z') + && !(previous >= '0' && previous <= '9') + && previous != '_'; + } + + public static boolean usingGeneratedFileLogger() { + ensureInitialized(); + return generatedLogFile != null && generatedLogFile.equals(CONFIG.getProperty(LOG_FILE)); + } + + private static void configureFileLogger() { + String logs = LAUNCH.get(LOG_DIRECTORY); + if (logs != null && !CONFIG.containsKey(LOG_FILE)) { + generatedLogFile = + new File(logs, "observer-" + Long.toHexString(System.nanoTime()) + ".log").getPath(); + CONFIG.setProperty(LOG_FILE, generatedLogFile); + } + } + + /** Resource files keep ordinary logger key spellings; values are never relocated. */ + public static Properties loggerProperties(Properties properties) { + if (properties == null) { + return null; + } + Properties result = new Properties(); + for (String key : properties.stringPropertyNames()) { + result.setProperty(privateSpelling(key), properties.getProperty(key)); + } + return result; + } + + private static String privateSpelling(String key) { + return key.startsWith("datadog.") || key.startsWith("net.bytebuddy.") + ? relocate(key, ROOTS) + : key; + } + + private static Set metadata(String resource) { + Set result = new HashSet<>(); + try (InputStream in = ObserverRuntime.class.getResourceAsStream("/" + resource)) { + if (in != null) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int n; + while ((n = in.read(buffer)) != -1) { + out.write(buffer, 0, n); + } + for (String line : new String(out.toByteArray(), StandardCharsets.UTF_8).split("\n")) { + if (!line.isEmpty()) { + result.add(line); + } + } + } + } catch (IOException e) { + throw new IllegalStateException("Cannot load observer metadata " + resource, e); + } + return result; + } + + private static boolean privateKey(String key) { + return key.startsWith("dd.") + || key.startsWith("datadog.") + || key.startsWith("otel.") + || key.startsWith("net.bytebuddy.") + || key.startsWith("tracing.observer.") + || CONFIG.containsKey(key) + || ALIASES.contains(key); + } + + public static String getProperty(String key) { + return getProperty(key, null); + } + + public static String getProperty(String key, String fallback) { + ensureInitialized(); + if (key == null) { + throw new NullPointerException("key"); + } + if (key.isEmpty()) { + throw new IllegalArgumentException("key is empty"); + } + if (!privateKey(key)) { + return System.getProperty(key, fallback); + } + String value = CONFIG.getProperty(key); + return value == null ? fallback : value; + } + + public static boolean getBoolean(String key) { + return key != null && !key.isEmpty() && Boolean.parseBoolean(getProperty(key)); + } + + public static String setProperty(String key, String value) { + getProperty(key); + return privateKey(key) + ? (String) CONFIG.setProperty(key, value) + : System.setProperty(key, value); + } + + public static String clearProperty(String key) { + getProperty(key); + return privateKey(key) ? (String) CONFIG.remove(key) : System.clearProperty(key); + } + + public static Properties getProperties() { + ensureInitialized(); + Properties result = new Properties(); + Properties real = System.getProperties(); + for (String key : real.stringPropertyNames()) { + if (!privateKey(key)) { + result.setProperty(key, real.getProperty(key)); + } + } + result.putAll(CONFIG); + return result; + } + + public static String getenv(String key) { + ensureInitialized(); + if (key == null) { + throw new NullPointerException("key"); + } + return key.startsWith("DD_") + || key.startsWith("OTEL_") + || key.startsWith("TRACING_OBSERVER_") + || ALIASES.contains(key) + || ENVIRONMENT.containsKey(key) + ? ENVIRONMENT.get(key) + : System.getenv(key); + } + + public static Map getenv() { + ensureInitialized(); + Map result = new HashMap<>(); + for (String key : System.getenv().keySet()) { + String value = getenv(key); + if (value != null) { + result.put(key, value); + } + } + result.putAll(ENVIRONMENT); + return unmodifiableMap(result); + } + + /** + * Only the parent-generated map can classify a worker; ambient HTTP headers are not a carrier. + */ + public static Map propagationProperties() { + ensureInitialized(); + return unmodifiableMap(CARRIER); + } + + /** Mark only generated settings; stock debug/additional arguments remain untouched. */ + public static Map generatedProperties(Map generated) { + ensureInitialized(); + Map properties = toMap(CONFIG); + for (Map.Entry entry : generated.entrySet()) { + String environmentKey = + entry.getKey().toUpperCase(Locale.ROOT).replace('.', '_').replace('-', '_'); + // A resolved environment/file credential must never be promoted into a JVM argument. + // Explicit caller JVM properties keep their original source role. + if (!SENSITIVE.contains(environmentKey) || CONFIG.containsKey(entry.getKey())) { + properties.put(entry.getKey(), entry.getValue()); + } + } + if (generatedLogFile != null) { + properties.remove(LOG_FILE); + } + Map marked = new HashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getValue() != null) { + marked.put(GENERATED + entry.getKey(), entry.getValue()); + } + } + return marked; + } + + /** Encode after stock Gradle project-property substitution, preserving non-generated args. */ + public static Iterable childArguments(Iterable arguments) { + ensureInitialized(); + Map carrier = new HashMap<>(); + List result = new ArrayList<>(); + for (String argument : arguments) { + if (argument.startsWith("-D" + GENERATED)) { + int separator = argument.indexOf('='); + carrier.put( + argument.substring(2 + GENERATED.length(), separator), + argument.substring(separator + 1)); + } else { + result.add(argument); + } + } + // Environment is inherited in its normal namespaced role, never Base64-encoded in argv. + Map none = new HashMap<>(); + result.add("-D" + CHILD + "=" + new Envelope(none, none, LAUNCH, carrier).encode()); + return result; + } + + public static boolean isOuterGradleExecution() { + StackTraceElement[] stack = Thread.currentThread().getStackTrace(); + if (isOuterGradleExecution(stack)) { + OUTER_ENGINES.incrementAndGet(); + return true; + } + if (countEngines(stack) == 1 && UNOWNED_ENGINE_WARNING.compareAndSet(false, true)) { + Runtime.getRuntime().addShutdownHook(new Thread(ObserverRuntime::warnIfNoOuterEngine)); + } + return false; + } + + private static void warnIfNoOuterEngine() { + if (OUTER_ENGINES.get() == 0) { + System.err.println( + "[dd.trace.observer] WARN - A JUnit Platform engine ran without the Gradle test worker" + + " frame the observer expects, and no outer engine was observed. No tests were" + + " reported from this JVM. Check whether Gradle's JUnit Platform test processor" + + " was renamed."); + } + } + + /** + * Fail closed: only the synchronous Gradle-owned outer engine launch is supported. A nested + * launch on another thread also has a single engine frame, but no Gradle frame. + */ + public static boolean isOuterGradleExecution(StackTraceElement[] stack) { + boolean gradle = false; + for (StackTraceElement frame : stack) { + if (frame + .getClassName() + .equals( + "org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestDefinitionProcessor$CollectThenExecuteTestDefinitionConsumer") + && frame.getMethodName().equals("processAllTestDefinitions")) { + gradle = true; + } + } + return gradle && countEngines(stack) == 1; + } + + private static int countEngines(StackTraceElement[] stack) { + int engines = 0; + for (StackTraceElement frame : stack) { + if (frame + .getClassName() + .equals("org.junit.platform.launcher.core.EngineExecutionOrchestrator") + && frame.getMethodName().equals("executeEngine")) { + engines++; + } + } + return engines; + } + + private static Map toMap(Properties properties) { + Map result = new HashMap<>(); + for (String key : properties.stringPropertyNames()) { + result.put(key, properties.getProperty(key)); + } + return result; + } + + /** Settings a parent hands to a worker. Not encrypted: never put secrets in it. */ + static final class Envelope { + private static final int LIMIT = 1024 * 1024; + private static final String[] SECTIONS = {"config", "environment", "launch", "carrier"}; + + final Map config; + final Map environment; + final Map launch; + final Map carrier; + + Envelope( + Map config, + Map environment, + Map launch, + Map carrier) { + this.config = new TreeMap<>(config); + this.environment = new TreeMap<>(environment); + this.launch = new TreeMap<>(launch); + this.carrier = new TreeMap<>(carrier); + } + + /** Sorted, NUL-separated section, key and value triples, so the encoding is deterministic. */ + String encode() { + StringBuilder text = new StringBuilder(); + List> maps = sections(); + for (int i = 0; i < SECTIONS.length; i++) { + for (Map.Entry e : maps.get(i).entrySet()) { + if (e.getValue() == null) { + continue; + } + if (e.getKey().indexOf('\0') >= 0 || e.getValue().indexOf('\0') >= 0) { + throw new IllegalArgumentException("Observer envelope entries cannot contain NUL"); + } + text.append(SECTIONS[i]).append('\0'); + text.append(e.getKey()).append('\0').append(e.getValue()).append('\0'); + } + } + byte[] bytes = text.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > LIMIT) { + throw new IllegalArgumentException("Observer envelope too large"); + } + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + static Envelope decode(String encoded) { + try { + byte[] bytes = Base64.getUrlDecoder().decode(encoded); + if (bytes.length > LIMIT) { + throw new IllegalArgumentException("too large"); + } + String text = new String(bytes, StandardCharsets.UTF_8); + Envelope envelope = + new Envelope( + new HashMap(), + new HashMap(), + new HashMap(), + new HashMap()); + List> maps = envelope.sections(); + int start = 0; + while (start < text.length()) { + int section = text.indexOf('\0', start); + int key = section < 0 ? -1 : text.indexOf('\0', section + 1); + int value = key < 0 ? -1 : text.indexOf('\0', key + 1); + if (value < 0) { + throw new IllegalArgumentException("truncated entry"); + } + int index = sectionIndex(text.substring(start, section)); + if (maps.get(index).put(text.substring(section + 1, key), text.substring(key + 1, value)) + != null) { + throw new IllegalArgumentException("duplicate key"); + } + start = value + 1; + } + return envelope; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Malformed observer child envelope", e); + } + } + + private static int sectionIndex(String name) { + for (int i = 0; i < SECTIONS.length; i++) { + if (SECTIONS[i].equals(name)) { + return i; + } + } + throw new IllegalArgumentException("unknown section " + name); + } + + private List> sections() { + List> maps = new ArrayList<>(); + maps.add(config); + maps.add(environment); + maps.add(launch); + maps.add(carrier); + return maps; + } + } +} diff --git a/dd-java-agent/observer/src/main/java/datadog/trace/observer/rewriter/ObserverAgentRewriter.java b/dd-java-agent/observer/src/main/java/datadog/trace/observer/rewriter/ObserverAgentRewriter.java new file mode 100644 index 00000000000..f03e604effc --- /dev/null +++ b/dd-java-agent/observer/src/main/java/datadog/trace/observer/rewriter/ObserverAgentRewriter.java @@ -0,0 +1,810 @@ +package datadog.trace.observer.rewriter; + +import static java.util.Arrays.asList; + +import datadog.trace.observer.bootstrap.ObserverBootstrap; +import datadog.trace.observer.bootstrap.ObserverRuntime; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Stream; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Handle; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.FieldInsnNode; +import org.objectweb.asm.tree.FrameNode; +import org.objectweb.asm.tree.InsnList; +import org.objectweb.asm.tree.InsnNode; +import org.objectweb.asm.tree.JumpInsnNode; +import org.objectweb.asm.tree.LabelNode; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodInsnNode; +import org.objectweb.asm.tree.MethodNode; +import org.objectweb.asm.tree.TypeInsnNode; +import org.objectweb.asm.tree.VarInsnNode; + +/** Offline, layout-checked namespace isolation of the stock agent. */ +public final class ObserverAgentRewriter { + private static final String RUNTIME = "datadog/trace/observer/bootstrap/ObserverRuntime"; + private static final String PROVIDER = "datadog/trace/bootstrap/config/provider/ConfigProvider"; + private static final String STABLE_SOURCE = + "datadog/trace/bootstrap/config/provider/StableConfigSource"; + private static final String LOGGER_SETTINGS = + "datadog/trace/logging/simplelogger/SLCompatSettings"; + private static final String JUNIT = "datadog/trace/instrumentation/junit5/"; + private static final String GRADLE = "datadog/trace/instrumentation/gradle/"; + private static final String ADD_TRACING_LISTENER = + "(Lorg/junit/platform/engine/TestEngine;Lorg/junit/platform/engine/ExecutionRequest;)V"; + private static final String STOCK_INDEX = "dd-java-agent.index"; + + /** The javac plugin writes these annotation types into user classes, so both tracers share it. */ + private static final String SHARED_ROOT = "datadog/compiler"; + + private static final List REQUIRED_ENTRIES = + asList( + "datadog/trace/bootstrap/AgentBootstrap.class", + "datadog/trace/bootstrap/AgentPreCheck.class", + PROVIDER + ".class", + "datadog/trace/bootstrap/config/provider/PropertiesConfigSource.class", + STOCK_INDEX, + "inst/instrumenter.index", + "inst/known-types.index"); + + private static final Set SYSTEM_METHODS = + new HashSet<>( + asList("getProperty", "getProperties", "setProperty", "clearProperty", "getenv")); + + /** Changes one stock method and returns how many places it changed. */ + private interface Patch { + int apply(ClassNode owner, MethodNode method); + } + + /** Only checks that the stock method still exists with this signature. */ + private static final Patch VALIDATE = (owner, method) -> 1; + + /** A stock method the observer depends on, and how many times its patch must apply. */ + private static final class Seam { + final String owner; + final String key; + final int expected; + final Patch patch; + + Seam(String owner, String name, String descriptor, int expected, Patch patch) { + this.owner = owner; + this.key = owner + "." + name + descriptor; + this.expected = expected; + this.patch = patch; + } + } + + /** A string constant renamed in every class under {@code ownerPrefix}. */ + private static final class Rename { + final String ownerPrefix; + final String from; + final String to; + final int expected; + + Rename(String ownerPrefix, String from, String to, int expected) { + this.ownerPrefix = ownerPrefix; + this.from = from; + this.to = to; + this.expected = expected; + } + } + + private static final List SEAMS = + asList( + new Seam(PROVIDER, "", "([L" + PROVIDER + "$Source;)V", 1, VALIDATE), + new Seam(PROVIDER, "createDefault", "()L" + PROVIDER + ";", 1, VALIDATE), + new Seam(PROVIDER, "withoutCollector", "()L" + PROVIDER + ";", 1, VALIDATE), + new Seam( + PROVIDER, + "withPropertiesOverride", + "(Ljava/util/Properties;)L" + PROVIDER + ";", + 1, + VALIDATE), + new Seam( + STABLE_SOURCE, + "", + "(Ljava/lang/String;Ldatadog/trace/api/ConfigOrigin;)V", + 1, + ObserverAgentRewriter::observerStableConfig), + new Seam( + LOGGER_SETTINGS, + "loadProperties", + "(Ljava/lang/String;)Ljava/util/Properties;", + 2, + beforeReturns("loggerProperties", "(Ljava/util/Properties;)Ljava/util/Properties;")), + new Seam( + JUNIT + "JUnit5Instrumentation$JUnit5Advice", + "addTracingListener", + ADD_TRACING_LISTENER, + 1, + returnUnless("isOuterGradleExecution", true)), + new Seam( + JUNIT + "JUnit5SpockInstrumentation$SpockAdvice", + "addTracingListener", + ADD_TRACING_LISTENER, + 1, + returnUnless("isOuterGradleExecution", true)), + new Seam( + GRADLE + "GradleDaemonLoggingInstrumentation$ReinitialiseLogging", + "reinitialiseTracerLogging", + "()V", + 1, + returnUnless("usingGeneratedFileLogger", false)), + new Seam( + GRADLE + "CiVisibilityService", + "getTracerJvmArgs", + "(Ljava/lang/String;)Ljava/util/Collection;", + 1, + afterCalls( + "datadog/trace/api/civisibility/domain/BuildModuleSettings", + "getSystemProperties", + "generatedProperties", + "(Ljava/util/Map;)Ljava/util/Map;")), + new Seam( + GRADLE + "TracerArgumentsProvider", + "asArguments", + "()Ljava/lang/Iterable;", + 1, + beforeReturns("childArguments", "(Ljava/lang/Iterable;)Ljava/lang/Iterable;")), + new Seam( + "datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl", + "getPropertiesPropagatedToChildProcess", + "(Ljava/lang/String;ZLjava/lang/String;Ljava/lang/String;Ljava/util/Collection;" + + "Ldatadog/trace/api/civisibility/domain/JavaAgent;Ljava/net/InetSocketAddress;" + + "Ldatadog/trace/civisibility/config/ExecutionSettings;" + + "Ldatadog/trace/api/civisibility/domain/BuildSessionSettings;)Ljava/util/Map;", + 1, + ObserverAgentRewriter::numericHost), + new Seam( + "datadog/trace/civisibility/ProcessHierarchy", + "", + "()V", + 1, + redirectCall( + "datadog/environment/SystemProperties", + "asStringMap", + "()Ljava/util/Map;", + "propagationProperties"))); + + private static final List RENAMES = + asList( + new Rename(GRADLE, "ciVisibilityService", "observerCiVisibilityService", 1), + new Rename(GRADLE, "dd-ci-visibility", "observer-ci-visibility", 2), + new Rename(GRADLE, "moduleLayout", "observerModuleLayout", 3), + new Rename( + LOGGER_SETTINGS, "simplelogger.properties", "observer-simplelogger.properties", 3)); + + private final Set roots; + private final Map seamCounts = new HashMap<>(); + private final Map renameCounts = new HashMap<>(); + private final Set configAliases = new TreeSet<>(); + private final Set sensitiveConfig = new TreeSet<>(); + + ObserverAgentRewriter(Set roots) { + this.roots = roots; + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + throw new IllegalArgumentException("Expected stock-agent.jar observer-agent.jar"); + } + rewrite(new File(args[0]), new File(args[1])); + } + + public static void rewrite(File input, File output) throws Exception { + if (input.getCanonicalFile().equals(output.getCanonicalFile())) { + throw new IllegalArgumentException("Never overwrite the stock agent"); + } + try (JarFile jar = new JarFile(input)) { + for (String required : REQUIRED_ENTRIES) { + require(jar.getJarEntry(required) != null, "Missing expected stock entry: " + required); + } + require( + "datadog.trace.bootstrap.AgentPreCheck" + .equals(jar.getManifest().getMainAttributes().getValue("Premain-Class")), + "Expected a stock agent manifest"); + new ObserverAgentRewriter(roots(jar)).rewrite(input, jar, output); + } + } + + /** Package roots that exist in the stock jar; only references to these are relocated. */ + static Set roots(JarFile jar) { + Set roots = new TreeSet<>(); + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + List parts = segments(entries.nextElement().getName()); + // Only directories count: the segment after the root must not be the file name. + for (int i = 0; i + 2 < parts.size(); i++) { + if (parts.get(i).equals("net") && parts.get(i + 1).equals("bytebuddy")) { + roots.add("net/bytebuddy"); + } + if (parts.get(i).equals("datadog")) { + String parent = i > 0 ? parts.get(i - 1) : ""; + String root = (parent.equals("com") ? "com/datadog/" : "datadog/") + parts.get(i + 1); + if (!root.equals(SHARED_ROOT)) { + roots.add(root); + } + } + } + } + return roots; + } + + private static List segments(String name) { + List parts = new ArrayList<>(); + int start = 0; + for (int slash = name.indexOf('/'); slash >= 0; slash = name.indexOf('/', start)) { + parts.add(name.substring(start, slash)); + start = slash + 1; + } + parts.add(name.substring(start)); + return parts; + } + + private void rewrite(File input, JarFile jar, File output) throws Exception { + Manifest manifest = new Manifest(jar.getManifest()); + Attributes attributes = manifest.getMainAttributes(); + for (String key : asList("Premain-Class", "Agent-Class", "Main-Class")) { + String value = attributes.getValue(key); + if (value != null) { + attributes.putValue(key, relocate(value)); + } + } + attributes.putValue(ObserverRuntime.MANIFEST_ATTRIBUTE, ObserverRuntime.MANIFEST_VERSION); + attributes.putValue("Premain-Class", ObserverBootstrap.class.getName()); + attributes.remove(new Attributes.Name("Agent-Class")); + + Map entries = new LinkedHashMap<>(); + Enumeration source = jar.entries(); + while (source.hasMoreElements()) { + JarEntry entry = source.nextElement(); + String name = entry.getName(); + // Known types are rebuilt from the relocated instrumenter index at startup. + if (entry.isDirectory() + || name.equals("META-INF/MANIFEST.MF") + || name.equals(STOCK_INDEX) + || name.equals("inst/known-types.index")) { + continue; + } + byte[] bytes; + try (InputStream stream = jar.getInputStream(entry)) { + bytes = readAll(stream); + } + if (name.endsWith(".class") || name.endsWith(".classdata")) { + bytes = rewriteClass(bytes); + } else if (name.equals("inst/instrumenter.index")) { + bytes = rewriteInstrumenterIndex(bytes); + } else if (name.contains("META-INF/services/")) { + bytes = + relocate(new String(bytes, StandardCharsets.UTF_8)).getBytes(StandardCharsets.UTF_8); + } + String targetName = relocate(name); + if (name.equals("simplelogger.properties") || name.endsWith("/simplelogger.properties")) { + targetName = + targetName.substring(0, targetName.length() - "simplelogger.properties".length()) + + "observer-simplelogger.properties"; + } + require(entries.put(targetName, bytes) == null, "Duplicate relocated entry: " + name); + } + checkLayout(); + + List> runtimeClasses = + new ArrayList<>(asList(ObserverRuntime.class, ObserverBootstrap.class)); + runtimeClasses.addAll(asList(ObserverRuntime.class.getDeclaredClasses())); + for (Class type : runtimeClasses) { + String resource = type.getName().replace('.', '/') + ".class"; + try (InputStream stream = type.getClassLoader().getResourceAsStream(resource)) { + entries.put(resource, readAll(stream)); + } + } + entries.put(ObserverRuntime.ROOTS_RESOURCE, lines(roots)); + require(!configAliases.isEmpty(), "Missing generated configuration metadata"); + entries.put(ObserverRuntime.ALIASES_RESOURCE, lines(configAliases)); + require(!sensitiveConfig.isEmpty(), "Missing sensitive configuration metadata"); + entries.put(ObserverRuntime.SENSITIVE_RESOURCE, lines(sensitiveConfig)); + List featureRoots = featureRoots(jar); + byte[] index = buildJarIndex(input, entries, featureRoots); + entries.put(relocate(STOCK_INDEX), index); + verifyJarIndex(input, index, entries, featureRoots); + write(output, manifest, entries); + } + + /** Every seam and rename must apply exactly as often as it did on the reviewed stock layout. */ + private void checkLayout() { + Map mismatches = new TreeMap<>(); + for (Seam seam : SEAMS) { + int actual = seamCounts.getOrDefault(seam.key, 0); + if (actual != seam.expected) { + mismatches.put(seam.key, "expected " + seam.expected + ", got " + actual); + } + } + for (Rename rename : RENAMES) { + int actual = renameCounts.getOrDefault(rename, 0); + if (actual != rename.expected) { + mismatches.put( + rename.ownerPrefix + " \"" + rename.from + "\"", + "expected " + rename.expected + ", got " + actual); + } + } + require(mismatches.isEmpty(), "Input agent layout changed: " + mismatches); + } + + String relocate(String value) { + return ObserverRuntime.relocate(value, roots) + .replace("dd-java-agent.index", "dd-observer-agent.index") + .replace("dd-java-agent.version", "dd-observer-agent.version") + .replace("instrumenter.index", "observer-instrumenter.index") + .replace("known-types.index", "observer-known-types.index") + .replace("__datadogContext$", "__datadogObserverContext$"); + } + + byte[] rewriteClass(byte[] bytes) { + ClassNode node = new ClassNode(); + new ClassReader(bytes).accept(node, 0); + Map seams = new HashMap<>(); + for (Seam seam : SEAMS) { + if (seam.owner.equals(node.name)) { + seams.put(seam.key, seam); + } + } + boolean generatedConfigurations = node.name.endsWith("/GeneratedSupportedConfigurations"); + for (MethodNode method : node.methods) { + Seam seam = seams.get(node.name + "." + method.name + method.desc); + if (seam != null) { + seamCounts.merge(seam.key, seam.patch.apply(node, method), Integer::sum); + } + for (AbstractInsnNode instruction : method.instructions) { + if (generatedConfigurations + && instruction instanceof LdcInsnNode + && ((LdcInsnNode) instruction).cst instanceof String) { + String value = (String) ((LdcInsnNode) instruction).cst; + if (method.name.startsWith("initSensitiveKeys")) { + sensitiveConfig.add(value); + } else if (!value.isEmpty() + && (method.name.startsWith("initAliasMapping") + || method.name.startsWith("initDeprecated"))) { + configAliases.add(value); + } + } + if (instruction instanceof MethodInsnNode + && redirectsToRuntime((MethodInsnNode) instruction)) { + ((MethodInsnNode) instruction).owner = RUNTIME; + } + } + } + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + node.accept(new ClassRemapper(writer, new ObserverRemapper(node.name))); + return writer.toByteArray(); + } + + private static boolean redirectsToRuntime(MethodInsnNode call) { + return (call.owner.equals("java/lang/System") && SYSTEM_METHODS.contains(call.name)) + || (call.owner.equals("java/lang/Boolean") && call.name.equals("getBoolean")); + } + + private final class ObserverRemapper extends Remapper { + private final String owner; + + ObserverRemapper(String owner) { + super(Opcodes.ASM9); + this.owner = owner; + } + + @Override + public String map(String name) { + return name.equals(RUNTIME) ? name : relocate(name); + } + + @Override + public String mapFieldName(String owner, String name, String descriptor) { + return name.replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public String mapMethodName(String owner, String name, String descriptor) { + return name.replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public String mapInvokeDynamicMethodName( + String name, String descriptor, Handle bootstrapMethod, Object... arguments) { + return super.mapInvokeDynamicMethodName(name, descriptor, bootstrapMethod, arguments) + .replace("__datadogContext$", "__datadogObserverContext$"); + } + + @Override + public Object mapValue(Object value) { + if (value instanceof String) { + String text = (String) value; + for (Rename rename : RENAMES) { + if (owner.startsWith(rename.ownerPrefix) && text.equals(rename.from)) { + renameCounts.merge(rename, 1, Integer::sum); + return rename.to; + } + } + return relocate(text); + } + if (value instanceof Handle) { + Handle handle = (Handle) value; + if ((handle.getOwner().equals("java/lang/System") + && SYSTEM_METHODS.contains(handle.getName())) + || (handle.getOwner().equals("java/lang/Boolean") + && handle.getName().equals("getBoolean"))) { + return new Handle(handle.getTag(), RUNTIME, handle.getName(), handle.getDesc(), false); + } + } + return super.mapValue(value); + } + } + + /** Stable config never opens host files; the local source carries the observer's defaults. */ + private static int observerStableConfig(ClassNode node, MethodNode method) { + method.instructions.clear(); + method.tryCatchBlocks.clear(); + if (method.localVariables != null) { + method.localVariables.clear(); + } + String config = node.name + "$StableConfig"; + InsnList code = method.instructions; + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add( + new MethodInsnNode(Opcodes.INVOKESPECIAL, PROVIDER + "$Source", "", "()V", false)); + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add(new VarInsnNode(Opcodes.ALOAD, 2)); + code.add( + new FieldInsnNode( + Opcodes.PUTFIELD, node.name, "fileOrigin", "Ldatadog/trace/api/ConfigOrigin;")); + code.add(new VarInsnNode(Opcodes.ALOAD, 0)); + code.add(new TypeInsnNode(Opcodes.NEW, config)); + code.add(new InsnNode(Opcodes.DUP)); + code.add(new InsnNode(Opcodes.ACONST_NULL)); + code.add(new VarInsnNode(Opcodes.ALOAD, 2)); + code.add( + new MethodInsnNode( + Opcodes.INVOKEVIRTUAL, + "datadog/trace/api/ConfigOrigin", + "name", + "()Ljava/lang/String;", + false)); + code.add( + new MethodInsnNode( + Opcodes.INVOKESTATIC, + RUNTIME, + "stableConfig", + "(Ljava/lang/String;)Ljava/util/Map;", + false)); + code.add( + new MethodInsnNode( + Opcodes.INVOKESPECIAL, + config, + "", + "(Ljava/lang/String;Ljava/util/Map;)V", + false)); + code.add(new FieldInsnNode(Opcodes.PUTFIELD, node.name, "config", "L" + config + ";")); + code.add(new InsnNode(Opcodes.RETURN)); + return 1; + } + + /** Passes every returned value through a runtime method with the same type. */ + private static Patch beforeReturns(String runtimeMethod, String descriptor) { + return (owner, method) -> { + int applied = 0; + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction.getOpcode() == Opcodes.ARETURN) { + method.instructions.insertBefore( + instruction, + new MethodInsnNode(Opcodes.INVOKESTATIC, RUNTIME, runtimeMethod, descriptor, false)); + applied++; + } + } + return applied; + }; + } + + /** Returns early from a void method unless a runtime predicate has the given value. */ + private static Patch returnUnless(String runtimeMethod, boolean continueWhen) { + return (owner, method) -> { + InsnList guard = new InsnList(); + LabelNode proceed = new LabelNode(); + guard.add(new MethodInsnNode(Opcodes.INVOKESTATIC, RUNTIME, runtimeMethod, "()Z", false)); + guard.add(new JumpInsnNode(continueWhen ? Opcodes.IFNE : Opcodes.IFEQ, proceed)); + guard.add(new InsnNode(Opcodes.RETURN)); + guard.add(proceed); + guard.add(new FrameNode(Opcodes.F_SAME, 0, null, 0, null)); + method.instructions.insert(guard); + return 1; + }; + } + + /** Passes the result of each matching call through a runtime method with the same type. */ + private static Patch afterCalls( + String callOwner, String callName, String runtimeMethod, String descriptor) { + return (owner, method) -> { + int applied = 0; + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction instanceof MethodInsnNode + && ((MethodInsnNode) instruction).owner.equals(callOwner) + && ((MethodInsnNode) instruction).name.equals(callName)) { + method.instructions.insert( + instruction, + new MethodInsnNode(Opcodes.INVOKESTATIC, RUNTIME, runtimeMethod, descriptor, false)); + applied++; + } + } + return applied; + }; + } + + /** Replaces each matching static call with a runtime method of the same signature. */ + private static Patch redirectCall( + String callOwner, String callName, String descriptor, String runtimeMethod) { + return (owner, method) -> { + int applied = 0; + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals(callOwner) + && call.name.equals(callName) + && call.desc.equals(descriptor)) { + call.owner = RUNTIME; + call.name = runtimeMethod; + applied++; + } + } + } + return applied; + }; + } + + /** Workers get a numeric IPC host, so name resolution cannot pick a different interface. */ + private static int numericHost(ClassNode owner, MethodNode method) { + int applied = 0; + for (AbstractInsnNode instruction : method.instructions.toArray()) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("java/net/InetSocketAddress") && call.name.equals("getHostName")) { + call.name = "getAddress"; + call.desc = "()Ljava/net/InetAddress;"; + method.instructions.insert( + call, + new MethodInsnNode( + Opcodes.INVOKEVIRTUAL, + "java/net/InetAddress", + "getHostAddress", + "()Ljava/lang/String;", + false)); + applied++; + } + } + } + return applied; + } + + /** Relocates module names in the packed instrumenter index, checking the stock layout. */ + byte[] rewriteInstrumenterIndex(byte[] bytes) throws IOException { + DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes)); + int modules = in.readInt(); + int transformations = in.readInt(); + require( + modules > 0 && modules < 10000 && transformations > 0, "Invalid instrumenter index header"); + require(in.readInt() == in.available(), "Invalid packed-name length"); + ByteArrayOutputStream packed = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(packed); + Set found = new HashSet<>(); + int actualTransformations = 0; + for (int i = 0; i < modules; i++) { + String module = readName(in); + require(found.add(module), "Duplicate instrumenter module"); + writeName(out, relocate(module)); + out.writeShort(in.readUnsignedShort()); + int flags = in.readUnsignedByte(); + require((flags & ~3) == 0, "Unknown module flags"); + out.writeByte(flags); + int members = in.readUnsignedByte(); + out.writeByte(members); + // 255 means the module is its own single member. + for (int member = 0; member < (members == 255 ? 1 : members); member++) { + if (members != 255) { + writeName(out, readName(in)); + } + if ((flags & 1) != 0) { + int overrides = in.readUnsignedByte(); + out.writeByte(overrides); + for (int o = 0; o < overrides; o++) { + writeName(out, readName(in)); + out.writeShort(in.readUnsignedShort()); + } + } + } + actualTransformations += members == 255 ? 1 : members; + } + require( + in.available() == 0 && actualTransformations == transformations, + "Instrumenter index layout changed"); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + DataOutputStream header = new DataOutputStream(result); + header.writeInt(modules); + header.writeInt(transformations); + header.writeInt(packed.size()); + packed.writeTo(header); + return result.toByteArray(); + } + + private static String readName(DataInputStream in) throws IOException { + byte[] bytes = new byte[in.readUnsignedByte()]; + in.readFully(bytes); + return new String(bytes, StandardCharsets.ISO_8859_1); + } + + private static void writeName(DataOutputStream out, String name) throws IOException { + require(name.length() < 256, "Indexed name exceeds one-byte length"); + out.writeByte(name.length()); + out.writeBytes(name); + } + + /** Feature roots such as {@code inst/}, read from the stock jar index. */ + private static List featureRoots(JarFile jar) throws IOException { + try (DataInputStream in = new DataInputStream(jar.getInputStream(jar.getEntry(STOCK_INDEX)))) { + List prefixes = new ArrayList<>(); + for (int i = in.readInt(); i > 0; i--) { + prefixes.add(in.readUTF()); + } + return prefixes; + } + } + + /** Runs the stock jar index generator over the relocated feature-root entries. */ + private static byte[] buildJarIndex( + File stock, Map entries, List featureRoots) throws Exception { + Path directory = Files.createTempDirectory("observer-index"); + try { + for (Map.Entry entry : entries.entrySet()) { + if (inFeatureRoot(entry.getKey(), featureRoots)) { + Path file = directory.resolve(entry.getKey()); + Files.createDirectories(file.getParent()); + Files.write(file, entry.getValue()); + } + } + Path index = Files.createTempDirectory("observer-index-output"); + try (URLClassLoader loader = new URLClassLoader(new URL[] {stock.toURI().toURL()}, null)) { + Method main = + loader + .loadClass("datadog.trace.bootstrap.AgentJarIndex$IndexGenerator") + .getMethod("main", String[].class); + main.setAccessible(true); + main.invoke(null, (Object) new String[] {directory.toString(), index.toString()}); + return Files.readAllBytes(index.resolve(STOCK_INDEX)); + } finally { + delete(index); + } + } finally { + delete(directory); + } + } + + /** Reads the generated index with the stock reader and checks it finds every feature class. */ + private static void verifyJarIndex( + File stock, byte[] index, Map entries, List featureRoots) + throws Exception { + Path probe = Files.createTempFile("observer-index", ".jar"); + try { + Map indexOnly = Collections.singletonMap(STOCK_INDEX, index); + write(probe.toFile(), new Manifest(), indexOnly); + try (URLClassLoader loader = new URLClassLoader(new URL[] {stock.toURI().toURL()}, null); + JarFile jar = new JarFile(probe.toFile())) { + Class type = loader.loadClass("datadog.trace.bootstrap.AgentJarIndex"); + Object reader = type.getMethod("readIndex", JarFile.class).invoke(null, jar); + require(reader != null, "Stock reader could not read the observer index"); + Method classEntryName = type.getMethod("classEntryName", String.class); + for (String name : entries.keySet()) { + if (name.endsWith(".classdata") && inFeatureRoot(name, featureRoots)) { + String className = + name.substring(name.indexOf('/') + 1, name.length() - ".classdata".length()) + .replace('/', '.'); + require( + name.equals(classEntryName.invoke(reader, className)), + "Index does not resolve " + className); + } + } + } + } finally { + Files.deleteIfExists(probe); + } + } + + private static boolean inFeatureRoot(String name, List featureRoots) { + for (String root : featureRoots) { + if (name.startsWith(root)) { + return true; + } + } + return false; + } + + /** Writes a sibling file and moves it into place, so a running JVM keeps its old jar. */ + private static void write(File output, Manifest manifest, Map entries) + throws IOException { + Path target = output.getAbsoluteFile().toPath(); + Files.createDirectories(target.getParent()); + Path temporary = target.resolveSibling(target.getFileName() + ".tmp"); + try { + try (JarOutputStream jar = new JarOutputStream(Files.newOutputStream(temporary), manifest)) { + for (Map.Entry entry : entries.entrySet()) { + JarEntry jarEntry = new JarEntry(entry.getKey()); + jarEntry.setTime(0); + jar.putNextEntry(jarEntry); + jar.write(entry.getValue()); + jar.closeEntry(); + } + } + Files.move( + temporary, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } finally { + Files.deleteIfExists(temporary); + } + } + + private static void delete(Path directory) throws IOException { + try (Stream paths = Files.walk(directory)) { + for (Path path : (Iterable) paths.sorted(Comparator.reverseOrder())::iterator) { + Files.delete(path); + } + } + } + + private static byte[] lines(Set values) { + return String.join("\n", values).getBytes(StandardCharsets.UTF_8); + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = stream.read(buffer)) != -1) { + bytes.write(buffer, 0, read); + } + return bytes.toByteArray(); + } + + private static void require(boolean condition, String message) { + if (!condition) { + throw new IllegalArgumentException(message); + } + } +} diff --git a/dd-java-agent/observer/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java b/dd-java-agent/observer/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java new file mode 100644 index 00000000000..783af87e629 --- /dev/null +++ b/dd-java-agent/observer/src/test/java/datadog/trace/observer/bootstrap/ObserverRuntimeTest.java @@ -0,0 +1,257 @@ +package datadog.trace.observer.bootstrap; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ObserverRuntimeTest { + private static final Set ROOTS = + new HashSet<>( + asList("datadog/trace", "datadog/slf4j", "com/datadog/debugger", "net/bytebuddy")); + + private Properties original; + + @BeforeEach + void saveProperties() { + original = (Properties) System.getProperties().clone(); + } + + @AfterEach + void restoreProperties() { + System.setProperties(original); + } + + /** A fresh runtime class with its own static state, plus the test package roots resource. */ + private static Class isolatedRuntime() throws Exception { + URL roots = ObserverRuntimeTest.class.getResource("/" + ObserverRuntime.ROOTS_RESOURCE); + URL resources = Paths.get(roots.toURI()).getParent().toUri().toURL(); + URLClassLoader loader = + new URLClassLoader( + new URL[] { + ObserverRuntime.class.getProtectionDomain().getCodeSource().getLocation(), resources + }, + null); + return loader.loadClass(ObserverRuntime.class.getName()); + } + + private static Object call(Class runtime, String name, Object... arguments) throws Exception { + for (Method method : runtime.getMethods()) { + if (method.getName().equals(name) && method.getParameterCount() == arguments.length) { + return method.invoke(null, arguments); + } + } + throw new NoSuchMethodException(name); + } + + @Test + void relocatesOnlyReferencesToKnownRoots() { + String[][] relocated = { + {"datadog.trace.api.Config", "datadog.trace.observer.trace.api.Config"}, + {"datadog/trace/api/Config", "datadog/trace/observer/trace/api/Config"}, + {"(ILdatadog/trace/api/Config;)V", "(ILdatadog/trace/observer/trace/api/Config;)V"}, + {"com.datadog.debugger.Probe", "datadog.trace.observer.com.datadog.debugger.Probe"}, + {"net.bytebuddy.ByteBuddy", "datadog.trace.observer.net.bytebuddy.ByteBuddy"}, + {"-Dnet.bytebuddy.dump=", "-Ddatadog.trace.observer.net.bytebuddy.dump="}, + {"META-INF/services/datadog.trace.X", "META-INF/services/datadog.trace.observer.trace.X"}, + { + "datadog.slf4j.simpleLogger.log.datadog.trace.api.Config", + "datadog.trace.observer.slf4j.simpleLogger.log.datadog.trace.observer.trace.api.Config" + } + }; + for (String[] pair : relocated) { + assertEquals(pair[1], ObserverRuntime.relocate(pair[0], ROOTS), pair[0]); + } + for (String unchanged : + asList( + "java:comp/env/datadog/tags/", + "/var/run/datadog/apm.socket", + ".inject.datadog.attribute.enabled", + "datadog.span.dispatch", + "datadog.debugger", + "datadog.", + "datadog/compiler/annotations/SourcePath", + "datadog.trace.observer.trace.api.Config", + "__datadogContext$", + "mydatadog.trace.api.Config", + "org.junit.platform.engine.TestEngine")) { + assertEquals(unchanged, ObserverRuntime.relocate(unchanged, ROOTS), unchanged); + } + } + + @Test + void premainNeedsNoRoleOrArtifactMetadataAndRunsOnce() throws Exception { + System.clearProperty(ObserverRuntime.CHILD); + Class runtime = isolatedRuntime(); + call(runtime, "initializePremain"); + assertNull(call(runtime, "getProperty", "dd.writer.type")); + assertNull(call(runtime, "getProperty", "dd.civisibility.enabled")); + assertTrue(((Map) call(runtime, "propagationProperties")).isEmpty()); + assertThrows(InvocationTargetException.class, () -> call(runtime, "initializePremain")); + } + + @Test + void onlyEffectiveGeneratedFileLoggerSkipsDaemonStreamReset() throws Exception { + System.setProperty("tracing.observer.log.directory", "owned-logs"); + String key = "tracing.observer.config.datadog.slf4j.simpleLogger.logFile"; + for (String destination : new String[] {"generated", "System.err", "custom.log"}) { + System.clearProperty(key); + if (!destination.equals("generated")) { + System.setProperty(key, destination); + } + Class runtime = isolatedRuntime(); + assertEquals(destination.equals("generated"), call(runtime, "usingGeneratedFileLogger")); + call(runtime, "setProperty", "datadog.trace.observer.slf4j.simpleLogger.logFile", "x"); + assertEquals(false, call(runtime, "usingGeneratedFileLogger")); + } + } + + @Test + void envelopeRoundTripIsLosslessAndDeterministic() { + Map values = new HashMap<>(); + values.put("traceparent", "00-1234-5678-01"); + values.put("x-datadog-parent-id", "18446744073709551615"); + values.put("baggage", "space=\"quoted\",unicode=\u03b1\n${projectProperty}"); + ObserverRuntime.Envelope envelope = + new ObserverRuntime.Envelope(values, singletonMap("DD_TAGS", "a:b c"), emptyMap(), values); + String encoded = envelope.encode(); + assertFalse(encoded.contains("${")); + assertEquals( + encoded, + new ObserverRuntime.Envelope(values, envelope.environment, emptyMap(), values).encode()); + ObserverRuntime.Envelope decoded = ObserverRuntime.Envelope.decode(encoded); + assertEquals(envelope.config, decoded.config); + assertEquals(envelope.environment, decoded.environment); + assertEquals(envelope.carrier, decoded.carrier); + assertTrue(decoded.launch.isEmpty()); + assertThrows(IllegalArgumentException.class, () -> ObserverRuntime.Envelope.decode("invalid")); + assertThrows( + IllegalArgumentException.class, + () -> + new ObserverRuntime.Envelope( + singletonMap("k", "a\0b"), emptyMap(), emptyMap(), emptyMap()) + .encode()); + } + + @Test + void workerReadsParentSnapshotWithPrivateMutableOverlay() throws Exception { + Map config = new HashMap<>(); + config.put("dd.service", "private-service"); + config.put("future.non.dd.key", "future-value"); + Map carrier = new HashMap<>(); + carrier.put("traceparent", "private-carrier"); + carrier.put("dd.civisibility.build.instrumentation.enabled", "false"); + System.setProperty( + ObserverRuntime.CHILD, + new ObserverRuntime.Envelope( + config, singletonMap("DD_TAGS", "source:environment"), emptyMap(), carrier) + .encode()); + System.setProperty("dd.service", "subject-service"); + System.setProperty("traceparent", "subject-carrier"); + Class runtime = isolatedRuntime(); + assertEquals("private-service", call(runtime, "getProperty", "dd.service")); + assertEquals("source:environment", call(runtime, "getenv", "DD_TAGS")); + assertNull(call(runtime, "getProperty", "dd.tags")); + System.setProperty("dd.service", "subject-mutated"); + assertEquals("private-service", call(runtime, "getProperty", "dd.service")); + Properties snapshot = (Properties) call(runtime, "getProperties"); + assertEquals("future-value", snapshot.getProperty("future.non.dd.key")); + snapshot.setProperty("dd.service", "copy-mutated"); + assertEquals("private-service", call(runtime, "getProperty", "dd.service")); + assertEquals(carrier, call(runtime, "propagationProperties")); + assertEquals("subject-carrier", System.getProperty("traceparent")); + assertNull(call(runtime, "setProperty", "dd.civisibility.signal.server.port", "12345")); + assertEquals("12345", call(runtime, "getProperty", "dd.civisibility.signal.server.port")); + assertNull(System.getProperty("dd.civisibility.signal.server.port")); + assertEquals("12345", call(runtime, "clearProperty", "dd.civisibility.signal.server.port")); + } + + @Test + void fingerprintIsComputedOnDemandFromTheCallerConfiguration() throws Exception { + System.setProperty("tracing.observer.config.dd.service", "first"); + Object first = call(isolatedRuntime(), "configurationFingerprint"); + assertEquals(first, call(isolatedRuntime(), "configurationFingerprint")); + System.setProperty("tracing.observer.config.dd.service", "second"); + assertFalse(first.equals(call(isolatedRuntime(), "configurationFingerprint"))); + } + + @Test + void localStableConfigCarriesTheObserverDefaults() { + Map local = ObserverRuntime.stableConfig("LOCAL_STABLE_CONFIG"); + for (String name : asList("JUNIT_4", "TESTNG", "KARATE", "SCALATEST", "WEAVER", "CUCUMBER")) { + assertEquals("false", local.get("DD_TRACE_" + name + "_ENABLED"), name); + } + assertEquals("datadog.*:com.datadog.*", local.get("DD_CIVISIBILITY_CODE_COVERAGE_INCLUDES")); + assertEquals(7, local.size()); + assertTrue(ObserverRuntime.stableConfig("FLEET_STABLE_CONFIG").isEmpty()); + } + + @Test + void fullNameNamespaceDoesNotWhitelistKeysOrRewriteValues() throws Exception { + System.setProperty("tracing.observer.config.dd.future-key", "datadog.value=\u03b1"); + System.setProperty( + "tracing.observer.config.datadog.slf4j.simpleLogger.defaultLogLevel", "WARN"); + Class runtime = isolatedRuntime(); + assertEquals("datadog.value=\u03b1", call(runtime, "getProperty", "dd.future-key")); + assertEquals( + "WARN", + call(runtime, "getProperty", "datadog.trace.observer.slf4j.simpleLogger.defaultLogLevel")); + assertNull(System.getProperty("dd.future-key")); + } + + @Test + void loggerKeysRelocateClassSuffixesOnceWithoutRewritingValues() throws Exception { + Properties properties = new Properties(); + properties.setProperty("datadog.slf4j.simpleLogger.log.datadog.trace.api.Config", "datadog.v"); + properties.setProperty("datadog.slf4j.simpleLogger.log.net.bytebuddy.ByteBuddy", "datadog.v"); + Class runtime = isolatedRuntime(); + Properties translated = (Properties) call(runtime, "loggerProperties", properties); + assertEquals( + "datadog.v", + translated.getProperty( + "datadog.trace.observer.slf4j.simpleLogger.log.datadog.trace.observer.trace.api.Config")); + assertEquals( + "datadog.v", + translated.getProperty( + "datadog.trace.observer.slf4j.simpleLogger.log.datadog.trace.observer.net.bytebuddy.ByteBuddy")); + assertEquals(translated, call(runtime, "loggerProperties", translated)); + } + + @Test + void onlyTheOuterGradleEngineLaunchOwnsObserverListeners() { + StackTraceElement engine = + new StackTraceElement( + "org.junit.platform.launcher.core.EngineExecutionOrchestrator", "executeEngine", "", 1); + StackTraceElement gradle = + new StackTraceElement( + "org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestDefinitionProcessor$CollectThenExecuteTestDefinitionConsumer", + "processAllTestDefinitions", + "", + 1); + assertTrue(ObserverRuntime.isOuterGradleExecution(new StackTraceElement[] {engine, gradle})); + assertFalse( + ObserverRuntime.isOuterGradleExecution(new StackTraceElement[] {engine, engine, gradle})); + // A nested launch on another thread has a single engine frame but no Gradle frame. + assertFalse(ObserverRuntime.isOuterGradleExecution(new StackTraceElement[] {engine})); + assertFalse(ObserverRuntime.isOuterGradleExecution(new StackTraceElement[] {gradle})); + assertFalse(ObserverRuntime.isOuterGradleExecution(new StackTraceElement[0])); + } +} diff --git a/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverAgentRewriterTest.java b/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverAgentRewriterTest.java new file mode 100644 index 00000000000..112411de686 --- /dev/null +++ b/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverAgentRewriterTest.java @@ -0,0 +1,300 @@ +package datadog.trace.observer.rewriter; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Handle; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.FieldInsnNode; +import org.objectweb.asm.tree.InvokeDynamicInsnNode; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodInsnNode; +import org.objectweb.asm.tree.MethodNode; + +class ObserverAgentRewriterTest { + private static final Set ROOTS = + new HashSet<>(asList("datadog/trace", "com/datadog/debugger", "net/bytebuddy")); + private static final String PROVIDER = "datadog/trace/bootstrap/config/provider/ConfigProvider"; + + @Test + void neverOverwritesTheStockArtifact(@TempDir Path directory) throws IOException { + File input = directory.resolve("stock.jar").toFile(); + Files.write(input.toPath(), "unchanged".getBytes(StandardCharsets.UTF_8)); + assertThrows(IllegalArgumentException.class, () -> ObserverAgentRewriter.rewrite(input, input)); + assertEquals( + "unchanged", new String(Files.readAllBytes(input.toPath()), StandardCharsets.UTF_8)); + } + + @Test + void refusesAnIncompleteStockLayoutBeforeWritingOutput(@TempDir Path directory) + throws IOException { + File input = directory.resolve("stock.jar").toFile(); + File output = directory.resolve("observer.jar").toFile(); + writeJar(input, new LinkedHashMap<>()); + IllegalArgumentException failure = + assertThrows( + IllegalArgumentException.class, () -> ObserverAgentRewriter.rewrite(input, output)); + assertTrue(failure.getMessage().contains("Missing expected stock entry")); + assertFalse(output.exists()); + } + + @Test + void reportsEveryMissingSeamBeforeWritingOutput(@TempDir Path directory) throws Exception { + Map entries = new LinkedHashMap<>(); + for (String name : + asList( + "datadog/trace/bootstrap/AgentBootstrap", + "datadog/trace/bootstrap/AgentPreCheck", + PROVIDER, + "datadog/trace/bootstrap/config/provider/PropertiesConfigSource")) { + entries.put(name + ".class", emptyClass(name)); + } + entries.put("dd-java-agent.index", new byte[0]); + entries.put( + "inst/instrumenter.index", instrumenterIndex(asList("datadog.trace.instrumentation.a.A"))); + entries.put("inst/known-types.index", new byte[0]); + File input = directory.resolve("stock.jar").toFile(); + File output = directory.resolve("observer.jar").toFile(); + writeJar(input, entries); + IllegalArgumentException failure = + assertThrows( + IllegalArgumentException.class, () -> ObserverAgentRewriter.rewrite(input, output)); + assertTrue(failure.getMessage().startsWith("Input agent layout changed"), failure.getMessage()); + assertTrue( + failure + .getMessage() + .contains(PROVIDER + ".createDefault()L" + PROVIDER + ";=expected 1, got 0")); + assertTrue(failure.getMessage().contains("\"moduleLayout\"=expected 3, got 0")); + assertFalse(output.exists()); + } + + @Test + void rewritingTheStockAgentIsDeterministic(@TempDir Path directory) throws Exception { + File output = directory.resolve("observer.jar").toFile(); + ObserverAgentRewriter.rewrite(new File(System.getProperty("observer.test.stock")), output); + try (JarFile actual = new JarFile(output); + JarFile expected = new JarFile(System.getProperty("observer.test.artifact"))) { + assertEquals(expected.getManifest(), actual.getManifest()); + List names = new ArrayList<>(); + for (JarEntry entry : java.util.Collections.list(expected.entries())) { + names.add(entry.getName()); + JarEntry rewritten = actual.getJarEntry(entry.getName()); + assertTrue(rewritten != null, entry.getName()); + assertTrue( + java.util.Arrays.equals(read(expected, entry), read(actual, rewritten)), + entry.getName()); + } + assertEquals(names.size(), java.util.Collections.list(actual.entries()).size()); + } + } + + private static byte[] read(JarFile jar, JarEntry entry) throws IOException { + try (java.io.InputStream in = jar.getInputStream(entry)) { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + for (int n = in.read(buffer); n != -1; n = in.read(buffer)) { + bytes.write(buffer, 0, n); + } + return bytes.toByteArray(); + } + } + + @Test + void rootsComeFromJarDirectoriesAndKeepTheJavacPluginShared(@TempDir Path directory) + throws IOException { + Map entries = new LinkedHashMap<>(); + for (String name : + asList( + "inst/datadog/trace/instrumentation/A.classdata", + "com/datadog/debugger/B.class", + "inst/net/bytebuddy/C.classdata", + "datadog/compiler/annotations/SourcePath.class", + "datadog/version.txt")) { + entries.put(name, new byte[0]); + } + File jar = directory.resolve("stock.jar").toFile(); + writeJar(jar, entries); + try (JarFile file = new JarFile(jar)) { + assertEquals( + new TreeSet<>(asList("com/datadog/debugger", "datadog/trace", "net/bytebuddy")), + ObserverAgentRewriter.roots(file)); + } + } + + @Test + void relocationAlsoRenamesAgentResourcesAndTheContextProtocol() { + ObserverAgentRewriter rewriter = new ObserverAgentRewriter(ROOTS); + assertEquals("inst/observer-instrumenter.index", rewriter.relocate("inst/instrumenter.index")); + assertEquals("dd-observer-agent.index", rewriter.relocate("dd-java-agent.index")); + assertEquals("__datadogObserverContext$0", rewriter.relocate("__datadogContext$0")); + assertEquals("java:comp/env/datadog/tags/", rewriter.relocate("java:comp/env/datadog/tags/")); + } + + @Test + void retainsTheFullCatalogAndValidatesThePackedIndex() throws IOException { + List names = + asList( + "datadog.trace.instrumentation.junit5.JUnit5Instrumentation", + "datadog.trace.instrumentation.junit5.JUnit5SkipInstrumentation", + "datadog.trace.instrumentation.gradle.GradleDaemonLoggingInstrumentation"); + byte[] index = instrumenterIndex(names); + ObserverAgentRewriter rewriter = new ObserverAgentRewriter(ROOTS); + byte[] result = rewriter.rewriteInstrumenterIndex(index); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(result))) { + assertEquals(names.size(), in.readInt()); + assertEquals(names.size(), in.readInt()); + assertEquals(in.available() - 4, in.readInt()); + } + String text = new String(result, StandardCharsets.ISO_8859_1); + assertTrue( + text.contains("datadog.trace.observer.trace.instrumentation.junit5.JUnit5Instrumentation")); + assertTrue(text.contains("JUnit5SkipInstrumentation")); + byte[] trailing = new byte[index.length + 1]; + System.arraycopy(index, 0, trailing, 0, index.length); + assertThrows(IllegalArgumentException.class, () -> rewriter.rewriteInstrumenterIndex(trailing)); + } + + @Test + void contextProtocolNamesAreIsolatedInDefinitionsCallsConstantsAndDynamicNames() { + String owner = "datadog/trace/bootstrap/FieldBackedContextAccessor"; + ClassWriter writer = new ClassWriter(0); + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, owner, null, "java/lang/Object", null); + writer.visitField(Opcodes.ACC_PUBLIC, "__datadogContext$0", "Ljava/lang/Object;", null, null); + MethodVisitor method = + writer.visitMethod(Opcodes.ACC_PUBLIC, "$get$__datadogContext$", "()V", null, null); + method.visitCode(); + method.visitLdcInsn("__datadogContext$"); + method.visitInsn(Opcodes.POP); + method.visitFieldInsn(Opcodes.GETSTATIC, owner, "__datadogContext$0", "Ljava/lang/Object;"); + method.visitInsn(Opcodes.POP); + method.visitMethodInsn(Opcodes.INVOKESTATIC, owner, "$put$__datadogContext$", "()V", false); + method.visitInvokeDynamicInsn( + "$get$__datadogContext$", + "()V", + new Handle(Opcodes.H_INVOKESTATIC, owner, "$put$__datadogContext$", "()V", false)); + method.visitInsn(Opcodes.RETURN); + method.visitMaxs(1, 1); + method.visitEnd(); + ClassNode node = rewrite(writer); + assertEquals("__datadogObserverContext$0", node.fields.get(0).name); + MethodNode rewritten = node.methods.get(0); + assertEquals("$get$__datadogObserverContext$", rewritten.name); + AbstractInsnNode[] instructions = rewritten.instructions.toArray(); + assertEquals("__datadogObserverContext$", ((LdcInsnNode) instructions[0]).cst); + assertEquals("__datadogObserverContext$0", ((FieldInsnNode) instructions[2]).name); + assertEquals("$put$__datadogObserverContext$", ((MethodInsnNode) instructions[4]).name); + InvokeDynamicInsnNode dynamic = (InvokeDynamicInsnNode) instructions[5]; + assertEquals("$get$__datadogObserverContext$", dynamic.name); + assertEquals("$put$__datadogObserverContext$", dynamic.bsm.getName()); + } + + @Test + void validatedFactoriesKeepTheirBodies() { + ClassWriter writer = new ClassWriter(0); + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, PROVIDER, null, "java/lang/Object", null); + MethodVisitor method = + writer.visitMethod( + Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, + "withPropertiesOverride", + "(Ljava/util/Properties;)L" + PROVIDER + ";", + null, + null); + method.visitCode(); + method.visitVarInsn(Opcodes.ALOAD, 0); + method.visitInsn(Opcodes.POP); + method.visitInsn(Opcodes.ACONST_NULL); + method.visitInsn(Opcodes.ARETURN); + method.visitMaxs(1, 1); + method.visitEnd(); + List opcodes = new ArrayList<>(); + for (AbstractInsnNode instruction : rewrite(writer).methods.get(0).instructions) { + opcodes.add(instruction.getOpcode()); + } + assertEquals(asList(Opcodes.ALOAD, Opcodes.POP, Opcodes.ACONST_NULL, Opcodes.ARETURN), opcodes); + } + + private static ClassNode rewrite(ClassWriter writer) { + writer.visitEnd(); + ClassNode node = new ClassNode(); + new ClassReader(new ObserverAgentRewriter(ROOTS).rewriteClass(writer.toByteArray())) + .accept(node, 0); + return node; + } + + private static byte[] emptyClass(String name) { + ClassWriter writer = new ClassWriter(0); + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, name, null, "java/lang/Object", null); + writer.visitEnd(); + return writer.toByteArray(); + } + + /** Modules that are their own single member, with one target-system override each. */ + private static byte[] instrumenterIndex(List names) throws IOException { + ByteArrayOutputStream packed = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(packed)) { + for (String name : names) { + out.writeByte(name.length()); + out.writeBytes(name); + out.writeShort(64); + out.writeByte(1); + out.writeByte(255); + out.writeByte(1); + out.writeByte(6); + out.writeBytes("Advice"); + out.writeShort(64); + } + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + out.writeInt(names.size()); + out.writeInt(names.size()); + out.writeInt(packed.size()); + out.write(packed.toByteArray()); + } + return bytes.toByteArray(); + } + + private static void writeJar(File jar, Map entries) throws IOException { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().putValue("Premain-Class", "datadog.trace.bootstrap.AgentPreCheck"); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jar.toPath()), manifest)) { + for (Map.Entry entry : entries.entrySet()) { + out.putNextEntry(new JarEntry(entry.getKey())); + out.write(entry.getValue()); + out.closeEntry(); + } + } + } +} diff --git a/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverConfigSourcesTest.java b/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverConfigSourcesTest.java new file mode 100644 index 00000000000..12e0a7828d9 --- /dev/null +++ b/dd-java-agent/observer/src/test/java/datadog/trace/observer/rewriter/ObserverConfigSourcesTest.java @@ -0,0 +1,814 @@ +package datadog.trace.observer.rewriter; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.ClassRemapper; +import org.objectweb.asm.commons.Remapper; +import org.objectweb.asm.tree.AbstractInsnNode; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.MethodInsnNode; +import org.objectweb.asm.tree.MethodNode; + +/** Differential source tests without installing either tracer or reading host stable files. */ +class ObserverConfigSourcesTest { + private static final String STOCK = "datadog.trace."; + private static final String OBSERVER = "datadog.trace.observer.trace."; + private static final String RUNTIME = "datadog/trace/observer/bootstrap/ObserverRuntime"; + + /** Only test environment transport is substituted, below the stock config helper. */ + public static class Environment { + public static Map values = new HashMap<>(); + + public static String getenv(String key) { + return values.get(key); + } + + public static Map getenv() { + return new HashMap<>(values); + } + } + + @Test + void defaultLoggerResourceIsPrivateButExplicitSelectionUsesOrdinaryKeys(@TempDir Path directory) + throws Exception { + Files.write( + directory.resolve("simplelogger.properties"), + "datadog.slf4j.simpleLogger.defaultLogLevel=ERROR\n".getBytes("ISO-8859-1")); + Properties original = (Properties) System.getProperties().clone(); + ClassLoader context = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader resources = + new URLClassLoader(new URL[] {directory.toUri().toURL()}, null)) { + Thread.currentThread().setContextClassLoader(resources); + for (boolean explicit : new boolean[] {false, true}) { + String key = "tracing.observer.config.datadog.slf4j.simpleLogger.configurationFile"; + if (explicit) { + System.setProperty(key, "simplelogger.properties"); + } else { + System.clearProperty(key); + } + try (SourceLoader loader = loader(true)) { + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + Properties properties = (Properties) runtime.getMethod("getProperties").invoke(null); + Class settings = loader.loadClass(OBSERVER + "logging.simplelogger.SLCompatSettings"); + Object instance = settings.getConstructor(Properties.class).newInstance(properties); + Map description = + (Map) settings.getMethod("getSettingsDescription").invoke(instance); + assertEquals(explicit ? "ERROR" : "INFO", description.get("defaultLogLevel")); + assertEquals( + explicit ? "simplelogger.properties" : "observer-simplelogger.properties", + description.get("configurationFile")); + } + } + } finally { + System.setProperties(original); + Thread.currentThread().setContextClassLoader(context); + } + } + + @Test + void stockFactoriesAndPrivateSourcesAgree(@TempDir Path directory) throws Exception { + Path file = directory.resolve("observer.properties"); + Files.write( + file, + ("dd.service=file-service\ndd.trace.sample.rate=0.25\ndd.tags=file:yes,shared:file\ndd.api-key=file-dummy\n") + .getBytes("ISO-8859-1")); + for (boolean withFile : new boolean[] {false, true}) { + for (String factory : + new String[] {"createDefault", "withoutCollector", "withPropertiesOverride"}) { + Map stock = evaluate(false, factory, withFile ? file : null); + Map observer = evaluate(true, factory, withFile ? file : null); + assertEquals(stock, observer, factory + " file=" + withFile); + assertEquals( + factory.equals("withPropertiesOverride") && withFile ? "provided" : "high-alias", + observer.get("service")); + assertEquals(71, observer.get("numeric")); + assertEquals(false, observer.get("boolean")); + assertEquals("env-dummy", observer.get("key")); + assertEquals(!factory.equals("withoutCollector"), observer.get("collect")); + @SuppressWarnings("unchecked") + Map tags = (Map) observer.get("tags"); + assertEquals("property", tags.get("shared")); + assertEquals("yes", tags.get("env")); + if (withFile) { + assertEquals("yes", tags.get("file")); + } + } + } + } + + @Test + void propertyCredentialsRemainExcludedAndExplicitFilesAreReread(@TempDir Path directory) + throws Exception { + Path file = directory.resolve("observer.properties"); + Files.write(file, "dd.api-key=file-dummy\n".getBytes("ISO-8859-1")); + Properties original = (Properties) System.getProperties().clone(); + try (SourceLoader loader = loader(true)) { + System.setProperty("tracing.observer.config.dd.api-key", "ignored-property-dummy"); + System.setProperty("dd.trace.config", "/subject-file-must-not-be-read"); + Class provider = loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object first = provider.getMethod("createDefault").invoke(null); + Class system = + loader.loadClass(OBSERVER + "bootstrap.config.provider.SystemPropertiesConfigSource"); + assertNull( + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(first, "api-key", null, system, new String[0])); + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + runtime + .getMethod("setProperty", String.class, String.class) + .invoke(null, "dd.trace.config", file.toString()); + Object second = provider.getMethod("createDefault").invoke(null); + assertEquals( + "file-dummy", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(second, "api-key", null, system, new String[0])); + Files.write(file, "dd.api-key=file-dummy-2\n".getBytes("ISO-8859-1")); + Object third = provider.getMethod("createDefault").invoke(null); + assertEquals( + "file-dummy-2", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(third, "api-key", null, system, new String[0])); + assertEquals("/subject-file-must-not-be-read", System.getProperty("dd.trace.config")); + Class stable = loader.loadClass(OBSERVER + "bootstrap.config.provider.StableConfigSource"); + // No host stable-config file is read: local only carries the observer's own defaults. + assertEquals( + new java.util.HashSet<>( + asList( + "DD_TRACE_JUNIT_4_ENABLED", + "DD_TRACE_TESTNG_ENABLED", + "DD_TRACE_KARATE_ENABLED", + "DD_TRACE_SCALATEST_ENABLED", + "DD_TRACE_WEAVER_ENABLED", + "DD_TRACE_CUCUMBER_ENABLED", + "DD_CIVISIBILITY_CODE_COVERAGE_INCLUDES")), + stable.getMethod("getKeys").invoke(stable.getField("LOCAL").get(null))); + assertEquals( + java.util.Collections.emptySet(), + stable.getMethod("getKeys").invoke(stable.getField("FLEET").get(null))); + } finally { + System.setProperties(original); + } + } + + @Test + void ordinaryDestinationsProductsRoleControlsAndFilesAreNotRestricted(@TempDir Path directory) + throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path key = directory.resolve("owned-dummy.key"); + Files.write(key, "ordinary-offline-dummy".getBytes("UTF-8")); + Map values = new LinkedHashMap<>(); + values.put("site", "datad0g.com"); + values.put("civisibility.agentless.url", "https://example.invalid/tests"); + values.put("trace.agent.url", "https://example.invalid/agent"); + values.put("api-key-file", key.toString()); + values.put("application-key-file", key.toString()); + values.put("profiling.apikey.file", key.toString()); + values.put("profiling.enabled", "true"); + values.put("civisibility.auto.configuration.enabled", "false"); + values.put("civisibility.build.instrumentation.enabled", "false"); + values.put("writer.type", "DDAgentWriter"); + try { + for (String source : new String[] {"property", "environment", "file", "override"}) { + System.setProperties((Properties) original.clone()); + Properties fileValues = new Properties(); + Map environment = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + String full = "dd." + entry.getKey(); + if (source.equals("property")) { + System.setProperty("tracing.observer.config." + full, entry.getValue()); + } else if (source.equals("environment")) { + environment.put( + "TRACING_OBSERVER_CONFIG_" + + full.toUpperCase(Locale.ROOT).replace('.', '_').replace('-', '_'), + entry.getValue()); + } else { + fileValues.setProperty(source.equals("file") ? full : entry.getKey(), entry.getValue()); + } + } + Path config = directory.resolve("ordinary.properties"); + if (source.equals("file")) { + try (java.io.OutputStream out = Files.newOutputStream(config)) { + fileValues.store(out, "offline owned inputs"); + } + System.setProperty("tracing.observer.config.dd.trace.config", config.toString()); + } + try (SourceLoader loader = loader(true)) { + loader.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class provider = + loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object instance = + source.equals("override") + ? provider + .getMethod("withPropertiesOverride", Properties.class) + .invoke(null, fileValues) + : provider.getMethod("createDefault").invoke(null); + for (Map.Entry entry : values.entrySet()) { + assertEquals( + entry.getValue(), + configString(provider, instance, entry.getKey()), + source + " " + entry.getKey()); + } + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void childTransportKeepsCarrierAndInheritedEnvironmentWithoutSerializingCredentials( + @TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path file = directory.resolve("roundtrip.properties"); + Files.write(file, "dd.service=file-parent\n".getBytes("ISO-8859-1")); + Map environment = new HashMap<>(); + environment.put("TRACING_OBSERVER_CONFIG_DD_ENV", "private-environment"); + environment.put("TRACING_OBSERVER_CONFIG_DD_API_KEY", "owned-secret-marker"); + try (SourceLoader parent = loader(true)) { + System.clearProperty("tracing.observer.child.v2"); + System.setProperty("tracing.observer.config.dd.trace.config", file.toString()); + parent.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class runtime = parent.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + initializeParent(runtime); + Map generated = new HashMap<>(); + generated.put("dd.civisibility.build.instrumentation.enabled", "false"); + generated.put("traceparent", "private-parent-context"); + generated.put("tracestate", "dd=s:1"); + generated.put("baggage", "a=b"); + generated.put("dd.api-key", "owned-secret-marker"); + @SuppressWarnings("unchecked") + Map marked = + (Map) + runtime.getMethod("generatedProperties", Map.class).invoke(null, generated); + assertTrue(marked.values().stream().noneMatch("owned-secret-marker"::equals)); + List stockArguments = new java.util.ArrayList<>(); + marked.forEach((key, value) -> stockArguments.add("-D" + key + "=" + value)); + stockArguments.add("-javaagent:owned.jar"); + stockArguments.add("-Xmx256m"); + @SuppressWarnings("unchecked") + List arguments = + (List) + runtime.getMethod("childArguments", Iterable.class).invoke(null, stockArguments); + assertEquals( + stockArguments.subList(stockArguments.size() - 2, stockArguments.size()), + arguments.subList(0, 2)); + String child = arguments.get(2).substring("-Dtracing.observer.child.v2=".length()); + assertTrue( + !new String(java.util.Base64.getUrlDecoder().decode(child), "UTF-8") + .contains("owned-secret-marker")); + Files.write(file, "dd.service=file-worker\n".getBytes("ISO-8859-1")); + System.setProperty("tracing.observer.child.v2", child); + try (SourceLoader worker = loader(true)) { + worker.loadClass(Environment.class.getName()).getField("values").set(null, environment); + Class provider = worker.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object config = provider.getMethod("createDefault").invoke(null); + assertEquals("private-environment", configString(provider, config, "env")); + assertEquals("file-worker", configString(provider, config, "service")); + assertEquals("owned-secret-marker", configString(provider, config, "api-key")); + assertEquals( + "false", configString(provider, config, "civisibility.build.instrumentation.enabled")); + Class workerRuntime = + worker.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + Map carrier = + (Map) workerRuntime.getMethod("propagationProperties").invoke(null); + assertEquals("private-parent-context", carrier.get("traceparent")); + assertEquals("dd=s:1", carrier.get("tracestate")); + assertEquals("a=b", carrier.get("baggage")); + } + } finally { + System.setProperties(original); + } + } + + @Test + void artifactRetainsCatalogCapabilitiesAndStockGradleArgumentBody() throws Exception { + try (java.util.jar.JarFile stock = + new java.util.jar.JarFile(System.getProperty("observer.test.stock")); + java.util.jar.JarFile observer = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + java.io.DataInputStream stockIndex = + new java.io.DataInputStream( + stock.getInputStream(stock.getJarEntry("inst/instrumenter.index"))); + java.io.DataInputStream observerIndex = + new java.io.DataInputStream( + observer.getInputStream(observer.getJarEntry("inst/observer-instrumenter.index"))); + int modules = stockIndex.readInt(); + assertTrue(modules > 6); + assertEquals(modules, observerIndex.readInt()); + assertEquals(stockIndex.readInt(), observerIndex.readInt()); + for (String[] seam : + new String[][] { + {"junit5/JUnitPlatformUtils", "capabilities"}, + {"gradle/CiVisibilityService", "getTracerJvmArgs"}, + {"gradle/CiVisibilityPluginExtension", "applyJacocoSettings"}, + {"gradle/TracerArgumentsProvider", "replaceProjectProperties"} + }) { + List> bodies = new java.util.ArrayList<>(); + for (boolean relocated : new boolean[] {false, true}) { + java.util.jar.JarFile jar = relocated ? observer : stock; + String resource = + "inst/datadog/" + + (relocated ? "trace/observer/" : "") + + "trace/instrumentation/" + + seam[0] + + ".classdata"; + ClassNode node = new ClassNode(); + new ClassReader(SourceLoader.read(jar.getInputStream(jar.getJarEntry(resource)))) + .accept(node, 0); + MethodNode method = + node.methods.stream().filter(m -> m.name.equals(seam[1])).findFirst().get(); + List body = new java.util.ArrayList<>(); + for (AbstractInsnNode instruction : method.instructions) { + if (!(instruction instanceof MethodInsnNode + && ((MethodInsnNode) instruction).name.equals("generatedProperties"))) { + body.add(instruction.getOpcode()); + } + } + bodies.add(body); + } + assertEquals(bodies.get(0), bodies.get(1), seam[0] + "." + seam[1]); + } + } + } + + @Test + void artifactKeepsExternalLiteralsThatLookLikePackages() throws Exception { + try (java.util.jar.JarFile observer = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + for (String[] expected : + new String[][] { + { + "datadog/trace/observer/trace/api/ConfigDefaults.class", "/var/run/datadog/apm.socket" + }, + {"datadog/trace/observer/trace/api/Config.class", ".inject.datadog.attribute.enabled"}, + { + "trace/datadog/trace/observer/trace/agent/core/otlp/metrics/OtlpStatsMetricWriter.classdata", + "datadog.origin" + }, + { + "logs-intake/datadog/trace/observer/trace/logging/intake/LogsWriterImpl.classdata", + "datadog.product:" + }, + { + "datadog/trace/observer/trace/api/ClassloaderConfigurationOverrides.class", + "java:comp/env/datadog/tags/" + }, + { + "ci-visibility/datadog/trace/observer/trace/civisibility/compiler/CompilerModuleExporter.classdata", + "datadog/compiler/" + } + }) { + java.util.jar.JarEntry entry = observer.getJarEntry(expected[0]); + assertTrue(entry != null, expected[0]); + List constants = new java.util.ArrayList<>(); + ClassNode node = new ClassNode(); + new ClassReader(SourceLoader.read(observer.getInputStream(entry))).accept(node, 0); + node.fields.forEach(field -> constants.add(field.value)); + for (MethodNode method : node.methods) { + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof org.objectweb.asm.tree.LdcInsnNode) { + constants.add(((org.objectweb.asm.tree.LdcInsnNode) instruction).cst); + } + } + } + assertTrue(constants.contains(expected[1]), expected[0] + " " + expected[1]); + } + // The javac plugin writes these annotation types into user classes; both tracers share them. + assertTrue(observer.getJarEntry("datadog/compiler/annotations/SourcePath.class") != null); + assertTrue( + observer.getJarEntry("datadog/trace/observer/compiler/annotations/SourcePath.class") + == null); + ClassNode utils = new ClassNode(); + new ClassReader( + SourceLoader.read( + observer.getInputStream( + observer.getJarEntry("datadog/compiler/utils/CompilerUtils.class")))) + .accept(utils, 0); + boolean readsSharedAnnotation = false; + for (MethodNode method : utils.methods) { + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof org.objectweb.asm.tree.LdcInsnNode + && String.valueOf(((org.objectweb.asm.tree.LdcInsnNode) instruction).cst) + .equals("Ldatadog/compiler/annotations/SourcePath;")) { + readsSharedAnnotation = true; + } + } + } + assertTrue(readsSharedAnnotation, "CompilerUtils must read the shared SourcePath annotation"); + } + } + + @Test + void nestedOnlyFrameworkDefaultsLoseToAnyExplicitSetting(@TempDir Path directory) + throws Exception { + Path file = directory.resolve("observer.properties"); + Files.write(file, "dd.trace.karate.enabled=true\n".getBytes("ISO-8859-1")); + Properties original = (Properties) System.getProperties().clone(); + try (SourceLoader loader = loader(true)) { + System.setProperty("tracing.observer.config.dd.integration.testng.enabled", "true"); + System.setProperty("tracing.observer.config.dd.trace.config", file.toString()); + Map env = new HashMap<>(); + env.put("TRACING_OBSERVER_CONFIG_DD_TRACE_JUNIT_4_ENABLED", "true"); + loader.loadClass(Environment.class.getName()).getField("values").set(null, env); + Class provider = loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object config = provider.getMethod("createDefault").invoke(null); + Method getBoolean = + provider.getMethod("getBoolean", String.class, boolean.class, String[].class); + Map enabled = new LinkedHashMap<>(); + for (String name : asList("junit-4", "testng", "karate", "scalatest", "junit-5")) { + // Same keys and order as InstrumenterConfig.isIntegrationEnabled. + enabled.put( + name, + getBoolean.invoke( + config, + "trace." + name + ".enabled", + true, + new String[] { + "trace.integration." + name + ".enabled", "integration." + name + ".enabled" + })); + } + Map expected = new LinkedHashMap<>(); + expected.put("junit-4", true); + expected.put("testng", true); + expected.put("karate", true); + expected.put("scalatest", false); + expected.put("junit-5", true); + assertEquals(expected, enabled); + assertEquals( + "datadog.*:com.datadog.*", + provider + .getMethod("getString", String.class) + .invoke(config, "civisibility.code.coverage.includes")); + } finally { + System.setProperties(original); + } + } + + @Test + void configReadsOwnedCredentialFilesWithStockSemantics(@TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path key = directory.resolve("owned-dummy.key"); + Files.write(key, "offline-credential-dummy\n".getBytes("UTF-8")); + try { + for (String spelling : + new String[] { + "dd.api-key-file", "dd.profiling.api-key-file", "dd.profiling.apikey.file" + }) { + System.setProperties((Properties) original.clone()); + System.setProperty("tracing.observer.config." + spelling, key.toString()); + try (SourceLoader loader = loader(true)) { + Class config = loader.loadClass(OBSERVER + "api.Config"); + Object instance = config.getMethod("get").invoke(null); + assertEquals( + "offline-credential-dummy", config.getMethod("getApiKey").invoke(instance), spelling); + } + } + } finally { + System.setProperties(original); + } + } + + @Test + void stockAgentArgumentsAndOtelFileInputsRemainPrivate(@TempDir Path directory) throws Exception { + Properties original = (Properties) System.getProperties().clone(); + Path otel = directory.resolve("owned-otel.properties"); + Files.write( + otel, + "otel.service.name=otel-file\notel.exporter.otlp.endpoint=https://example.invalid\n" + .getBytes("ISO-8859-1")); + try (SourceLoader loader = loader(true)) { + System.setProperty("tracing.observer.config.dd.trace.otel.enabled", "true"); + System.setProperty( + "tracing.observer.config.otel.javaagent.configuration-file", otel.toString()); + System.setProperty("dd.service", "subject"); + Class provider = loader.loadClass(OBSERVER + "bootstrap.config.provider.ConfigProvider"); + Object initial = provider.getMethod("createDefault").invoke(null); + assertEquals("true", configString(provider, initial, "trace.otel.enabled")); + assertEquals("otel-file", configString(provider, initial, "service.name")); + Class injector = + loader.loadClass(OBSERVER + "bootstrap.config.provider.AgentArgsInjector"); + injector + .getMethod("injectAgentArgsConfig", String.class) + .invoke(null, "dd.service=ordinary,dd.site=datad0g.com"); + Object config = provider.getMethod("createDefault").invoke(null); + assertEquals("ordinary", configString(provider, config, "service")); + assertEquals("datad0g.com", configString(provider, config, "site")); + assertEquals("subject", System.getProperty("dd.service")); + } finally { + System.setProperties(original); + } + } + + @Test + void generatedSettingsUseStockPlaceholderSubstitutionBeforeEncoding() throws Exception { + String name = OBSERVER + "instrumentation.gradle.TracerArgumentsProvider"; + ClassNode node = new ClassNode(); + try (java.util.jar.JarFile jar = + new java.util.jar.JarFile(System.getProperty("observer.test.artifact"))) { + new ClassReader( + SourceLoader.read( + jar.getInputStream( + jar.getJarEntry("inst/" + name.replace('.', '/') + ".classdata")))) + .accept(node, 0); + } + // Make only the abstract Gradle service accessor concrete so the real substitution method + // can be exercised without starting Gradle or either tracer. + node.access &= ~Opcodes.ACC_ABSTRACT; + MethodNode service = + node.methods.stream() + .filter(m -> m.name.equals("getCiVisibilityService")) + .findFirst() + .get(); + service.access &= ~Opcodes.ACC_ABSTRACT; + service.instructions.add(new org.objectweb.asm.tree.InsnNode(Opcodes.ACONST_NULL)); + service.instructions.add(new org.objectweb.asm.tree.InsnNode(Opcodes.ARETURN)); + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + node.accept(writer); + class FixtureLoader extends ClassLoader { + FixtureLoader() { + super(ObserverConfigSourcesTest.class.getClassLoader()); + } + + Class define() { + byte[] bytes = writer.toByteArray(); + return defineClass(name, bytes, 0, bytes.length); + } + } + Class provider = new FixtureLoader().define(); + Object instance = + provider + .getConstructor(String.class, Map.class) + .newInstance(":test", singletonMap("fixture", "resolved $ value")); + Method replace = provider.getDeclaredMethod("replaceProjectProperties", String.class); + replace.setAccessible(true); + try (SourceLoader loader = loader(true)) { + Class runtime = loader.loadClass("datadog.trace.observer.bootstrap.ObserverRuntime"); + @SuppressWarnings("unchecked") + Map marked = + (Map) + runtime + .getMethod("generatedProperties", Map.class) + .invoke(null, singletonMap("dd.tags", "tag:${fixture}")); + List arguments = new java.util.ArrayList<>(); + for (Map.Entry entry : marked.entrySet()) { + arguments.add( + (String) replace.invoke(instance, "-D" + entry.getKey() + "=" + entry.getValue())); + } + arguments.add((String) replace.invoke(instance, "-Dadditional=${fixture}")); + @SuppressWarnings("unchecked") + List result = + (List) + runtime.getMethod("childArguments", Iterable.class).invoke(null, arguments); + assertEquals("-Dadditional=resolved $ value", result.get(0)); + String decoded = + new String( + java.util.Base64.getUrlDecoder() + .decode(result.get(1).substring("-Dtracing.observer.child.v2=".length())), + "UTF-8"); + assertTrue(decoded.contains("tag:resolved $ value")); + assertTrue(!decoded.contains("${fixture}")); + } + MethodNode asArguments = + node.methods.stream().filter(m -> m.name.equals("asArguments")).findFirst().get(); + List calls = new java.util.ArrayList<>(); + for (AbstractInsnNode instruction : asArguments.instructions) { + if (instruction instanceof MethodInsnNode) { + calls.add(((MethodInsnNode) instruction).name); + } + } + assertEquals("collect", calls.get(calls.size() - 2)); + assertEquals("childArguments", calls.get(calls.size() - 1)); + } + + private static void initializeParent(Class runtime) throws Exception { + runtime.getMethod("initializePremain").invoke(null); + } + + private static Object configString(Class provider, Object config, String key) + throws Exception { + return provider.getMethod("getString", String.class).invoke(config, key); + } + + private Map evaluate(boolean observer, String factory, Path file) + throws Exception { + Properties original = (Properties) System.getProperties().clone(); + try (SourceLoader loader = loader(observer)) { + String prefix = observer ? "tracing.observer.config." : ""; + System.setProperty(prefix + "dd.service.name", "high-alias"); + System.setProperty(prefix + "dd.observer.probe.integer", "invalid-number"); + System.setProperty(prefix + "dd.observer.probe.boolean", "invalid-boolean"); + System.setProperty(prefix + "dd.tags", "prop:yes,shared:property"); + System.setProperty(prefix + "dd.api-key", "ignored-property-dummy"); + if (file != null) { + System.setProperty(prefix + "dd.trace.config", file.toString()); + } + Map env = new HashMap<>(); + String ep = observer ? "TRACING_OBSERVER_CONFIG_" : ""; + env.put(ep + "DD_SERVICE", "env-service"); + env.put(ep + "DD_OBSERVER_PROBE_INTEGER", "71"); + env.put(ep + "DD_OBSERVER_PROBE_BOOLEAN", "true"); + env.put(ep + "DD_TAGS", "env:yes,shared:environment"); + env.put(ep + "DD_API_KEY", "env-dummy"); + env.put(ep + "DD_APP_KEY", "app-dummy"); + loader.loadClass(Environment.class.getName()).getField("values").set(null, env); + String name = observer ? OBSERVER : STOCK; + Class provider = loader.loadClass(name + "bootstrap.config.provider.ConfigProvider"); + Properties provided = new Properties(); + provided.setProperty("service", "provided"); + Object instance = + factory.equals("withPropertiesOverride") + ? provider.getMethod(factory, Properties.class).invoke(null, provided) + : provider.getMethod(factory).invoke(null); + Map result = new LinkedHashMap<>(); + result.put( + "service", + provider + .getMethod("getString", String.class, String.class, String[].class) + .invoke(instance, "service", null, new String[] {"service.name"})); + result.put( + "numeric", + provider + .getMethod("getInteger", String.class, int.class, String[].class) + .invoke(instance, "observer.probe.integer", 9, new String[0])); + result.put( + "boolean", + provider + .getMethod("getBoolean", String.class, boolean.class, String[].class) + .invoke(instance, "observer.probe.boolean", true, new String[0])); + result.put( + "tags", + provider + .getMethod("getMergedMap", String.class, String[].class) + .invoke(instance, "tags", new String[0])); + Class system = + loader.loadClass(name + "bootstrap.config.provider.SystemPropertiesConfigSource"); + result.put( + "key", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(instance, "api-key", null, system, new String[0])); + result.put( + "applicationKey", + provider + .getMethod( + "getStringExcludingSource", + String.class, + String.class, + Class.class, + String[].class) + .invoke(instance, "application-key", null, system, new String[] {"app-key"})); + assertEquals("app-dummy", result.get("applicationKey")); + Field collect = provider.getDeclaredField("collectConfig"); + collect.setAccessible(true); + result.put("collect", collect.get(instance)); + return result; + } finally { + System.setProperties(original); + } + } + + private SourceLoader loader(boolean observer) throws Exception { + Path artifact = Paths.get(System.getProperty("observer.test.artifact")); + Path stock = Paths.get(System.getProperty("observer.test.stock")); + return new SourceLoader(observer, observer ? artifact : stock, artifact); + } + + private static class SourceLoader extends URLClassLoader { + private final boolean observer; + private final Path artifact; + + SourceLoader(boolean observer, Path jar, Path artifact) throws Exception { + super( + new URL[] { + jar.toUri().toURL(), + Environment.class.getProtectionDomain().getCodeSource().getLocation() + }, + null); + this.observer = observer; + this.artifact = artifact; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + String resource = name.replace('.', '/') + ".class"; + try { + byte[] bytes; + if (!observer && name.equals(RUNTIME.replace('/', '.'))) { + // The stock oracle shares the observer's stable-config seam, which calls the runtime. + try (java.util.jar.JarFile jar = new java.util.jar.JarFile(artifact.toFile())) { + bytes = read(jar.getInputStream(jar.getJarEntry(resource))); + } + } else if (!observer + && name.equals(STOCK + "bootstrap.config.provider.StableConfigSource")) { + // Only the host-file seam is disabled in the stock oracle. All factory/parsing code is + // stock. + try (java.util.jar.JarFile jar = new java.util.jar.JarFile(artifact.toFile())) { + bytes = + read( + jar.getInputStream( + jar.getJarEntry(resource.replace("datadog/", "datadog/trace/observer/")))); + } + ClassWriter writer = new ClassWriter(0); + new ClassReader(bytes) + .accept( + new ClassRemapper( + writer, + new Remapper(Opcodes.ASM9) { + @Override + public String map(String value) { + return value.equals(RUNTIME) + ? value + : value.replace("datadog/trace/observer/", "datadog/"); + } + }), + 0); + bytes = writer.toByteArray(); + } else { + URL url = findResource(resource); + if (url == null) { + throw new ClassNotFoundException(name); + } + try (InputStream in = url.openStream()) { + bytes = read(in); + } + } + ClassNode node = new ClassNode(); + new ClassReader(bytes).accept(node, 0); + if (!name.equals(Environment.class.getName())) { + for (MethodNode method : node.methods) { + for (AbstractInsnNode instruction : method.instructions) { + if (instruction instanceof MethodInsnNode) { + MethodInsnNode call = (MethodInsnNode) instruction; + if (call.owner.equals("java/lang/System") && call.name.equals("getenv")) { + call.owner = Environment.class.getName().replace('.', '/'); + } + } + } + } + } + ClassWriter writer = new ClassWriter(0); + node.accept(writer); + bytes = writer.toByteArray(); + return defineClass(name, bytes, 0, bytes.length); + } catch (Exception e) { + throw new ClassNotFoundException(name, e); + } + } + + private static byte[] read(InputStream in) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int n; + while ((n = in.read(buffer)) != -1) { + bytes.write(buffer, 0, n); + } + return bytes.toByteArray(); + } + } +} diff --git a/dd-java-agent/observer/src/test/resources/observer-relocated-roots.txt b/dd-java-agent/observer/src/test/resources/observer-relocated-roots.txt new file mode 100644 index 00000000000..87fcc689494 --- /dev/null +++ b/dd-java-agent/observer/src/test/resources/observer-relocated-roots.txt @@ -0,0 +1,4 @@ +com/datadog/debugger +datadog/slf4j +datadog/trace +net/bytebuddy diff --git a/docs/tracing_the_tracer.md b/docs/tracing_the_tracer.md new file mode 100644 index 00000000000..35e0ea47325 --- /dev/null +++ b/docs/tracing_the_tracer.md @@ -0,0 +1,231 @@ +# Tracing the tracer: experimental namespace-isolated agent + +The observer lets us trace this repository's own test runs with CI Visibility while +the tracer under test runs in the same JVMs. It is the stock Java agent relocated +into a private namespace. It keeps the full instrumenter catalog, capability +reporting and product configuration. Stock configuration decides what runs. + +Having the full catalog does not mean every integration or product can coexist with +the tracer under test. See [Limitations](#limitations). + +## Build + +The observer is a developer-only artifact. It is not part of `assemble`, publication +or ordinary test runs. + +```sh +./gradlew :dd-java-agent:observer:observerJar +# Result: dd-java-agent/observer/build/libs/dd-observer-agent.jar +``` + +The rewrite is reproducible: the same stock jar always gives the same observer jar. It +replaces the output atomically, so a JVM still running from the previous jar is not +affected. Copy the jar to a stable path before a long run anyway. + +Ordinary repository tests have no observer dependency or configuration. +`-PtraceTracer=true` only tracks the attached observer jar and configuration as test +inputs. It does not inject an agent or build the jar. + +## Injection + +Use ordinary premain. No role, hash or custom argument is required: + +```sh +java -javaagent:/absolute/path/dd-observer-agent.jar -jar application.jar +java -javaagent:/absolute/path/dd-observer-agent.jar=dd.service=observer -jar application.jar +``` + +Normal injection uses stock defaults and can start stock transports. Configure +destinations and products for your run. + +## Configuration + +Use full stock configuration names in the private namespace: + +| Input | Observer source | +| --- | --- | +| `-Dtracing.observer.config.dd.service=observer` | JVM property `dd.service` | +| `TRACING_OBSERVER_CONFIG_DD_SERVICE=observer` | Environment `DD_SERVICE` | +| `TRACING_OBSERVER_CONFIG_DD_API_KEY_FILE=/your/key-file` | Environment credential-file selector | +| `-Dtracing.observer.config.dd.trace.config=/your/observer.properties` | Stock properties-file selector | +| `TRACING_OBSERVER_CONFIG_OTEL_SERVICE_NAME=observer` | Stock OTEL environment source | + +Stock precedence, aliases, parsing, defaults, collection flags, caller overrides and +API-key property exclusion all apply. Credential files are reread, not snapshotted. +There is no allowlist for sites, endpoints, credentials, writers, products or OTEL. +Stock `Agent.configureCiVisibility` supplies the agent jar URI and CI defaults. +Ordinary properties files cannot activate early bootstrap product gates, as in stock. + +The observer has two defaults of its own: + +- It turns off the `junit-4`, `testng`, `karate`, `scalatest`, `weaver` and `cucumber` + integrations. In this repository those frameworks only run as fixtures inside JUnit + Platform tests, so observing them would report the fixtures as our tests. In Karate + 1.0 it also broke the fixtures. +- It limits per-test code coverage to `datadog.*:com.datadog.*` + (`civisibility.code.coverage.includes`). This repository has more top-level packages + than the stock root-package limit (50), so stock would infer no packages and cover + everything. That instrumented JDK classes and the test-only instrumentation classes + the tracer under test rewrites, and broke test workers. + +The defaults live in the local stable-config source, which has the lowest precedence. +Any explicit setting overrides them: a property, an environment variable or a +properties file, under any stock spelling such as `trace..enabled`. + +The observer does not import the subject's DD/OTEL properties, environment, +config-file selection or ambient propagation headers. JVM and CI platform facts are +shared. The stable-config sources never open host `/etc/datadog-agent` files: the +fleet source is empty and the local source only carries the defaults above. + +The implicit logger resource is `observer-simplelogger.properties`. Explicit logger +resources use ordinary logger keys. Logger and Byte Buddy keys are relocated. Their +values are not. + +These are source-isolation boundaries. They do not protect against same-process +reflection. + +## Gradle daemon injection + +Attach the observer to the Gradle daemon, not only the wrapper JVM. Gradle applies +`-D` entries from `org.gradle.jvmargs` after premain, so inherited namespaced +environment variables are the simplest way to configure it. + +Example for running against your own staging account: + +```sh +: "${STAGING_API_KEY_FILE:?Set STAGING_API_KEY_FILE to your staging key file}" +OBSERVER="$PWD/dd-java-agent/observer/build/libs/dd-observer-agent.jar" +TRACING_OBSERVER_CONFIG_DD_CIVISIBILITY_ENABLED=true \ +TRACING_OBSERVER_CONFIG_DD_TRACE_ENABLED=false \ +TRACING_OBSERVER_CONFIG_DD_CIVISIBILITY_AGENTLESS_ENABLED=true \ +TRACING_OBSERVER_CONFIG_DD_SITE=datad0g.com \ +TRACING_OBSERVER_CONFIG_DD_API_KEY_FILE="$STAGING_API_KEY_FILE" \ +TRACING_OBSERVER_CONFIG_DD_SERVICE=tracing-the-tracer \ +./gradlew --no-daemon --no-configuration-cache --no-scan \ + "-Dorg.gradle.jvmargs=-XX:MaxMetaspaceSize=1g -javaagent:$OBSERVER" \ + -PtraceTracer=true \ + :dd-java-agent:instrumentation-testing:test --tests AgentTestRunnerTest +``` + +The stock Gradle integration owns sessions, `Test` task modules and worker injection. +Workers receive the generated module settings privately. The subject's +`traceparent`, `tracestate`, baggage and Datadog headers cannot classify an observer +worker. Debug ports, extra JVM args, project-property substitution and JaCoCo +behavior stay stock. Only generated settings are marked, substituted by the stock +argument provider, and then encoded at the worker boundary. + +The worker carrier is a bounded, sorted list of NUL-separated UTF-8 entries in +Base64, passed as `tracing.observer.child.v2`. It is not encryption. Inherited namespaced environment is never serialized +into child arguments. Sensitive generated keys, identified by stock metadata, are not +promoted from environment or file sources into JVM properties. Explicit caller JVM +properties keep their role, so do not put secrets on command lines. The optional +`tracing.observer.log.directory` property or `TRACING_OBSERVER_LOG_DIRECTORY` +environment variable writes separate observer log files. + +## How isolation works + +The rewriter runs offline on the stock jar. Every stock method it depends on is listed +in one table in `ObserverAgentRewriter`, with how often its patch must apply. Renamed +Gradle service and extension names are counted the same way. The rewrite fails if any +count changes. `:dd-java-agent:check` builds the observer and runs its tests, so a +stock change that breaks a seam fails CI instead of the next observed run. + +- Classes, resources, service files and indexes move under `datadog.trace.observer`. + This covers tracer globals, shaded dependencies, Byte Buddy and Gradle + services/resources. The jar index is built and checked with the stock index + generator and reader. +- The field-backed context protocol is renamed: `__datadogObserverContext$` fields and + `$get$__datadogObserverContext$` / `$put$__datadogObserverContext$` methods, + including dynamic names. Field injection stays enabled. +- `System` property, environment and `Boolean.getBoolean` calls go through a private + source view. +- String constants only move when they name something under a package root that + exists in the stock jar: `datadog.`, `com.datadog.` or `net.bytebuddy`. + Everything else keeps its stock spelling, such as socket paths, metric and OTLP + names, JNDI names, bare `datadog.` prefixes and request attribute keys. The runtime + uses the same rule for logger and Byte Buddy property keys. +- `datadog.compiler` is never relocated. The javac plugin writes its annotation types + into compiled classes, so both tracers must read the same types. +- The JUnit 5 and Spock advice only open observer spans for the synchronous outer + Gradle engine launch. Nested launchers are ignored. Other test frameworks are off by + default, see [Configuration](#configuration). If a worker runs an engine but never + sees Gradle's launch, it prints a warning at exit, because no tests were reported. + +Some patches are permanent, because they are what makes the copy private: relocation, +the context protocol rename, the `System` redirection and the worker carrier. Others +could become small stock options and be removed from the rewriter: + +- Ignoring nested JUnit Platform launchers. +- Not reading host stable-config files. +- A configurable Gradle service and extension name. +- A numeric IPC host for workers (`getHostAddress`), which looks useful in stock too. + +The modifiable-config convention recognizes the attached observer without path or +hash metadata. + +Four stock fixes are part of this work: + +- `TypeFactory` resolves the current transform target from the supplied bytes when the + type cache already has an entry for it. That entry can predate changes from earlier + transformers. Without this, the subject transformer removed an interface the + observer had injected. This was reproduced on Mockito's `DetachedThreadLocal` and + aborted a whole retransformation batch. On a cache miss, the target is parsed and + shared as before. The extra parse only happens on a cache hit for the target, for + example after a supertype lookup or on retransformation. It has not been + benchmarked. +- `UnknownCIInfo` accepts `.git` as a file when looking for the repository root, as in + linked worktrees and submodules. Before, local runs from a worktree had no + repository, branch or commit tags. +- `LineCoverageStore.Factory` loads the per-test coverage recording classes up front. + JaCoCo probes also land in the tracer under test's `defineClass` hook. Loading a + class lazily while recording ran that hook, which recorded again until a + `StackOverflowError`. The JVM then printed + `java.lang.instrument ASSERTION FAILED ... transform method call failed` and loaded + the class untransformed. A test fails if recording loads a class again. +- An empty code coverage include or exclude entry now matches nothing. A parent that + infers no root packages propagates an empty include list to its workers. Each worker + parsed it as one empty prefix, which matched every class, including the JDK's + generated reflection accessors. Without JaCoCo, file-level coverage then crashed the + worker before any test ran. + +## Tests + +The rewriter and runtime live in `:dd-java-agent:observer`. Its tests build the stock +and observer jars first, then compare stock and relocated configuration on the real +artifacts and check that rewriting in-process gives the same jar: + +```sh +./gradlew :dd-java-agent:observer:test +``` + +Only the attach detection used by the Gradle conventions stays in `buildSrc`. + +The `TypeFactory`, `UnknownCIInfo` and `LineCoverageStore` changes are covered by the +regular module tests. End-to-end runs against a local mock intake were done by hand +and are not part of the repository. + +## Limitations + +- Target: the repository's Gradle 9.8 daemon with the stock Gradle 8.10+ hook, the + JUnit Jupiter and Spock outer lifecycle, and the subject instrumentation harness. +- Stock capabilities are reported. Optional capabilities these tests do not select are + not proven compatible. No new product features were added. +- Per-test code coverage and coverage report upload were checked on a sample of + modules: `junit-5.3`, `junit-4.10`, `instrumentation-testing`, `java-concurrent-1.8`, + `okhttp-3.0` and `dd-trace-core`. They work on the default test JVM, where coverage + uses JaCoCo, and with `-PtestJvm`, where the build turns JaCoCo off and coverage is + file-level. Other modules are not proven. +- Failed Test Replay, test skipping, Auto Test Retries, Early Flake Detection and Test + Management have not been verified with the observer. +- Every Gradle build opens its own session, including `buildSrc`, `build-logic` and + Kotlin DSL accessor builds. Those sessions run no tests but still fetch settings. +- Unverified: arbitrary asynchronous engines, other runners, Maven, launcher injection + and generic child processes. `JavaExec`, TestKit and smoke-test subprocesses are not + injected automatically. +- Unsupported: configuration cache, daemon reuse, dynamic attach, AOT/CDS and security + manager environments. +- Request attribute keys such as `datadog.span.dispatch` keep their stock spelling. If + both tracers ran the same server tracing integrations in one JVM, they would share + those attributes. The CI setup above runs the observer with tracing off. +- The outer-engine check matches Gradle's internal JUnit Platform test processor. If a + Gradle upgrade renames it, workers report no tests and print a warning. diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 2db203dcbd4..3acdd51f7ac 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -3546,7 +3546,9 @@ private RumInjectorConfig parseRumConfig(ConfigProvider configProvider) { * my/package/,my/other/package/}) */ public static String[] convertJacocoExclusionFormatToPackagePrefixes(List packages) { + // An empty entry, such as the propagated value of an empty include list, matches nothing. return packages.stream() + .filter(s -> !s.isEmpty()) .map(s -> (s.endsWith("*") ? s.substring(0, s.length() - 1) : s).replace('.', '/')) .toArray(String[]::new); } diff --git a/internal-api/src/test/java/datadog/trace/api/ConfigCodeCoveragePackagesTest.java b/internal-api/src/test/java/datadog/trace/api/ConfigCodeCoveragePackagesTest.java new file mode 100644 index 00000000000..3b46981a49b --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/ConfigCodeCoveragePackagesTest.java @@ -0,0 +1,30 @@ +package datadog.trace.api; + +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_INCLUDES; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(WithConfigExtension.class) +class ConfigCodeCoveragePackagesTest { + + @Test + void anEmptyIncludeListMatchesNoPackage() { + // A parent with no root packages propagates an empty value to its workers. + WithConfigExtension.injectSysConfig(CIVISIBILITY_CODE_COVERAGE_INCLUDES, ""); + + assertArrayEquals(new String[0], Config.get().getCiVisibilityCodeCoverageIncludedPackages()); + } + + @Test + void includesBecomePackagePrefixes() { + WithConfigExtension.injectSysConfig( + CIVISIBILITY_CODE_COVERAGE_INCLUDES, "datadog.*:com.example:*"); + + assertArrayEquals( + new String[] {"datadog/", "com/example", ""}, + Config.get().getCiVisibilityCodeCoverageIncludedPackages()); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ad3c1a2012c..8ea658c0763 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -171,6 +171,7 @@ include( // misc include( + ":dd-java-agent:observer", ":dd-java-agent:testing", ":utils:config-utils", ":utils:container-utils",