diff --git a/dd-java-agent/agent-bootstrap/build.gradle b/dd-java-agent/agent-bootstrap/build.gradle index c554039afa5..398c9ad1351 100644 --- a/dd-java-agent/agent-bootstrap/build.gradle +++ b/dd-java-agent/agent-bootstrap/build.gradle @@ -72,4 +72,9 @@ tasks.withType(Test).configureEach { JavaVersion.VERSION_16, ['--add-opens', 'java.base/java.net=ALL-UNNAMED'] // for HostNameResolverForkedTest ) + conditionalJvmArgs( + it, + JavaVersion.VERSION_11, + ['--add-opens', 'jdk.jfr/jdk.jfr.events=ALL-UNNAMED'] // for JfrEventHolderInitForkedTest + ) } diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 8a2de041314..555e027d498 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -1,5 +1,6 @@ package datadog.trace.bootstrap; +import static datadog.environment.JavaVirtualMachine.isHotspot; import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; import static datadog.environment.JavaVirtualMachine.isOracleJDK8; import static datadog.trace.api.Config.isExplicitlyDisabled; @@ -908,6 +909,8 @@ private static synchronized void startJmx() { } } if (profilingEnabled) { + // Both of these register JFR events through registerJfrEvents(), which force-initializes the + // JDK's JFR event-holder class first (see initializeJfrEventHolderClass) to avoid a deadlock. registerDeadlockDetectionEvent(); registerSmapEntryEvent(); if (PROFILER_INIT_AFTER_JMX != null) { @@ -937,40 +940,143 @@ When getJmxStartDelay() is set to 0 we will attempt to initialize the JMX subsys private static synchronized void registerDeadlockDetectionEvent() { log.debug("Initializing JMX thread deadlock detector"); + registerJfrEvents( + "com.datadog.profiling.controller.openjdk.events.DeadlockEventFactory", + "JMX thread deadlock detection"); + } + + private static void initializeJfrEventHolderClass() { + initializeJfrEventHolderClass(AGENT_CLASSLOADER); + } + + /** + * Force-initializes the JDK's JFR event-holder class early to avoid an ABBA deadlock between + * {@code jdk.jfr.internal.Utils}'s monitor and the holder class's initialization lock. See JDK-8371889 and the SCP-1278 thread + * dump. + * + *

The holder's {@code } looks up the JDK's built-in event handlers through {@code + * jdk.jfr.internal.Utils} (taking its monitor); conversely, JFR event registration initializes + * the holder while holding that same monitor. The deadlock forms when one thread holds the {@code + * Utils} monitor and wants the holder's class-init lock, while another thread holds the + * class-init lock (running {@code }) and wants the {@code Utils} monitor. In SCP-1278 + * this was the profiler's smap-event registration (holding {@code Utils}) against a concurrent + * JMXFetch ByteBuddy transform that had triggered the holder {@code }. Running the {@code + * } here, on this thread, before we register our own JFR events, means the holder is + * already initialized by then, so the cycle cannot form. + * + *

Ordering matters twice over: + * + *

+ * + *

The holder class was renamed across JDK versions, so it is selected by version (see {@link + * #jfrEventHolderClassName()}): {@code jdk.jfr.events.Handlers} on JDK 15-18, {@code + * jdk.jfr.events.EventConfigurations} on JDK 19-22. Earlier JDKs (including 11 LTS) predate the + * holder and JDK 23+ removed the eager-init pattern; on those this method does nothing. On + * patched JDKs (the JDK-8371889 fix was backported to 21.0.11) the JDK already initializes the + * holder safely during {@code FlightRecorder} startup, so forcing it here is a harmless no-op. + * + * @param loader class loader used to resolve the JFR classes (package-private for testing; see + * JfrEventHolderInitForkedTest) + */ + static void initializeJfrEventHolderClass(final ClassLoader loader) { + final String holderClassName = jfrEventHolderClassName(); + if (holderClassName == null) { + return; // no eager-init holder on this JDK, so there is no deadlock to prevent + } try { - final Class deadlockFactoryClass = - AGENT_CLASSLOADER.loadClass( - "com.datadog.profiling.controller.openjdk.events.DeadlockEventFactory"); - final Method registerMethod = deadlockFactoryClass.getMethod("registerEvents"); - registerMethod.invoke(null); - } catch (final NoClassDefFoundError - | ClassNotFoundException - | UnsupportedClassVersionError ignored) { - log.debug("JMX deadlock detection not supported"); - } catch (final Throwable ex) { - log.error("Unable to initialize JMX thread deadlock detector", ex); + // Register the JDK's built-in JFR events first, so the holder's below sees non-null + // handlers instead of caching null. + Class.forName("jdk.jfr.FlightRecorder", true, loader) + .getMethod("getFlightRecorder") + .invoke(null); + // Force the holder's . + Class.forName(holderClassName, true, loader); + } catch (final Throwable ignored) { + // JFR unavailable/disabled or initialization failed: nothing (left) to do. We catch Throwable + // rather than Exception because forcing initialization can surface Errors such as + // ExceptionInInitializerError, NoClassDefFoundError or LinkageError. + } + } + + /** + * Returns the fully-qualified name of the JDK's JFR event-holder class for the running JVM, or + * {@code null} if this JDK has no such class. The class is selected by JDK version rather than by + * probing with {@link ClassNotFoundException} so the mapping is explicit: + * + *

+ * + *

Also returns {@code null} on non-HotSpot VMs: JDK-8371889 is a HotSpot JFR bug, and other + * VMs (e.g. Eclipse OpenJ9 / IBM Semeru) ship a different JFR implementation where this holder / + * {@code Utils} mechanism does not apply. + */ + static String jfrEventHolderClassName() { + if (!isHotspot()) { + return null; } + if (isJavaVersionAtLeast(15) && !isJavaVersionAtLeast(19)) { + return "jdk.jfr.events.Handlers"; + } + if (isJavaVersionAtLeast(19) && !isJavaVersionAtLeast(23)) { + return "jdk.jfr.events.EventConfigurations"; + } + return null; } private static synchronized void registerSmapEntryEvent() { log.debug("Initializing smap entry scraping"); + registerJfrEvents( + "com.datadog.profiling.controller.openjdk.events.SmapEntryFactory", "Smap entry scraping"); + } - // Load JFR Handlers class early, if present (it has been moved and renamed in JDK23+). - // This prevents a deadlock. See https://bugs.openjdk.org/browse/JDK-8371889. - try { - AGENT_CLASSLOADER.loadClass("jdk.jfr.events.Handlers"); - } catch (Exception e) { - // Ignore when the class is not found or anything else goes wrong. - } - + /** + * Registers a profiling JFR event factory's events, after force-initializing the JDK's JFR + * event-holder class (see {@link #initializeJfrEventHolderClass(ClassLoader)}). + * + *

All JFR event registration during agent startup must go through this + * method. Registering a JFR event triggers the holder class's initialization while + * holding the {@code jdk.jfr.internal.Utils} monitor; unless the holder has already been fully + * initialized, that can deadlock (JDK-8371889). Routing every registration through here + * guarantees the holder is initialized first, so future event registrations cannot reintroduce + * the deadlock by running before it. + * + * @param factoryClassName fully-qualified name of the event factory with a static {@code + * registerEvents()} method + * @param description human-readable name used in log messages + */ + private static void registerJfrEvents(final String factoryClassName, final String description) { + // Enforce the ordering invariant: the holder must be initialized before any JFR event is + // registered. Idempotent and cheap once done, so it is safe to call before every registration. + initializeJfrEventHolderClass(); try { - final Class smapFactoryClass = - AGENT_CLASSLOADER.loadClass( - "com.datadog.profiling.controller.openjdk.events.SmapEntryFactory"); - final Method registerMethod = smapFactoryClass.getMethod("registerEvents"); - registerMethod.invoke(null); - } catch (final Exception ignored) { - log.debug("Smap entry scraping not supported"); + final Class factoryClass = AGENT_CLASSLOADER.loadClass(factoryClassName); + factoryClass.getMethod("registerEvents").invoke(null); + } catch (final NoClassDefFoundError + | ClassNotFoundException + | UnsupportedClassVersionError ignored) { + log.debug("{} not supported", description); + } catch (final Throwable ex) { + log.error("Unable to initialize {}", description, ex); } } diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/JfrEventHolderInitForkedTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/JfrEventHolderInitForkedTest.java new file mode 100644 index 00000000000..ebe9e8443da --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/JfrEventHolderInitForkedTest.java @@ -0,0 +1,81 @@ +package datadog.trace.bootstrap; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Regression test for the JFR startup-deadlock workaround in {@link Agent}. + * + *

{@link Agent#initializeJfrEventHolderClass(ClassLoader)} force-initializes the JDK's JFR + * event-holder class ({@code jdk.jfr.events.Handlers} on JDK 15-18, {@code + * jdk.jfr.events.EventConfigurations} on JDK 19-22) to avoid an ABBA deadlock. Its static-final + * handler fields are populated from {@code jdk.jfr.internal.Utils} at {@code } time and + * stay {@code null} forever if the class is initialized before the JDK's built-in events are + * registered (i.e. before {@code FlightRecorder} initialization). A null-poisoned holder silently + * disables the built-in socket/file/exception JFR events. + * + *

This test runs the production initialization path and asserts none of the handler fields were + * poisoned. It is a {@code ForkedTest} because JFR/class initialization happens once per JVM, so + * the ordering under test is only exercised in a fresh JVM that has not yet touched JFR. + * + *

Scope and limitations: + * + *

+ * + *

Requires {@code --add-opens jdk.jfr/jdk.jfr.events=ALL-UNNAMED} (configured in build.gradle) + * to read the holder's fields reflectively. Automatically skipped where there is no holder to + * initialize: JDK 14 and earlier, JDK 23+ (eager-init pattern removed), and non-HotSpot VMs such as + * Eclipse OpenJ9 / IBM Semeru (different JFR implementation). + */ +public class JfrEventHolderInitForkedTest { + + @Test + public void productionInitOrderingDoesNotPoisonHandlers() throws Exception { + final ClassLoader loader = getClass().getClassLoader(); + + // The holder class (if any) is selected by JDK version; skip when this JDK has none (JDK 8, + // 23+). + final String holderName = Agent.jfrEventHolderClassName(); + assumeTrue(holderName != null, "No JFR event-holder class on this JDK; nothing to test"); + + // Exercise the exact production path: FlightRecorder init first, then holder . + Agent.initializeJfrEventHolderClass(loader); + + // The holder is now initialized; read its static handler fields and check none are null. + final Class holder = Class.forName(holderName, false, loader); + final List nullFields = new ArrayList<>(); + int handlerFields = 0; + for (final Field field : holder.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers()) || field.getType().isPrimitive()) { + continue; + } + handlerFields++; + field.setAccessible(true); + if (field.get(null) == null) { + nullFields.add(field.getName()); + } + } + + assertTrue(handlerFields > 0, "expected " + holderName + " to declare handler fields"); + assertTrue( + nullFields.isEmpty(), + "JFR handler fields were poisoned to null (holder initialized before FlightRecorder): " + + nullFields); + } +}