Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 215 additions & 3 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ include:
- local: ".gitlab/java-benchmark-configs.yml"

stages:
- ddprof-native
- ddprof-assemble
- build
- publish
- java-spring-petclinic-parallel
Expand Down Expand Up @@ -66,6 +68,16 @@ variables:
NON_DEFAULT_JVMS:
description: "Enable tests on JVMs that are not the default"
value: "false"
# Temporary cross-repository pin for the TaskBlock rollout. java-profiler is fetched from its
# public GitHub repository because GitLab rejects cross-project CI_JOB_TOKEN authentication.
# The branch check makes movement explicit, while the commit SHA makes the build input immutable.
DDPROF_BRANCH: "paul.fournillon/taskblock-native-io"
DDPROF_COMMIT_SHA: "18130c4978fe2b737d86c05dee0ff742885c1b5a"
DDPROF_ARTIFACT_VERSION: "1.47.0-taskblock-native-io-18130c497"
DDPROF_BUILD_IMAGE_X64: "registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-2.17-base@sha256:03fb65c5c7fb2866397ceaf18ccb5a0b91bb377a9f89c94264427b62d48cfffd"
DDPROF_BUILD_IMAGE_X64_MUSL: "registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-musl-base@sha256:0efe0c5fbd04e5647a53235d4c4942a2dd305b3d76bf5afdc052d64a7c0b9505"
DDPROF_BUILD_IMAGE_ARM64: "registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-base@sha256:15670752fa8e67c81c1b7fd4d93ef34c668649f117584ab29327d8d7733d64d6"
DDPROF_BUILD_IMAGE_ARM64_MUSL: "registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-musl-base@sha256:1c8a96ded178bfa130077166db049cbea422c94c96a8c5172d66f8c3581d57cf"
RUN_FLAKY_TESTS:
description: "Enable flaky tests"
value: "false"
Expand Down Expand Up @@ -247,6 +259,12 @@ default:
EOF
- mkdir -p .gradle
- export GRADLE_USER_HOME=$(pwd)/.gradle
- |
if [[ -n "${DDPROF_JAR:-}" ]]; then
test -n "${DDPROF_SHA256:-}"
test -f "${DDPROF_JAR}"
echo "${DDPROF_SHA256} ${DDPROF_JAR}" | sha256sum -c -
fi
# Apache Maven Wrapper supports MVNW_REPOURL for repository-manager downloads:
# https://maven.apache.org/tools/wrapper/#Using_a_Maven_Repository_Manager
- export MVNW_REPOURL=${MAVEN_REPOSITORY_PROXY%/}
Expand All @@ -266,6 +284,10 @@ default:
fi
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
- export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
- |
if [[ -n "${DDPROF_JAR:-}" ]]; then
export GRADLE_ARGS="${GRADLE_ARGS} -Pddprof.jar=${CI_PROJECT_DIR}/${DDPROF_JAR}"
fi
- *normalize_node_index
# GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001),
# and Gradle does `chmod 700 .gradle` on startup which requires user ownership.
Expand Down Expand Up @@ -328,8 +350,158 @@ dd-octo-sts-pre-release-check:
max: 2
when: always

.checkout_pinned_ddprof: &checkout_pinned_ddprof
- |
set -euo pipefail
export DDPROF_REPO_DIR="${CI_PROJECT_DIR}/java-profiler"
: "${DDPROF_BRANCH:?DDPROF_BRANCH must be set}"
: "${DDPROF_COMMIT_SHA:?DDPROF_COMMIT_SHA must be set}"
: "${DDPROF_ARTIFACT_VERSION:?DDPROF_ARTIFACT_VERSION must be set}"

git init "${DDPROF_REPO_DIR}"
(
cd "${DDPROF_REPO_DIR}"
git remote add origin "https://github.com/DataDog/java-profiler.git"
git fetch --depth=1 origin "${DDPROF_BRANCH}"

resolved_commit=$(git rev-parse FETCH_HEAD)
if [[ "${resolved_commit}" != "${DDPROF_COMMIT_SHA}" ]]; then
echo "Pinned java-profiler branch moved: expected ${DDPROF_COMMIT_SHA}, got ${resolved_commit}" >&2
exit 1
fi
git checkout --detach "${resolved_commit}"
)

if [[ -z "${JAVA_HOME:-}" || ! -x "${JAVA_HOME}/bin/java" ]]; then
if [[ -x "${HOME}/.sdkman/candidates/java/current/bin/java" ]]; then
export JAVA_HOME="${HOME}/.sdkman/candidates/java/current"
else
echo "JAVA_HOME does not point to a usable JDK" >&2
exit 1
fi
fi
export ORG_GRADLE_PROJECT_mavenRepositoryProxy="${MAVEN_REPOSITORY_PROXY}"

.build_pinned_ddprof_native:
stage: ddprof-native
variables:
KUBERNETES_CPU_REQUEST: 10
KUBERNETES_MEMORY_REQUEST: 20Gi
KUBERNETES_MEMORY_LIMIT: 20Gi
script:
- *checkout_pinned_ddprof
- |
set -euo pipefail
: "${TARGET:?TARGET must be set for native builds}"
(
cd "${DDPROF_REPO_DIR}"
./gradlew \
-Pddprof_version="${DDPROF_ARTIFACT_VERSION}" \
-Pskip-cpp-tests \
:ddprof-lib:assembleReleaseJar \
--no-build-cache \
--stacktrace \
--info \
--no-watch-fs \
--no-daemon
)

native_lib="${DDPROF_REPO_DIR}/ddprof-lib/build/native/release/META-INF/native-libs/${TARGET}/libjavaProfiler.so"
test -f "${native_lib}"
mkdir -p "${CI_PROJECT_DIR}/ddprof-native/${TARGET}"
cp "${native_lib}" "${CI_PROJECT_DIR}/ddprof-native/${TARGET}/libjavaProfiler.so"
artifacts:
paths:
- ddprof-native/$TARGET/libjavaProfiler.so
expire_in: 1 day

build_ddprof_x64:
extends: .build_pinned_ddprof_native
image: $DDPROF_BUILD_IMAGE_X64
tags: [ "arch:amd64" ]
variables:
TARGET: linux-x64

build_ddprof_x64_musl:
extends: .build_pinned_ddprof_native
image: $DDPROF_BUILD_IMAGE_X64_MUSL
tags: [ "arch:amd64" ]
variables:
TARGET: linux-x64-musl

build_ddprof_arm64:
extends: .build_pinned_ddprof_native
image: $DDPROF_BUILD_IMAGE_ARM64
tags: [ "arch:arm64" ]
variables:
TARGET: linux-arm64

build_ddprof_arm64_musl:
extends: .build_pinned_ddprof_native
image: $DDPROF_BUILD_IMAGE_ARM64_MUSL
tags: [ "arch:arm64" ]
timeout: 3h
variables:
TARGET: linux-arm64-musl

assemble_ddprof:
stage: ddprof-assemble
image: $DDPROF_BUILD_IMAGE_X64
tags: [ "arch:amd64" ]
needs:
- job: build_ddprof_x64
artifacts: true
- job: build_ddprof_x64_musl
artifacts: true
- job: build_ddprof_arm64
artifacts: true
- job: build_ddprof_arm64_musl
artifacts: true
script:
- *checkout_pinned_ddprof
- |
set -euo pipefail
for target in linux-x64 linux-x64-musl linux-arm64 linux-arm64-musl; do
test -f "${CI_PROJECT_DIR}/ddprof-native/${target}/libjavaProfiler.so"
done

(
cd "${DDPROF_REPO_DIR}"
./gradlew \
-Pskip-native \
-Pskip-tests \
-Pddprof_version="${DDPROF_ARTIFACT_VERSION}" \
-Pwith-libs="${CI_PROJECT_DIR}/ddprof-native" \
:ddprof-lib:assembleReleaseJar \
--max-workers=1 \
--no-build-cache \
--stacktrace \
--info \
--no-watch-fs \
--no-daemon
)

assembled_jar="${DDPROF_REPO_DIR}/ddprof-lib/build/libs/ddprof-${DDPROF_ARTIFACT_VERSION}.jar"
test -f "${assembled_jar}"
mkdir -p "${CI_PROJECT_DIR}/ddprof-input"
cp "${assembled_jar}" "${CI_PROJECT_DIR}/ddprof-input/ddprof.jar"

ddprof_sha256=$(sha256sum "${CI_PROJECT_DIR}/ddprof-input/ddprof.jar" | cut -d' ' -f1)
{
echo "DDPROF_JAR=ddprof-input/ddprof.jar"
echo "DDPROF_SHA256=${ddprof_sha256}"
} > "${CI_PROJECT_DIR}/ddprof.env"
artifacts:
paths:
- ddprof-input/ddprof.jar
reports:
dotenv: ddprof.env
expire_in: 1 day

build:
needs:
- job: assemble_ddprof
artifacts: true
- job: maven-central-pre-release-check
optional: true
- job: dd-octo-sts-pre-release-check
Expand All @@ -340,25 +512,40 @@ build:
CACHE_TYPE: "lib"
DEPENDENCY_CACHE_POLICY: pull
script:
- test -n "$DDPROF_JAR" && test -n "$DDPROF_SHA256" && test -f "$DDPROF_JAR"
- if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi
- ./gradlew --version
- ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests -x spotlessCheck $GRADLE_ARGS
- echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
- AGENT_JARS=$(find workspace/dd-java-agent/build/libs -maxdepth 1 -type f -name 'dd-java-agent-*.jar' ! -name '*-javadoc.jar' ! -name '*-sources.jar' -print)
- test "$(printf '%s\n' "$AGENT_JARS" | grep -c .)" -eq 1
- AGENT_JAR="$AGENT_JARS"
- AGENT_SHA256=$(sha256sum "$AGENT_JAR" | cut -d' ' -f1)
- echo "$AGENT_SHA256 $(basename "$AGENT_JAR")" > "$AGENT_JAR.sha256"
- echo UPSTREAM_TRACER_VERSION=$(java -jar "$AGENT_JAR") >> upstream.env
- echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
- echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
- echo "SMOKETEST_AGENT_JAR=$AGENT_JAR" >> build.env
- echo "SMOKETEST_AGENT_SHA256=$AGENT_SHA256" >> build.env
- echo "DDPROF_JAR=$DDPROF_JAR" >> build.env
- echo "DDPROF_SHA256=$DDPROF_SHA256" >> build.env
artifacts:
when: always
paths:
- 'workspace/dd-java-agent/build/libs/*.jar'
- 'workspace/dd-java-agent/build/libs/*.sha256'
- 'workspace/dd-trace-api/build/libs/*.jar'
- 'workspace/dd-trace-ot/build/libs/*.jar'
- 'ddprof-input/ddprof.jar'
- 'upstream.env'
- '.gradle/daemon/*/*.out.log'
reports:
dotenv: build.env

build_tests:
extends: .gradle_build
needs:
- job: assemble_ddprof
artifacts: true
variables:
<<: *tier_l_variables
BUILD_CACHE_POLICY: push
Expand All @@ -377,6 +564,7 @@ build_tests:
CACHE_TYPE: "smoke"
MAVEN_OPTS: "-Xms256M -Xmx1024M"
script:
- test -n "$DDPROF_JAR" && test -n "$DDPROF_SHA256" && test -f "$DDPROF_JAR"
- *gitlab_base_ref_params
- ./gradlew --version
- ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS
Expand Down Expand Up @@ -602,6 +790,8 @@ muzzle:
# Keep matrix vars exact and in build_tests declaration order:
# https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
needs: &needs_build_tests_inst
- job: build
artifacts: true
- job: build_tests
parallel:
matrix:
Expand Down Expand Up @@ -687,7 +877,11 @@ muzzle-dep-report:
extends: .gradle_build
image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
needs: [ build_tests ]
needs:
- job: build
artifacts: true
- job: build_tests
artifacts: false
stage: tests
variables:
<<: *tier_m_variables
Expand Down Expand Up @@ -722,6 +916,8 @@ muzzle-dep-report:
- *prepare_test_env
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
- ./gradlew --version
- test -n "$SMOKETEST_AGENT_JAR" && test -n "$SMOKETEST_AGENT_SHA256" && test -f "$SMOKETEST_AGENT_JAR"
- echo "$SMOKETEST_AGENT_SHA256 $SMOKETEST_AGENT_JAR" | sha256sum -c -
- ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
after_script:
- *restore_pretest_env
Expand Down Expand Up @@ -769,7 +965,9 @@ muzzle-dep-report:
image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
tags: [ "docker-in-docker:arm64" ]
stage: tests-arm64
needs: []
needs:
- job: build
artifacts: true
variables:
<<: *tier_m_variables
DEFAULT_TEST_JVMS: /^(8|11|17|21|25|27|tip)$/ # Java 27 TODO: remove 27 after GA (tip will move to 27)
Expand Down Expand Up @@ -847,7 +1045,13 @@ muzzle-dep-report:
# Disable CDS in forked JVMs to avoid SIGSEGVs on Linux arm64.
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xshare:off -Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
- export GRADLE_ARGS="--build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
- |
if [[ -n "${DDPROF_JAR:-}" ]]; then
export GRADLE_ARGS="${GRADLE_ARGS} -Pddprof.jar=${CI_PROJECT_DIR}/${DDPROF_JAR}"
fi
- ./gradlew --version
- test -n "$SMOKETEST_AGENT_JAR" && test -n "$SMOKETEST_AGENT_SHA256" && test -f "$SMOKETEST_AGENT_JAR"
- echo "$SMOKETEST_AGENT_SHA256 $SMOKETEST_AGENT_JAR" | sha256sum -c -
script:
- *gitlab_base_ref_params
- ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue
Expand Down Expand Up @@ -946,6 +1150,8 @@ test_base:
# Keep matrix vars exact and in build_tests declaration order:
# https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
needs:
- job: build
artifacts: true
- job: build_tests
parallel:
matrix:
Expand Down Expand Up @@ -997,6 +1203,8 @@ test_inst_latest:
# Keep matrix vars exact and in build_tests declaration order:
# https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
needs:
- job: build
artifacts: true
- job: build_tests
parallel:
matrix:
Expand Down Expand Up @@ -1065,6 +1273,8 @@ test_profiling:
# Keep matrix vars exact and in build_tests declaration order:
# https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
needs:
- job: build
artifacts: true
- job: build_tests
parallel:
matrix:
Expand Down Expand Up @@ -1119,6 +1329,8 @@ test_smoke:
# Keep matrix vars exact and in build_tests declaration order:
# https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
needs: &needs_build_tests_smoke
- job: build
artifacts: true
- job: build_tests
parallel:
matrix:
Expand Down
Loading