Skip to content

Commit bfff55a

Browse files
lqiu96lqiu96
committed
perf(ci): replace mvn help:evaluate and redundant subprocesses with native bash, sed, and git optimizations
This PR is part 2 of 2 in a stacked series: 1. #14217: fix: avoid running split ITs unexpectedly on unrelated changes (#12029) 2. This PR: perf(ci): replace mvn help:evaluate with native bash and sed extraction In Kokoro CI and automation scripts, heavy subprocesses and JVM invocations were used: - In generate_modified_modules_list (.kokoro/common.sh), evaluating project.modules launched a full JVM and evaluated the monorepo POMs, taking 20–30+ seconds on every single CI run. - In downstream-build.sh (.kokoro/presubmit/downstream-build.sh) and showcase-native.sh (sdk-platform-java/.kokoro/presubmit/showcase-native.sh), evaluating gapic-showcase.version launched Maven JVM processes. - In check_status.sh (java-cloud-bom/tests/release-repository-readiness/check_status.sh), mvn help:evaluate was called for shared dependencies and generator versions, and table formatting piped echo into awk. - In update_javadoc.sh (google-auth-library-java/scripts/update_javadoc.sh), maven-help-plugin:evaluate was called to obtain the project version. - In update_googleapis_commit.sh (sdk-platform-java/.github/scripts/update_googleapis_commit.sh), a full git clone of googleapis was performed just to inspect HEAD commitish. - In downstream-protobuf-binary-compatibility.sh (sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh), cat/grep/cut was run on versions.txt on every loop iteration. - In check_existing_release_versions.sh (generation/check_existing_release_versions.sh), xmllint was run 3 separate times per POM. Optimizations: - Pure-Bash Module Extraction: Updated generate_modified_modules_list to use extract_pom_modules pom.xml, extracting all 270 modules in ~0.02s without JVM boot overhead. - Sed Showcase Version Extraction: Updated downstream-build.sh and showcase-native.sh to parse <gapic-showcase.version> directly using sed with fail-fast validation, and added --fail to curl. - XML Tag Extraction & Printf Formatting: Updated check_status.sh to extract XML tags directly with sed and replaced pipe-to-awk with built-in printf formatting. - Sed Javadoc Version Extraction: Updated update_javadoc.sh to parse project version from pom.xml using sed. - Git Ls-Remote: Updated update_googleapis_commit.sh to query HEAD commit using git ls-remote instead of cloning the repository. - Associative Array Pre-caching: Pre-cached versions.txt into declare -A versions_map in downstream-protobuf-binary-compatibility.sh instead of invoking cat/grep/cut per artifact. - Combined XPath Query: Combined 3 xmllint calls into a single concat() XPath extraction in check_existing_release_versions.sh. - Unit Tests: Verified existing unit tests in .kokoro/common_test.sh and ran bash -n and git diff --check across all modified scripts. - Refactoring & Best Practices: Scoped temporary variables as local, used pure-bash whitespace trimming and string manipulations, and anchored relative paths.
1 parent f237e41 commit bfff55a

9 files changed

Lines changed: 197 additions & 82 deletions

File tree

‎.kokoro/common.sh‎

Lines changed: 54 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -90,53 +90,55 @@ function retry_with_backoff {
9090
# and naturally survives single-module components without throwing exit signals.
9191
function extract_pom_modules() {
9292
local pom_file="$1"
93-
local modules_list=""
93+
if [[ ! -f "${pom_file}" ]]; then
94+
return 1
95+
fi
96+
local line module
9497
local in_profiles=false
9598
local in_modules=false
96-
97-
while IFS= read -r line || [ -n "$line" ]; do
98-
if [[ "$line" == *"<profiles>"* ]]; then
99+
local -a modules=()
100+
101+
while IFS= read -r line || [[ -n "${line}" ]]; do
102+
if [[ "${line}" == *"<profiles>"* ]]; then
99103
in_profiles=true
100-
elif [[ "$line" == *"</profiles>"* ]]; then
104+
elif [[ "${line}" == *"</profiles>"* ]]; then
101105
in_profiles=false
102-
elif [[ "$line" == *"<modules>"* ]] && [ "$in_profiles" = false ]; then
106+
elif [[ "${line}" == *"<modules>"* && "${in_profiles}" == "false" ]]; then
103107
in_modules=true
104-
elif [[ "$line" == *"</modules>"* ]] && [ "$in_profiles" = false ]; then
108+
elif [[ "${line}" == *"</modules>"* && "${in_profiles}" == "false" ]]; then
105109
in_modules=false
106110
break
107-
elif [ "$in_modules" = true ] && [[ "$line" == *"<module>"* ]]; then
111+
elif [[ "${in_modules}" == "true" && "${line}" == *"<module>"* ]]; then
108112
# Extract text between tags
109-
local module="${line#*<module>}"
113+
module="${line#*<module>}"
110114
module="${module%</module>*}"
111-
112-
# Trim whitespace natively
113-
module="${module#"${module%%[![:space:]]*}"}"
114-
module="${module%"${module##*[![:space:]]}"}"
115-
116-
if [ -z "$modules_list" ]; then
117-
modules_list="$module"
118-
else
119-
modules_list="${modules_list} ${module}"
115+
116+
# Trim leading/trailing whitespace without spawning external processes
117+
read -r module <<< "${module}"
118+
119+
if [[ -n "${module}" ]]; then
120+
modules+=("${module}")
120121
fi
121122
fi
122-
done < "$pom_file"
123-
124-
echo "$modules_list"
123+
done < "${pom_file}"
124+
125+
echo "${modules[*]}"
125126
}
126127

127128
# Given a folder containing a maven multi-module, assign the variable 'submodules' to a
128129
# comma-delimited list of <folder>/<submodule>.
129130
function parse_submodules() {
130131
submodules_array=()
131-
if [ -f "$1/pom.xml" ]; then
132+
if [[ -f "$1/pom.xml" ]]; then
132133
local modules
134+
local submodule
133135

134136
# Use pure Bash extraction to find the modules in the aggregator pom file.
135137
# Faster than invoking mvn help:evaluate to list all the project modules,
136138
# cleanly ignores optional <profiles>, and gracefully skips flat POMs.
137139
modules=$(extract_pom_modules "$1/pom.xml")
138-
if [ -n "$modules" ]; then
139-
for submodule in $modules; do
140+
if [[ -n "${modules}" ]]; then
141+
for submodule in ${modules}; do
140142
# Each entry = <folder>/<submodule>
141143
submodules_array+=("$1/${submodule}")
142144
done
@@ -282,39 +284,50 @@ function generate_modified_modules_list() {
282284
files=$(get_modified_files)
283285
printf "Modified files:\n%s\n" "${files}"
284286

285-
# Generate the list of valid maven modules
286-
maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')
287+
# Extract valid maven modules directly from pom.xml in pure Bash (~0.02s).
288+
# This replaces 'mvn help:evaluate -Dexpression=project.modules' which previously
289+
# spent 20-30+ seconds booting a JVM and evaluating the monorepo POMs on every run.
290+
local maven_modules_list
291+
maven_modules_list=$(extract_pom_modules pom.xml)
287292
maven_modules=()
288293

289-
# If the first argument is "true" (default), then use the module exclusion list
290-
use_exclusion_list=${1:-true}
294+
# Positional parameter $1 specifies whether to apply the exclusion list (defaults to true).
295+
local use_exclusion_list="${1:-true}"
296+
local -a all_modules=()
297+
read -r -a all_modules <<< "${maven_modules_list}"
298+
299+
local module
291300
if [[ "${use_exclusion_list}" == "true" ]]; then
292301
echo "Excluding modules from the global exclusion list"
293-
for module in $maven_modules_list; do
294-
if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then
302+
for module in "${all_modules[@]}"; do
303+
if [[ ! " ${excluded_modules[*]} " == *" ${module} "* ]]; then
295304
maven_modules+=("${module}")
296305
fi
297306
done
298307
else
299-
maven_modules=(${maven_modules_list[*]})
308+
maven_modules=("${all_modules[@]}")
300309
fi
301310

302311
modified_module_list=()
303312
# If either parent pom.xml or core shared dependency is touched, run ITs on all the modules
304313
if should_test_all_modules; then
305-
modified_module_list=(${maven_modules[*]})
314+
# '("${maven_modules[@]}")' copies the array elements safely.
315+
modified_module_list=("${maven_modules[@]}")
306316
echo "Testing the entire monorepo"
307317
else
308-
modules=$(echo "${files}" | grep -E '(google-auth|java)-.*' || true)
318+
# Extract the top-level directory from each modified file path:
319+
# 'cut -d '/' -f1' takes the first path segment (e.g. 'java-bigquery/src/...' -> 'java-bigquery').
320+
# 'sort -u' sorts and deduplicates the candidate directory names.
321+
local modules
322+
modules=$(cut -d '/' -f1 <<< "${files}" | sort -u)
309323
printf "Files in java modules:\n%s\n" "${modules}"
310-
if [[ -n $modules ]]; then
311-
modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u)
312-
for module in $modules; do
313-
if [[ " ${maven_modules[*]} " =~ " ${module} " ]]; then
314-
modified_module_list+=("${module}")
315-
fi
316-
done
317-
else
324+
for module in ${modules}; do
325+
# If this top-level directory is a recognized Maven module, add it to our list.
326+
if [[ " ${maven_modules[*]} " == *" ${module} "* ]]; then
327+
modified_module_list+=("${module}")
328+
fi
329+
done
330+
if [[ ${#modified_module_list[@]} -eq 0 ]]; then
318331
echo "Found no changes in the java modules"
319332
fi
320333
fi

‎.kokoro/common_test.sh‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,35 @@ function test_parse_pom_version {
8181
popd
8282
}
8383

84+
# Test that extract_pom_modules correctly extracts modules from root pom.xml
85+
# in pure Bash without launching Maven, and handles non-existent files safely.
86+
function test_extract_pom_modules {
87+
local -a modules
88+
read -r -a modules <<< "$(extract_pom_modules "${scriptDir}/../pom.xml")"
89+
if (( ${#modules[@]} < 200 )); then
90+
echo "extract_pom_modules failed: expected at least 200 modules, got ${#modules[@]}"
91+
exit 1
92+
fi
93+
if [[ ! " ${modules[*]} " =~ " java-bigquery " ]]; then
94+
echo "extract_pom_modules missing java-bigquery"
95+
exit 1
96+
fi
97+
if [[ ! " ${modules[*]} " =~ " java-bigquerystorage " ]]; then
98+
echo "extract_pom_modules missing java-bigquerystorage"
99+
exit 1
100+
fi
101+
if [[ ! " ${modules[*]} " =~ " sdk-platform-java " ]]; then
102+
echo "extract_pom_modules missing sdk-platform-java"
103+
exit 1
104+
fi
105+
106+
# Verify non-existent file returns 1 and empty output
107+
if extract_pom_modules "non_existent_pom.xml" &>/dev/null; then
108+
echo "extract_pom_modules should return non-zero for non-existent pom"
109+
exit 1
110+
fi
111+
}
112+
84113
# Tests that is_module_modified strictly matches the module directory prefix,
85114
# preventing prefix collisions (e.g. java-bigquery vs java-bigquerystorage).
86115
function test_is_module_modified {
@@ -191,10 +220,51 @@ function test_mock_get_modified_files {
191220
unset TEST_MODIFIED_FILES
192221
}
193222

223+
# Test that generate_modified_modules_list correctly maps modified files to
224+
# their top-level module names without requiring mvn help:evaluate.
225+
function test_generate_modified_modules_list {
226+
pushd "${scriptDir}/.." >/dev/null
227+
TEST_MODIFIED_FILES="java-bigquery/google-cloud-bigquery/src/main/java/Foo.java
228+
sdk-platform-java/gax-java/pom.xml"
229+
generate_modified_modules_list false >/dev/null
230+
231+
local has_bigquery="false"
232+
local has_sdk_platform="false"
233+
local has_bigquerystorage="false"
234+
235+
if [[ " ${modified_module_list[*]} " =~ " java-bigquery " ]]; then
236+
has_bigquery="true"
237+
fi
238+
if [[ " ${modified_module_list[*]} " =~ " sdk-platform-java " ]]; then
239+
has_sdk_platform="true"
240+
fi
241+
if [[ " ${modified_module_list[*]} " =~ " java-bigquerystorage " ]]; then
242+
has_bigquerystorage="true"
243+
fi
244+
245+
popd >/dev/null
246+
unset TEST_MODIFIED_FILES
247+
248+
if [[ "${has_bigquery}" != "true" ]]; then
249+
echo "generate_modified_modules_list missing java-bigquery"
250+
exit 1
251+
fi
252+
if [[ "${has_sdk_platform}" != "true" ]]; then
253+
echo "generate_modified_modules_list missing sdk-platform-java"
254+
exit 1
255+
fi
256+
if [[ "${has_bigquerystorage}" == "true" ]]; then
257+
echo "generate_modified_modules_list incorrectly included java-bigquerystorage"
258+
exit 1
259+
fi
260+
}
261+
194262
test_find_all_poms_with_versioned_dependency
195263
test_update_pom_dependency
196264
test_parse_pom_version
197265
test_mock_get_modified_files
266+
test_extract_pom_modules
198267
test_should_test_all_modules
199268
test_is_module_modified
269+
test_generate_modified_modules_list
200270

‎.kokoro/presubmit/downstream-build.sh‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,23 @@ pushd java-showcase
4747
modify_shared_config
4848
popd
4949

50-
# Parse showcase version from the local directory
51-
pushd java-showcase/gapic-showcase
52-
SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)
53-
popd
50+
# Extract the showcase version directly from pom.xml using sed:
51+
# - 'sed -n': suppresses default line printing.
52+
# - 's:...[[:space:]]*\([^<[:space:]]*\).*:\1:p': captures non-whitespace version text and prints it.
53+
# - '/.../q': quits immediately on first match, avoiding trailing passes and external pipe utilities.
54+
# This replaces 'mvn help:evaluate' which previously took 15+ seconds to boot Maven.
55+
SHOWCASE_VERSION=$(sed -n 's:.*<gapic-showcase\.version>[[:space:]]*\([^<[:space:]]*\).*:\1:p; /<gapic-showcase\.version>/q' java-showcase/gapic-showcase/pom.xml)
56+
57+
# Fail fast with a clear error message if the version could not be parsed,
58+
# preventing malformed curl URLs and ambiguous downstream failures.
59+
if [[ -z "${SHOWCASE_VERSION}" ]]; then
60+
echo "Error: Failed to parse gapic-showcase.version from java-showcase/gapic-showcase/pom.xml" >&2
61+
exit 1
62+
fi
5463

5564
# Start showcase server
5665
mkdir -p /usr/src/showcase
57-
curl --location https://github.com/googleapis/gapic-showcase/releases/download/v"${SHOWCASE_VERSION}"/gapic-showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz --output /usr/src/showcase/showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz
66+
curl --fail --location https://github.com/googleapis/gapic-showcase/releases/download/v"${SHOWCASE_VERSION}"/gapic-showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz --output /usr/src/showcase/showcase-"${SHOWCASE_VERSION}"-linux-amd64.tar.gz
5867
pushd /usr/src/showcase/
5968
tar -xf showcase-*
6069
./gapic-showcase run &

‎generation/check_existing_release_versions.sh‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@ function find_existing_version_pom() {
1111
echo "Empty pom file name"
1212
exit 1
1313
fi
14-
local group_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="groupId"]/text()' \
15-
"${pom_file}")
16-
local artifact_id=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="artifactId"]/text()' \
17-
"${pom_file}")
18-
local version=$(xmllint --xpath '/*[local-name()="project"]/*[local-name()="version"]/text()' \
19-
"${pom_file}")
14+
local coordinates=$(xmllint --xpath 'concat(normalize-space(/*[local-name()="project"]/*[local-name()="groupId"]), ":", normalize-space(/*[local-name()="project"]/*[local-name()="artifactId"]), ":", normalize-space(/*[local-name()="project"]/*[local-name()="version"]))' \
15+
"${pom_file}" 2>/dev/null || true)
16+
local group_id artifact_id version
17+
IFS=':' read -r group_id artifact_id version <<< "${coordinates}"
2018
echo -n "Checking ${group_id}:${artifact_id}:${version}:"
2119
if [ -z "${artifact_id}" ]; then
2220
echo "Couldn't parse artifact_id in the pom file: $pom_file"

‎google-auth-library-java/scripts/update_javadoc.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@
3131

3232
set -e
3333

34-
VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev '(^\[|\w+:)')
34+
VERSION=$(sed -n '/<parent>/,/<\/parent>/d; /<version>/{s:.*<version>[[:space:]]*\([^<[:space:]]*\).*:\1:p; q;}' pom.xml)
3535

3636
if [ -z "$VERSION" ]; then
37-
echo "Error updating Javadoc: could not obtain version number from maven-help-plugin."
37+
echo "Error updating Javadoc: could not obtain version number from pom.xml."
3838
exit 1
3939
fi
4040

‎java-cloud-bom/tests/release-repository-readiness/check_status.sh‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,28 @@ if [[ ! -d "${WORK_DIR}/sdk-platform-java" ]]; then
1111
exit 1
1212
fi
1313

14+
function extract_xml_tag() {
15+
local tag=$1
16+
local file=$2
17+
[[ ! -f "${file}" ]] && return 0
18+
sed -n "\|<${tag}[ >]|{s|.*<${tag}[^>]*>[[:space:]]*\([^<[:space:]]*\).*|\1|p; q;}" "${file}"
19+
}
20+
1421
cd "${WORK_DIR}/sdk-platform-java"
15-
expected_shared_deps_version=$(mvn -pl java-shared-dependencies help:evaluate -Dexpression=project.version -q -DforceStdout)
16-
expected_generator_version=$(mvn -pl gapic-generator-java help:evaluate -Dexpression=project.version -q -DforceStdout)
22+
expected_shared_deps_version=$(extract_xml_tag version java-shared-dependencies/pom.xml)
23+
expected_generator_version=$(extract_xml_tag version gapic-generator-java/pom.xml)
1724
echo "Expected google-cloud-shared-dependencies BOM version: ${expected_shared_deps_version}"
1825
echo "Expected GAPIC generator Java version: ${expected_generator_version}"
1926

2027

2128
function check_shared_dependency_status() {
2229
local project=$1
23-
local actual_shared_deps_version=$(mvn -pl ${project} help:evaluate -Dexpression=google-cloud-shared-dependencies.version -q -DforceStdout)
30+
local pom="${project}/pom.xml"
31+
local actual_shared_deps_version=$(extract_xml_tag google-cloud-shared-dependencies.version "${pom}")
32+
if [[ -z "${actual_shared_deps_version}" && -f "${pom}" ]]; then
33+
# Fallback for parent POMs that inherit or track via version-update comment
34+
actual_shared_deps_version=$(sed -n 's|.*<version>[[:space:]]*\([^<[:space:]]*\).*x-version-update:google-cloud-shared-dependencies:current.*|\1|p; /x-version-update:google-cloud-shared-dependencies:current/q' "${pom}")
35+
fi
2436
if [[ "${expected_shared_deps_version}" != "${actual_shared_deps_version}" ]]; then
2537
local shared_deps_status="! ${actual_shared_deps_version}"
2638
else
@@ -42,11 +54,8 @@ function check_generated_code_status() {
4254
fi
4355
echo "${generator_status}"
4456
}
45-
echo ", main, released" |\
46-
awk -F',' '{printf "%-20s|%-21s|%-21s|\n", $1, $2, $3}'
47-
48-
echo " repository,shared dep,code gen,shared dep,code gen" |\
49-
awk -F',' '{printf "%-20s|%-10s|%-10s|%-10s|%-10s|\n", $1, $2, $3, $4, $5}'
57+
printf "%-20s|%-21s|%-21s|\n" "" " main" " released"
58+
printf "%-20s|%-10s|%-10s|%-10s|%-10s|\n" " repository" "shared dep" "code gen" "shared dep" "code gen"
5059

5160
repositories=$(find "${WORK_DIR}" -mindepth 1 -maxdepth 1 -type d -not -name "sdk-platform-java")
5261
for repo_folder in $repositories; do
@@ -72,9 +81,12 @@ for repo_folder in $repositories; do
7281
shared_deps_status_last_release=$(check_shared_dependency_status "${project}")
7382
generated_code_status_last_release=$(check_generated_code_status)
7483

75-
echo "${repo},${shared_deps_status_main},${generated_code_status_main},\
76-
${shared_deps_status_last_release},${generated_code_status_last_release}" | \
77-
awk -F',' '{printf "%-20s|%-10s|%-10s|%-10s|%-10s|\n", $1, $2, $3, $4, $5}'
84+
printf "%-20s|%-10s|%-10s|%-10s|%-10s|\n" \
85+
"${repo}" \
86+
"${shared_deps_status_main}" \
87+
"${generated_code_status_main}" \
88+
"${shared_deps_status_last_release}" \
89+
"${generated_code_status_last_release}"
7890
done
7991

8092

‎sdk-platform-java/.github/scripts/update_googleapis_commit.sh‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -72,14 +72,11 @@ fi
7272
# conflict.
7373
git merge -m "chore: merge ${base_branch} into ${current_branch}" "${base_branch}"
7474

75-
mkdir tmp-googleapis
76-
# Use partial clone because only commit history is needed.
77-
git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
78-
pushd tmp-googleapis
79-
git pull
80-
latest_commit=$(git rev-parse HEAD)
81-
popd
82-
rm -rf tmp-googleapis
75+
latest_commit=$(git ls-remote https://github.com/googleapis/googleapis.git HEAD | awk '{print $1}')
76+
if [ -z "${latest_commit}" ]; then
77+
echo "Error: Failed to obtain latest commit from googleapis/googleapis" >&2
78+
exit 1
79+
fi
8380
sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}"
8481

8582
git add "${generation_config}"

0 commit comments

Comments
 (0)