From 39981c5328b6ad5114eec3bb9dc45591ea91dd95 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Mar 2026 23:29:18 -0800 Subject: [PATCH 01/22] get list of changed files and their dependants --- .../workflows/cpp-ci-serial-programs-base.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 09d52a5ba1..a9d5cb8776 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -131,6 +131,87 @@ jobs: run : | cd Arduino-Source + # git diff with relative paths + git diff --name-only origin/main...HEAD > changed_files.txt + + echo "Generating clang-scan-deps experimental-full > deps.json." + # get dependency graph + clang-scan-deps -compilation-database compile_commands.json -format experimental-full > deps.json + + # normalize slashes + # sed 's|\\\\|/|g' deps.json > normalized_deps.json + sed -i 's|\\\\|/|g' deps.json + + # check if deps.json has the expected keys + # because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. + TU_KEY="translation-units" + CMD_KEY="commands" + DEPS="file-deps" + INPUT="input-file" + + JQ_SCRIPT=$(cat << 'EOF' + # 1. Access the target object + (.[$TU][0][$CMD][0]) as $target + + # 2. Define the required keys + + | [$DEPS, $INPUT] as $required + + | ( + if .[$TU] == null then + "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" + elif .[$TU][0] == null then + "Missing: \($TU)[0]" + elif .[$TU][0].[$CMD] == null then + "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" + elif $target == null then + "Missing: \($TU)[0].[$CMD][0]" + elif ($required | all(. as $req | $target | has($req)) | not) then + "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" + # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then + # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" + else + "All keys \($required) found in \($TU)[0].\($CMD)[0]" + end + ) as $result + + | if ($result | type == "string" and startswith("Missing:")) then + ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error + else $result end + EOF + ) + + echo "Checking keys in deps.json." + jq -r "$JQ_SCRIPT" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" \ + deps.json + + echo "Generating files_to_query.txt." + # for each line in changed_files.txt, search deps.json to find all their dependants + # + jq -r --rawfile mod changed_files.txt \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" ' + # 1. Clean the list of changed files + ($mod | split("\n") | map(select(length > 0))) as $changes | + + # 2. Access the translation-units array + [ .[$TU][] | .[$CMD][] | + select( + # 3. Check "file-deps" for matches + .[$DEPS][] | . as $dp | + any($changes[]; . as $c | $dp | endswith($c)) + ) | + # 4. Get the source file path + .[$INPUT] + ] | unique[] + ' normalized_deps.json > files_to_query.txt + cat << 'EOF' > query.txt set output dump match invocation( From af593bae37d4436d0e37031349b130e507d26f0e Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 09:40:56 -0800 Subject: [PATCH 02/22] filter compile_commands.json, to remove .rc files --- .github/workflows/cpp-ci-serial-programs-base.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index a9d5cb8776..5b779492ef 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -135,8 +135,12 @@ jobs: git diff --name-only origin/main...HEAD > changed_files.txt echo "Generating clang-scan-deps experimental-full > deps.json." + + # filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format + jq '[.[] | select(.file | endswith(".rc") | not)]' compile_commands.json > compile_commands_filtered.json + # get dependency graph - clang-scan-deps -compilation-database compile_commands.json -format experimental-full > deps.json + clang-scan-deps -compilation-database compile_commands_filtered.json -format experimental-full > deps.json # normalize slashes # sed 's|\\\\|/|g' deps.json > normalized_deps.json From 63031c89b37d2f1db8789ac33ebea1ed7545ff33 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 11:54:12 -0800 Subject: [PATCH 03/22] move clang-query into its own shell script --- .github/scripts/clang-query.sh | 124 ++++++++++++++++++ .../workflows/cpp-ci-serial-programs-base.yml | 104 +-------------- 2 files changed, 125 insertions(+), 103 deletions(-) create mode 100644 .github/scripts/clang-query.sh diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh new file mode 100644 index 0000000000..96341177b6 --- /dev/null +++ b/.github/scripts/clang-query.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TMP_DIR="$REPO_ROOT/.ci_tmp" +mkdir -p "$TMP_DIR" + +cd "$REPO_ROOT" + +# find path to compile_commands.json +if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json" +elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" +else + echo "Error: compile_commands.json not found!" + exit 1 +fi + + +# git diff with relative paths +git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" + +echo "Generating clang-scan-deps experimental-full > deps.json." + +# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format +jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json" + +# get dependency graph +clang-scan-deps -compilation-database compile_commands_filtered.json -format experimental-full > "$TMP_DIR/deps.json" + +# normalize slashes +# sed 's|\\\\|/|g' deps.json > normalized_deps.json +sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json" + +# check if deps.json has the expected keys +# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. +TU_KEY="translation-units" +CMD_KEY="commands" +DEPS="file-deps" +INPUT="input-file" + +JQ_SCRIPT=$(cat << 'EOF' + # 1. Access the target object + (.[$TU][0][$CMD][0]) as $target + + # 2. Define the required keys + + | [$DEPS, $INPUT] as $required + + | ( + if .[$TU] == null then + "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" + elif .[$TU][0] == null then + "Missing: \($TU)[0]" + elif .[$TU][0].[$CMD] == null then + "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" + elif $target == null then + "Missing: \($TU)[0].[$CMD][0]" + elif ($required | all(. as $req | $target | has($req)) | not) then + "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" + # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then + # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" + else + "All keys \($required) found in \($TU)[0].\($CMD)[0]" + end + ) as $result + + | if ($result | type == "string" and startswith("Missing:")) then + ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error + else $result end +EOF +) + +echo "Checking keys in deps.json." +jq -r "$JQ_SCRIPT" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" \ + "$TMP_DIR/deps.json" + +echo "Generating files_to_query.txt." +# for each line in changed_files.txt, search deps.json to find all their dependants +# +jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" ' + # 1. Clean the list of changed files + ($mod | split("\n") | map(select(length > 0))) as $changes | + + # 2. Access the translation-units array + [ .[$TU][] | .[$CMD][] | + select( + # 3. Check "file-deps" for matches + .[$DEPS][] | . as $dp | + any($changes[]; . as $c | $dp | endswith($c)) + ) | + # 4. Get the source file path + .[$INPUT] + ] | unique[] +' "$TMP_DIR/deps.json" > "$TMP_DIR/files_to_query.txt" + +cat << 'EOF' > "$TMP_DIR/query.txt" +set output dump +match invocation( + isExpansionInFileMatching("SerialPrograms/"), + hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), + hasArgument(0, hasType(asString("std::string"))) +) +EOF + +echo "Running clang-query." + +files=$(jq -r '.[].file' SerialPrograms/bin/compile_commands.json) +echo "$files" | xargs --max-args=150 clang-query -p SerialPrograms/bin/ -f "$TMP_DIR/query.txt" >> output.txt +cat output.txt +if grep --silent "Match #" output.txt; then + echo "::error Forbidden std::filesystem::path construction detected!" + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 5b779492ef..de4f60128c 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -128,107 +128,5 @@ jobs: - name: Run clang query if: inputs.run-clang-query - run : | - cd Arduino-Source - - # git diff with relative paths - git diff --name-only origin/main...HEAD > changed_files.txt - - echo "Generating clang-scan-deps experimental-full > deps.json." + run : bash ./.github/scripts/clang-query.sh - # filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format - jq '[.[] | select(.file | endswith(".rc") | not)]' compile_commands.json > compile_commands_filtered.json - - # get dependency graph - clang-scan-deps -compilation-database compile_commands_filtered.json -format experimental-full > deps.json - - # normalize slashes - # sed 's|\\\\|/|g' deps.json > normalized_deps.json - sed -i 's|\\\\|/|g' deps.json - - # check if deps.json has the expected keys - # because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. - TU_KEY="translation-units" - CMD_KEY="commands" - DEPS="file-deps" - INPUT="input-file" - - JQ_SCRIPT=$(cat << 'EOF' - # 1. Access the target object - (.[$TU][0][$CMD][0]) as $target - - # 2. Define the required keys - - | [$DEPS, $INPUT] as $required - - | ( - if .[$TU] == null then - "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" - elif .[$TU][0] == null then - "Missing: \($TU)[0]" - elif .[$TU][0].[$CMD] == null then - "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" - elif $target == null then - "Missing: \($TU)[0].[$CMD][0]" - elif ($required | all(. as $req | $target | has($req)) | not) then - "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" - # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then - # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" - else - "All keys \($required) found in \($TU)[0].\($CMD)[0]" - end - ) as $result - - | if ($result | type == "string" and startswith("Missing:")) then - ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error - else $result end - EOF - ) - - echo "Checking keys in deps.json." - jq -r "$JQ_SCRIPT" \ - --arg TU "$TU_KEY" \ - --arg CMD "$CMD_KEY" \ - --arg DEPS "$DEPS" \ - --arg INPUT "$INPUT" \ - deps.json - - echo "Generating files_to_query.txt." - # for each line in changed_files.txt, search deps.json to find all their dependants - # - jq -r --rawfile mod changed_files.txt \ - --arg TU "$TU_KEY" \ - --arg CMD "$CMD_KEY" \ - --arg DEPS "$DEPS" \ - --arg INPUT "$INPUT" ' - # 1. Clean the list of changed files - ($mod | split("\n") | map(select(length > 0))) as $changes | - - # 2. Access the translation-units array - [ .[$TU][] | .[$CMD][] | - select( - # 3. Check "file-deps" for matches - .[$DEPS][] | . as $dp | - any($changes[]; . as $c | $dp | endswith($c)) - ) | - # 4. Get the source file path - .[$INPUT] - ] | unique[] - ' normalized_deps.json > files_to_query.txt - - cat << 'EOF' > query.txt - set output dump - match invocation( - isExpansionInFileMatching("SerialPrograms/"), - hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), - hasArgument(0, hasType(asString("std::string"))) - ) - EOF - - files=$(jq -r '.[].file' SerialPrograms/bin/compile_commands.json) - echo "$files" | xargs --max-args=150 clang-query -p SerialPrograms/bin/ -f query.txt >> output.txt - cat output.txt - if grep --silent "Match #" output.txt; then - echo "::error Forbidden std::filesystem::path construction detected!" - exit 1 - fi From 48178fb2a45d28af81cfbddc25ac04142e2c835a Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 12:33:11 -0800 Subject: [PATCH 04/22] update clang-query.sh. update .gitattributes to force shell scipts to use LF --- .gitattributes | 5 +- .github/scripts/clang-query.sh | 257 +++++++++++++++++---------------- 2 files changed, 138 insertions(+), 124 deletions(-) diff --git a/.gitattributes b/.gitattributes index 5378fe089b..5b82ef218b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,4 @@ -* -text \ No newline at end of file +* -text + +# Explicitly ensure shell scripts use LF for CI compatibility +*.sh text eol=lf \ No newline at end of file diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index 96341177b6..b6924bce91 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -1,124 +1,135 @@ -#!/bin/bash -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -TMP_DIR="$REPO_ROOT/.ci_tmp" -mkdir -p "$TMP_DIR" - -cd "$REPO_ROOT" - -# find path to compile_commands.json -if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then - DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json" -elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then - DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" -else - echo "Error: compile_commands.json not found!" - exit 1 -fi - - -# git diff with relative paths -git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" - -echo "Generating clang-scan-deps experimental-full > deps.json." - -# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format -jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json" - -# get dependency graph -clang-scan-deps -compilation-database compile_commands_filtered.json -format experimental-full > "$TMP_DIR/deps.json" - -# normalize slashes -# sed 's|\\\\|/|g' deps.json > normalized_deps.json -sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json" - -# check if deps.json has the expected keys -# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. -TU_KEY="translation-units" -CMD_KEY="commands" -DEPS="file-deps" -INPUT="input-file" - -JQ_SCRIPT=$(cat << 'EOF' - # 1. Access the target object - (.[$TU][0][$CMD][0]) as $target - - # 2. Define the required keys - - | [$DEPS, $INPUT] as $required - - | ( - if .[$TU] == null then - "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" - elif .[$TU][0] == null then - "Missing: \($TU)[0]" - elif .[$TU][0].[$CMD] == null then - "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" - elif $target == null then - "Missing: \($TU)[0].[$CMD][0]" - elif ($required | all(. as $req | $target | has($req)) | not) then - "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" - # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then - # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" - else - "All keys \($required) found in \($TU)[0].\($CMD)[0]" - end - ) as $result - - | if ($result | type == "string" and startswith("Missing:")) then - ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error - else $result end -EOF -) - -echo "Checking keys in deps.json." -jq -r "$JQ_SCRIPT" \ - --arg TU "$TU_KEY" \ - --arg CMD "$CMD_KEY" \ - --arg DEPS "$DEPS" \ - --arg INPUT "$INPUT" \ - "$TMP_DIR/deps.json" - -echo "Generating files_to_query.txt." -# for each line in changed_files.txt, search deps.json to find all their dependants -# -jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ - --arg TU "$TU_KEY" \ - --arg CMD "$CMD_KEY" \ - --arg DEPS "$DEPS" \ - --arg INPUT "$INPUT" ' - # 1. Clean the list of changed files - ($mod | split("\n") | map(select(length > 0))) as $changes | - - # 2. Access the translation-units array - [ .[$TU][] | .[$CMD][] | - select( - # 3. Check "file-deps" for matches - .[$DEPS][] | . as $dp | - any($changes[]; . as $c | $dp | endswith($c)) - ) | - # 4. Get the source file path - .[$INPUT] - ] | unique[] -' "$TMP_DIR/deps.json" > "$TMP_DIR/files_to_query.txt" - -cat << 'EOF' > "$TMP_DIR/query.txt" -set output dump -match invocation( - isExpansionInFileMatching("SerialPrograms/"), - hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), - hasArgument(0, hasType(asString("std::string"))) -) -EOF - -echo "Running clang-query." - -files=$(jq -r '.[].file' SerialPrograms/bin/compile_commands.json) -echo "$files" | xargs --max-args=150 clang-query -p SerialPrograms/bin/ -f "$TMP_DIR/query.txt" >> output.txt -cat output.txt -if grep --silent "Match #" output.txt; then - echo "::error Forbidden std::filesystem::path construction detected!" - exit 1 +#!/usr/bin/env bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TMP_DIR="$REPO_ROOT/.ci_tmp" +mkdir -p "$TMP_DIR" + +# Define the cleanup function +cleanup() { + echo "Cleaning up temporary files..." + rm -rf "$TMP_DIR" +} + +# Register the trap: run cleanup on EXIT, plus common signals like INT (Ctrl+C) or TERM +trap cleanup EXIT INT TERM + + +cd "$REPO_ROOT" + +# find path to compile_commands.json +if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json" +elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then + DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" +else + echo "Error: compile_commands.json not found!" + exit 1 +fi + + +# git diff with relative paths +git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" + +echo "Generating clang-scan-deps experimental-full > deps.json." + +# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format +jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json" + +# get dependency graph +clang-scan-deps -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json" + +# normalize slashes +# sed 's|\\\\|/|g' deps.json > normalized_deps.json +sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json" + +# check if deps.json has the expected keys +# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change. +TU_KEY="translation-units" +CMD_KEY="commands" +DEPS="file-deps" +INPUT="input-file" + +JQ_SCRIPT=$(cat << 'EOF' + # 1. Access the target object + (.[$TU][0][$CMD][0]) as $target + + # 2. Define the required keys + + | [$DEPS, $INPUT] as $required + + | ( + if .[$TU] == null then + "Missing: \($TU). Keys found at top-level: \(keys_unsorted)" + elif .[$TU][0] == null then + "Missing: \($TU)[0]" + elif .[$TU][0].[$CMD] == null then + "Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)" + elif $target == null then + "Missing: \($TU)[0].[$CMD][0]" + elif ($required | all(. as $req | $target | has($req)) | not) then + "Missing: One or more required keys \($required). Found: \($target | keys_unsorted)" + # elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then + # "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)" + else + "All keys \($required) found in \($TU)[0].\($CMD)[0]" + end + ) as $result + + | if ($result | type == "string" and startswith("Missing:")) then + ("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error + else $result end +EOF +) + +echo "Checking keys in deps.json." +jq -r "$JQ_SCRIPT" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" \ + "$TMP_DIR/deps.json" + +echo "Generating files_to_query.txt." +# for each line in changed_files.txt, search deps.json to find all their dependants +# +jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ + --arg TU "$TU_KEY" \ + --arg CMD "$CMD_KEY" \ + --arg DEPS "$DEPS" \ + --arg INPUT "$INPUT" ' + # 1. Clean the list of changed files + ($mod | split("\n") | map(select(length > 0))) as $changes | + + # 2. Access the translation-units array + [ .[$TU][] | .[$CMD][] | + select( + # 3. Check "file-deps" for matches + .[$DEPS][] | . as $dp | + any($changes[]; . as $c | $dp | endswith($c)) + ) | + # 4. Get the source file path + .[$INPUT] + ] | unique[] +' "$TMP_DIR/deps.json" > "$TMP_DIR/files_to_query.txt" + +cat << 'EOF' > "$TMP_DIR/query.txt" +set output dump +match invocation( + isExpansionInFileMatching("SerialPrograms/"), + hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))), + hasArgument(0, hasType(asString("std::string"))) +) +EOF + +echo "Running clang-query." + +files=$(jq -r '.[].file' "$DB_PATH") +DB_DIR=$(dirname "$DB_PATH") +echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt +cat output.txt +if grep --silent "Match #" output.txt; then + echo "::error Forbidden std::filesystem::path construction detected!" + exit 1 fi \ No newline at end of file From 9caa7de6231d023e3a226ab494db506be742742e Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 21:24:02 -0800 Subject: [PATCH 05/22] update clang-query. --- .github/scripts/clang-query.sh | 43 +++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index b6924bce91..a74fb6cab7 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -13,7 +13,7 @@ cleanup() { } # Register the trap: run cleanup on EXIT, plus common signals like INT (Ctrl+C) or TERM -trap cleanup EXIT INT TERM +# trap cleanup EXIT INT TERM cd "$REPO_ROOT" @@ -114,6 +114,8 @@ jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ ] | unique[] ' "$TMP_DIR/deps.json" > "$TMP_DIR/files_to_query.txt" + + cat << 'EOF' > "$TMP_DIR/query.txt" set output dump match invocation( @@ -125,9 +127,44 @@ EOF echo "Running clang-query." -files=$(jq -r '.[].file' "$DB_PATH") +# files=$(jq -r '.[].file' "$DB_PATH") DB_DIR=$(dirname "$DB_PATH") -echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt +#echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt + +# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" -- -Wno-unused-command-line-argument >> "$TMP_DIR/output.txt" + +# this works +# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + +# also works +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 \ +# clang-query -p "$DB_DIR" \ +# --extra-arg="-Wno-unused-command-line-argument" \ +# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + +# also works +# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' | \ +# xargs -d '\n' --max-args=150 \ +# clang-query -p "$DB_DIR" \ +# --extra-arg="-Wno-unused-command-line-argument" \ +# --extra-arg="-Wno-unused-function" \ +# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + + +LIST_FILE="$TMP_DIR/file_list.txt" + +jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE" + +# 2. Run clang-query using the list file +# We use -a to read arguments from the file +xargs -d '\n' -a "$LIST_FILE" --max-args=150 \ + clang-query -p "$DB_DIR" \ + --extra-arg="-Wno-unused-command-line-argument" \ + --extra-arg="-Wno-unused-function" \ + -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" + + cat output.txt if grep --silent "Match #" output.txt; then echo "::error Forbidden std::filesystem::path construction detected!" From 2d48c3c0c1425187aeb0522b92b095ca5b4aae2d Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 21:38:48 -0800 Subject: [PATCH 06/22] run on files_to_query instead of the full file_list. --- .github/scripts/clang-query.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index a74fb6cab7..af0b114d83 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -153,20 +153,28 @@ DB_DIR=$(dirname "$DB_PATH") LIST_FILE="$TMP_DIR/file_list.txt" - jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE" -# 2. Run clang-query using the list file -# We use -a to read arguments from the file -xargs -d '\n' -a "$LIST_FILE" --max-args=150 \ - clang-query -p "$DB_DIR" \ - --extra-arg="-Wno-unused-command-line-argument" \ - --extra-arg="-Wno-unused-function" \ - -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +LIST_FILE="$TMP_DIR/files_to_query.txt" + +> "$TMP_DIR/output.txt" + +# Run clang-query using the list file +# check if LIST_FILE has any data to analyze +if [ ! -s "$LIST_FILE" ]; then + echo "No files found to analyze. Skipping Clang-Query." +else + xargs -d '\n' -a "$LIST_FILE" --max-args=150 \ + clang-query -p "$DB_DIR" \ + --extra-arg="-Wno-unused-command-line-argument" \ + --extra-arg="-Wno-unused-function" \ + -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +fi + -cat output.txt -if grep --silent "Match #" output.txt; then +cat "$TMP_DIR/output.txt" +if grep --silent "Match #" "$TMP_DIR/output.txt"; then echo "::error Forbidden std::filesystem::path construction detected!" exit 1 fi \ No newline at end of file From c7d092f356c7a58c048780f00e6af1ddc027cd45 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:21:02 -0800 Subject: [PATCH 07/22] files_to_query to use LF. added ONLY_CHECK_CHANGED_FILES flag --- .github/scripts/clang-query.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index af0b114d83..802a3da672 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -29,9 +29,6 @@ else fi -# git diff with relative paths -git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" - echo "Generating clang-scan-deps experimental-full > deps.json." # filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format @@ -91,9 +88,14 @@ jq -r "$JQ_SCRIPT" \ --arg INPUT "$INPUT" \ "$TMP_DIR/deps.json" -echo "Generating files_to_query.txt." -# for each line in changed_files.txt, search deps.json to find all their dependants -# +echo "Generating changed_files.txt from git diff." + +# git diff with relative paths +git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" + +echo "Generating files_to_query.txt, based on changed_files.txt and deps.json." + +# for each line in changed_files_unix.txt, search deps.json to find all their dependants jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ --arg TU "$TU_KEY" \ --arg CMD "$CMD_KEY" \ @@ -112,7 +114,7 @@ jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \ # 4. Get the source file path .[$INPUT] ] | unique[] -' "$TMP_DIR/deps.json" > "$TMP_DIR/files_to_query.txt" +' "$TMP_DIR/deps.json" | tr -d '\r' > "$TMP_DIR/files_to_query.txt" @@ -151,11 +153,13 @@ DB_DIR=$(dirname "$DB_PATH") # --extra-arg="-Wno-unused-function" \ # -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" - -LIST_FILE="$TMP_DIR/file_list.txt" -jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE" - -LIST_FILE="$TMP_DIR/files_to_query.txt" +ONLY_CHECK_CHANGED_FILES=true +if [ "$ONLY_CHECK_CHANGED_FILES" = "true" ]; then + LIST_FILE="$TMP_DIR/files_to_query.txt" +else # check all files + LIST_FILE="$TMP_DIR/file_list.txt" + jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE" +fi > "$TMP_DIR/output.txt" From c1db53a3d07db083559e4d23a75717b9a695fbec Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:22:43 -0800 Subject: [PATCH 08/22] test --- .../Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 409b205636..9cfdeb061a 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -532,6 +532,9 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& detector.process_frame(snapshot, current_time()); #endif + std::filesystem::path("hello"); + + #if 0 UpdateMenuWatcher update_menu(console, COLOR_PURPLE); CheckOnlineWatcher check_online(COLOR_CYAN); From ce64e5dca8d64176292c5c38d9a5bab0cfba8004 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:31:42 -0800 Subject: [PATCH 09/22] test --- .../Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 9cfdeb061a..e86e296528 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -532,7 +532,8 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& detector.process_frame(snapshot, current_time()); #endif - std::filesystem::path("hello"); + std::string test = "hello"; + std::filesystem::path(test); #if 0 From fb847a7e9f2111bef1675f5c88b829162df0a845 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:33:51 -0800 Subject: [PATCH 10/22] test --- .../Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index e86e296528..d683eeb2e4 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -533,7 +533,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& #endif std::string test = "hello"; - std::filesystem::path(test); + std::filesystem::path p{test}; #if 0 From a2b872c38ea88e8d8ac4e175c7c1ebe17077f312 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:39:09 -0800 Subject: [PATCH 11/22] test --- .../Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index d683eeb2e4..4b9c6ab8ce 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -535,6 +535,9 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& std::string test = "hello"; std::filesystem::path p{test}; + std::string out_path = "hello" + "/" + file_stat.m_filename; + std::filesystem::path const parent_dir{std::filesystem::path(out_path).parent_path()}; + #if 0 UpdateMenuWatcher update_menu(console, COLOR_PURPLE); From ac6bb68a27a83fbca465ecc878c88fea1c1889ef Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 5 Mar 2026 22:39:44 -0800 Subject: [PATCH 12/22] test --- .../Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 4b9c6ab8ce..1a65c3c3bc 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -535,7 +535,7 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& std::string test = "hello"; std::filesystem::path p{test}; - std::string out_path = "hello" + "/" + file_stat.m_filename; + std::string out_path = "hello/"; std::filesystem::path const parent_dir{std::filesystem::path(out_path).parent_path()}; From 7c992e06cf366fc9ce4b71eb3bca5c61d547ab92 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 08:54:44 -0800 Subject: [PATCH 13/22] update working directory --- .github/workflows/cpp-ci-serial-programs-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index de4f60128c..1e4c6aa685 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -128,5 +128,6 @@ jobs: - name: Run clang query if: inputs.run-clang-query + working-directory: ./Arduino-Source run : bash ./.github/scripts/clang-query.sh From a650b1788ce66f4d5ee82c05fb3084a50a85226a Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 12:38:29 -0800 Subject: [PATCH 14/22] update usage of clang-scan-deps to be cross platform --- .github/scripts/clang-query.sh | 12 +++++++++++- .github/workflows/cpp-ci-serial-programs-base.yml | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index 802a3da672..0817de7815 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -31,11 +31,21 @@ fi echo "Generating clang-scan-deps experimental-full > deps.json." +# in ubuntu, the command is clang-scan-deps-18. in Windows, it is clang-scan-deps +SCAN_DEPS=$(command -v clang-scan-deps-18 || command -v clang-scan-deps) + +# Safety check: Exit if the tool isn't found +if [ -z "$SCAN_DEPS" ]; then + echo "Error: clang-scan-deps (or version -18) not found in PATH." + exit 1 +fi + + # filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json" # get dependency graph -clang-scan-deps -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json" +"$SCAN_DEPS" -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json" # normalize slashes # sed 's|\\\\|/|g' deps.json > normalized_deps.json diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 1e4c6aa685..d352f5120d 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -64,7 +64,7 @@ jobs: cd Arduino-Source sudo apt update sudo apt upgrade - sudo apt install clang-tools libopencv-dev + sudo apt install clang-tools-18 libopencv-dev sudo apt install ./3rdPartyBinaries/libdpp-10.0.28-linux-x64.deb From f01b621b341fc372c2895876eb3a2cae024ecda5 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 12:38:48 -0800 Subject: [PATCH 15/22] delete unused CI runners for testing --- .../workflows/cpp-ci-serial-programs-mac-intel.yml | 10 ---------- .github/workflows/cpp-ci-serial-programs-mac.yml | 10 ---------- .../cpp-ci-serial-programs-ubuntu-default.yml | 10 ---------- .../cpp-ci-serial-programs-windows-clang.yml | 11 ----------- .../cpp-ci-serial-programs-windows-default.yml | 11 ----------- 5 files changed, 52 deletions(-) delete mode 100644 .github/workflows/cpp-ci-serial-programs-mac-intel.yml delete mode 100644 .github/workflows/cpp-ci-serial-programs-mac.yml delete mode 100644 .github/workflows/cpp-ci-serial-programs-ubuntu-default.yml delete mode 100644 .github/workflows/cpp-ci-serial-programs-windows-clang.yml delete mode 100644 .github/workflows/cpp-ci-serial-programs-windows-default.yml diff --git a/.github/workflows/cpp-ci-serial-programs-mac-intel.yml b/.github/workflows/cpp-ci-serial-programs-mac-intel.yml deleted file mode 100644 index d3664a0b22..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-mac-intel.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Mac Intel -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: macos-15-intel - compiler: default - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-mac.yml b/.github/workflows/cpp-ci-serial-programs-mac.yml deleted file mode 100644 index a021d44c13..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-mac.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Mac -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: macos-15 - compiler: default - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml b/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml deleted file mode 100644 index 5c4a9b2fcb..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: C++ CI Serial Programs Ubuntu Default -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: ubuntu-24.04 - compiler: default - run-clang-query: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-windows-clang.yml b/.github/workflows/cpp-ci-serial-programs-windows-clang.yml deleted file mode 100644 index 7c616ff1ac..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-windows-clang.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: C++ CI Serial Programs Windows Clang -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: windows-2025 - compiler: clang - upload-build: true - run-tests: true \ No newline at end of file diff --git a/.github/workflows/cpp-ci-serial-programs-windows-default.yml b/.github/workflows/cpp-ci-serial-programs-windows-default.yml deleted file mode 100644 index 4e835a3f94..0000000000 --- a/.github/workflows/cpp-ci-serial-programs-windows-default.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: C++ CI Serial Programs Windows Default -on: [push, pull_request, workflow_dispatch] - -jobs: - build: - uses: ./.github/workflows/cpp-ci-serial-programs-base.yml - with: - os: windows-2025 - compiler: default - upload-build: true - run-tests: true \ No newline at end of file From e3394b9e57e732ada06b202b29dd6a924d828de1 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 12:49:25 -0800 Subject: [PATCH 16/22] only run cmake, without building. for testing. --- .github/workflows/cpp-ci-serial-programs-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index d352f5120d..79e1d37122 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -86,7 +86,7 @@ jobs: mkdir bin cd bin cmake .. -DQT_MAJOR:STRING=6 ${{env.CMAKE_ADDITIONAL_FLAGS}} - cmake --build . --config RelWithDebInfo --parallel 10 + # cmake --build . --config RelWithDebInfo --parallel 10 - name: Prepare upload build if: inputs.upload-build From fdd1ca8753e6bedfd6316f510ca2bde6f4200aa8 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 13:59:55 -0800 Subject: [PATCH 17/22] run autogen, for testing --- .github/workflows/cpp-ci-serial-programs-base.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 79e1d37122..228ab6c0d9 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -87,6 +87,12 @@ jobs: cd bin cmake .. -DQT_MAJOR:STRING=6 ${{env.CMAKE_ADDITIONAL_FLAGS}} # cmake --build . --config RelWithDebInfo --parallel 10 + - name: Generate Qt Autogen Files + if: inputs.run-clang-query + run: | + cd Arduino-Source/SerialPrograms/bin + # This triggers only the moc/uic/rcc generation targets + cmake --build . --target SerialProgramsCommandLine_autogen SerialPrograms_autogen SerialProgramsLib_autogen --parallel 10 - name: Prepare upload build if: inputs.upload-build From 64dc059026a86204a54792ec907bf90d73d88e14 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 14:24:04 -0800 Subject: [PATCH 18/22] test --- .github/workflows/cpp-ci-serial-programs-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 228ab6c0d9..331cfc1440 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -85,7 +85,7 @@ jobs: cd Arduino-Source/SerialPrograms mkdir bin cd bin - cmake .. -DQT_MAJOR:STRING=6 ${{env.CMAKE_ADDITIONAL_FLAGS}} + cmake .. -DQT_MAJOR:STRING=6 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${{env.CMAKE_ADDITIONAL_FLAGS}} # cmake --build . --config RelWithDebInfo --parallel 10 - name: Generate Qt Autogen Files if: inputs.run-clang-query From 2179cf2545b9707f92534f03c23e4ea77461b32d Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 14:26:28 -0800 Subject: [PATCH 19/22] test --- .../workflows/cpp-ci-serial-programs-base.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 331cfc1440..de8ceff7e6 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -87,13 +87,24 @@ jobs: cd bin cmake .. -DQT_MAJOR:STRING=6 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ${{env.CMAKE_ADDITIONAL_FLAGS}} # cmake --build . --config RelWithDebInfo --parallel 10 - - name: Generate Qt Autogen Files + - name: Fake missing Qt generated files if: inputs.run-clang-query run: | - cd Arduino-Source/SerialPrograms/bin - # This triggers only the moc/uic/rcc generation targets - cmake --build . --target SerialProgramsCommandLine_autogen SerialPrograms_autogen SerialProgramsLib_autogen --parallel 10 - + # Create the bin directory if it doesn't exist + mkdir -p Arduino-Source/SerialPrograms/bin + + # Fake the RCC (resource) files + touch Arduino-Source/SerialPrograms/bin/qrc_darkstyle.cpp + + # Fake the MOC/Autogen files for all components + mkdir -p Arduino-Source/SerialPrograms/bin/SerialProgramsLib_autogen + touch Arduino-Source/SerialPrograms/bin/SerialProgramsLib_autogen/mocs_compilation.cpp + + mkdir -p Arduino-Source/SerialPrograms/bin/SerialPrograms_autogen + touch Arduino-Source/SerialPrograms/bin/SerialPrograms_autogen/mocs_compilation.cpp + + mkdir -p Arduino-Source/SerialPrograms/bin/SerialProgramsCommandLine_autogen + touch Arduino-Source/SerialPrograms/bin/SerialProgramsCommandLine_autogen/mocs_compilation.cpp - name: Prepare upload build if: inputs.upload-build shell: bash From b0afc5f36896b37eff0f4c3c290cbdb82b5d19ec Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 15:19:43 -0800 Subject: [PATCH 20/22] fetch origin main --- .github/scripts/clang-query.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index 0817de7815..cedd7e9c88 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -101,6 +101,7 @@ jq -r "$JQ_SCRIPT" \ echo "Generating changed_files.txt from git diff." # git diff with relative paths +git fetch origin main --depth=1 git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" echo "Generating files_to_query.txt, based on changed_files.txt and deps.json." From 9a6f2212bc3e03bd56be269707a674d8e4a25ce0 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 15:38:52 -0800 Subject: [PATCH 21/22] fix fetch of git history --- .github/scripts/clang-query.sh | 1 - .github/workflows/cpp-ci-serial-programs-base.yml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index cedd7e9c88..0817de7815 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -101,7 +101,6 @@ jq -r "$JQ_SCRIPT" \ echo "Generating changed_files.txt from git diff." # git diff with relative paths -git fetch origin main --depth=1 git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt" echo "Generating files_to_query.txt, based on changed_files.txt and deps.json." diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index de8ceff7e6..dac82ba996 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -51,6 +51,8 @@ jobs: with: path: 'Arduino-Source' submodules: 'recursive' + fetch-depth: 0 + filter: blob:none - name: Install Qt uses: jurplel/install-qt-action@v4 From 434a4f3733db221fff93c39c5ea153bb2a28ac69 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Mar 2026 16:00:40 -0800 Subject: [PATCH 22/22] update usage of clang query command --- .github/scripts/clang-query.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/scripts/clang-query.sh b/.github/scripts/clang-query.sh index 0817de7815..5b915e43a4 100644 --- a/.github/scripts/clang-query.sh +++ b/.github/scripts/clang-query.sh @@ -163,6 +163,13 @@ DB_DIR=$(dirname "$DB_PATH") # --extra-arg="-Wno-unused-function" \ # -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt" +CLANG_QUERY=$(command -v clang-query-18 || command -v clang-query) + +if [ -z "$CLANG_QUERY" ]; then + echo "Error: clang-query (or version -18) not found!" + exit 1 +fi + ONLY_CHECK_CHANGED_FILES=true if [ "$ONLY_CHECK_CHANGED_FILES" = "true" ]; then LIST_FILE="$TMP_DIR/files_to_query.txt" @@ -179,7 +186,7 @@ if [ ! -s "$LIST_FILE" ]; then echo "No files found to analyze. Skipping Clang-Query." else xargs -d '\n' -a "$LIST_FILE" --max-args=150 \ - clang-query -p "$DB_DIR" \ + "$CLANG_QUERY" -p "$DB_DIR" \ --extra-arg="-Wno-unused-command-line-argument" \ --extra-arg="-Wno-unused-function" \ -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"