Skip to content

Commit 0daaca6

Browse files
ZacharyPLZacharyPL
authored
Merge branch 'PokemonAutomation:main' into main
2 parents f7931eb + 241b34d commit 0daaca6

1 file changed

Lines changed: 256 additions & 0 deletions

File tree

‎.github/scripts/clang-query.sh‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
# This script will get the diff between the current branch and the main branch.
5+
# It will then find all files that depend on these diff files, by
6+
# using clang-scan-deps to generate the dependency graph.
7+
# The clang-query is then run on these diffed files (and their dependants).
8+
# One weakness of this is that this diff only compares the current branch and main branch,
9+
# and so is mainly helpful for PRs. But doesn't help when committing directly to main, or
10+
# after merging the PR. This isn't insurmountable, but I felt it was getting too messy.
11+
12+
13+
# to get this script to work in Github CI:
14+
# Update cpp-ci-serial-programs-base.yml:
15+
# - Set fetch-depth to 0 to fetch all commits for the diff. with blob:none so you're not downloading all that data
16+
# with:
17+
# path: 'Arduino-Source'
18+
# submodules: 'recursive'
19+
# fetch-depth: 0
20+
# filter: blob:none
21+
#
22+
# - when installing clang-tools, specify a version. e.g. clang-tools-18
23+
# sudo apt install clang-tools-18 libopencv-dev
24+
#
25+
# - under run clang query: set working directory. run this script
26+
# name: Run clang query
27+
# if: inputs.run-clang-query
28+
# working-directory: ./Arduino-Source
29+
# run : bash ./.github/scripts/clang-query.sh
30+
# - this script should be placed within the folder .github/scripts
31+
#
32+
# other files to consider updating
33+
# .gitattributes: *.sh text eol=lf
34+
35+
36+
# to get this script to work locally in Windows:
37+
# Open Git Bash. cd to root of the repo (Arduino-Source). Run the following command:
38+
# sh .github/scripts/clang-query.sh
39+
40+
41+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
42+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
43+
TMP_DIR="$REPO_ROOT/.ci_tmp"
44+
mkdir -p "$TMP_DIR"
45+
46+
# Define the cleanup function
47+
cleanup() {
48+
echo "Cleaning up temporary files..."
49+
rm -rf "$TMP_DIR"
50+
}
51+
52+
# Register the trap: run cleanup on EXIT, plus common signals like INT (Ctrl+C) or TERM
53+
# trap cleanup EXIT INT TERM
54+
55+
56+
cd "$REPO_ROOT"
57+
58+
# find path to compile_commands.json
59+
if [ -f "$REPO_ROOT/SerialPrograms/bin/compile_commands.json" ]; then
60+
DB_PATH="$REPO_ROOT/SerialPrograms/bin/compile_commands.json"
61+
elif [ -f "$REPO_ROOT/build/RelWithDebInfo/compile_commands.json" ]; then
62+
DB_PATH="$REPO_ROOT/build/RelWithDebInfo/compile_commands.json"
63+
else
64+
echo "Error: compile_commands.json not found!"
65+
exit 1
66+
fi
67+
68+
69+
echo "Generating clang-scan-deps experimental-full > deps.json."
70+
71+
# in ubuntu, the command is clang-scan-deps-18. in Windows, it is clang-scan-deps
72+
SCAN_DEPS=$(command -v clang-scan-deps-18 || command -v clang-scan-deps)
73+
74+
# Safety check: Exit if the tool isn't found
75+
if [ -z "$SCAN_DEPS" ]; then
76+
echo "Error: clang-scan-deps (or version -18) not found in PATH."
77+
exit 1
78+
fi
79+
80+
81+
# filter compile_commands.json, to remove .rc files, since clang-scan-deps doesn't recognize this format
82+
jq '[.[] | select(.file | endswith(".rc") | not)]' "$DB_PATH" > "$TMP_DIR/compile_commands_filtered.json"
83+
84+
# get dependency graph
85+
"$SCAN_DEPS" -compilation-database "$TMP_DIR/compile_commands_filtered.json" -format experimental-full > "$TMP_DIR/deps.json"
86+
87+
# normalize slashes
88+
# sed 's|\\\\|/|g' deps.json > normalized_deps.json
89+
sed -i 's|\\\\|/|g' "$TMP_DIR/deps.json"
90+
91+
# check if deps.json has the expected keys
92+
# because we are relying on clang-scan-deps experimental-full, where the names of the keys can change.
93+
TU_KEY="translation-units"
94+
CMD_KEY="commands"
95+
DEPS="file-deps"
96+
INPUT="input-file"
97+
98+
JQ_SCRIPT=$(cat << 'EOF'
99+
# 1. Access the target object
100+
(.[$TU][0][$CMD][0]) as $target
101+
102+
# 2. Define the required keys
103+
104+
| [$DEPS, $INPUT] as $required
105+
106+
| (
107+
if .[$TU] == null then
108+
"Missing: \($TU). Keys found at top-level: \(keys_unsorted)"
109+
elif .[$TU][0] == null then
110+
"Missing: \($TU)[0]"
111+
elif .[$TU][0].[$CMD] == null then
112+
"Missing: \($TU)[0].\($CMD). Keys found from \($TU)[0]: \(.[$TU][0] | keys_unsorted)"
113+
elif $target == null then
114+
"Missing: \($TU)[0].[$CMD][0]"
115+
elif ($required | all(. as $req | $target | has($req)) | not) then
116+
"Missing: One or more required keys \($required). Found: \($target | keys_unsorted)"
117+
# elif (.[$TU][0].[$CMD][0] | keys_unsorted | any(. == [$DEPS] or . == [$INPUT]) | not) then
118+
# "Missing: both \($DEPS) and \($INPUT). Found: \(.[$TU][0][$CMD][0] | keys_unsorted)"
119+
else
120+
"All keys \($required) found in \($TU)[0].\($CMD)[0]"
121+
end
122+
) as $result
123+
124+
| if ($result | type == "string" and startswith("Missing:")) then
125+
("\($result). The keys within the experimental-full format from clang-scan-deps can change over time. Please fix the CI to use the correct keys.") | halt_error
126+
else $result end
127+
EOF
128+
)
129+
130+
echo "Checking keys in deps.json."
131+
jq -r "$JQ_SCRIPT" \
132+
--arg TU "$TU_KEY" \
133+
--arg CMD "$CMD_KEY" \
134+
--arg DEPS "$DEPS" \
135+
--arg INPUT "$INPUT" \
136+
"$TMP_DIR/deps.json"
137+
138+
echo "Generating changed_files.txt from git diff."
139+
140+
# git diff with relative paths
141+
git diff --name-only origin/main...HEAD > "$TMP_DIR/changed_files.txt"
142+
143+
# if [ "$GITHUB_ACTIONS" == "true" ]; then
144+
# if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
145+
# # Compare PR branch to target branch (main)
146+
# BASE="origin/$GITHUB_BASE_REF"
147+
# else
148+
# # Compare current push to previous state
149+
# BASE="${{ github.event.before }}"
150+
# fi
151+
# HEAD="$GITHUB_SHA"
152+
# else
153+
# # LOCAL: Compare local main to the last known state on server
154+
# # (Assuming you are on main and want to see what you just pushed/changed)
155+
# git fetch origin main
156+
# BASE="origin/main"
157+
# HEAD="HEAD"
158+
# fi
159+
160+
echo "Generating files_to_query.txt, based on changed_files.txt and deps.json."
161+
162+
# for each line in changed_files_unix.txt, search deps.json to find all their dependants
163+
jq -r --rawfile mod "$TMP_DIR/changed_files.txt" \
164+
--arg TU "$TU_KEY" \
165+
--arg CMD "$CMD_KEY" \
166+
--arg DEPS "$DEPS" \
167+
--arg INPUT "$INPUT" '
168+
# 1. Clean the list of changed files
169+
($mod | split("\n") | map(select(length > 0))) as $changes |
170+
171+
# 2. Access the translation-units array
172+
[ .[$TU][] | .[$CMD][] |
173+
select(
174+
# 3. Check "file-deps" for matches
175+
.[$DEPS][] | . as $dp |
176+
any($changes[]; . as $c | $dp | endswith($c))
177+
) |
178+
# 4. Get the source file path
179+
.[$INPUT]
180+
] | unique[]
181+
' "$TMP_DIR/deps.json" | tr -d '\r' > "$TMP_DIR/files_to_query.txt"
182+
183+
184+
185+
cat << 'EOF' > "$TMP_DIR/query.txt"
186+
set output dump
187+
match invocation(
188+
isExpansionInFileMatching("SerialPrograms/"),
189+
hasDeclaration(cxxConstructorDecl(ofClass(hasName("std::filesystem::path")))),
190+
hasArgument(0, hasType(asString("std::string")))
191+
)
192+
EOF
193+
194+
echo "Running clang-query."
195+
196+
# files=$(jq -r '.[].file' "$DB_PATH")
197+
DB_DIR=$(dirname "$DB_PATH")
198+
#echo "$files" | xargs --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> output.txt
199+
200+
# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" -- -Wno-unused-command-line-argument >> "$TMP_DIR/output.txt"
201+
202+
# this works
203+
# jq -r '.[].file' "$DB_PATH" | sed 's/\\/\//g' | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
204+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 clang-query -p "$DB_DIR" -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
205+
206+
# also works
207+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | xargs -d '\n' --max-args=150 \
208+
# clang-query -p "$DB_DIR" \
209+
# --extra-arg="-Wno-unused-command-line-argument" \
210+
# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
211+
212+
# also works
213+
# jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' | \
214+
# xargs -d '\n' --max-args=150 \
215+
# clang-query -p "$DB_DIR" \
216+
# --extra-arg="-Wno-unused-command-line-argument" \
217+
# --extra-arg="-Wno-unused-function" \
218+
# -f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
219+
220+
# in ubuntu, the command is clang-query-18. in Windows, it is clang-query
221+
CLANG_QUERY=$(command -v clang-query-18 || command -v clang-query)
222+
223+
if [ -z "$CLANG_QUERY" ]; then
224+
echo "Error: clang-query (or version -18) not found!"
225+
exit 1
226+
fi
227+
228+
ONLY_CHECK_CHANGED_FILES=true
229+
if [ "$ONLY_CHECK_CHANGED_FILES" = "true" ]; then
230+
LIST_FILE="$TMP_DIR/files_to_query.txt"
231+
else # check all files
232+
LIST_FILE="$TMP_DIR/file_list.txt"
233+
jq -r '.[].file' "$DB_PATH" | tr -d '\r' | sed 's|\\|/|g' > "$LIST_FILE"
234+
fi
235+
236+
> "$TMP_DIR/output.txt"
237+
238+
# Run clang-query using the list file
239+
# check if LIST_FILE has any data to analyze
240+
if [ ! -s "$LIST_FILE" ]; then
241+
echo "No files found to analyze. Skipping Clang-Query."
242+
else
243+
xargs -d '\n' -a "$LIST_FILE" --max-args=150 \
244+
"$CLANG_QUERY" -p "$DB_DIR" \
245+
--extra-arg="-Wno-unused-command-line-argument" \
246+
--extra-arg="-Wno-unused-function" \
247+
-f "$TMP_DIR/query.txt" >> "$TMP_DIR/output.txt"
248+
fi
249+
250+
251+
252+
cat "$TMP_DIR/output.txt"
253+
if grep --silent "Match #" "$TMP_DIR/output.txt"; then
254+
echo "::error Forbidden std::filesystem::path construction detected!"
255+
exit 1
256+
fi

0 commit comments

Comments
 (0)