Skip to content

Commit dfdafb5

Browse files
christophdbclaude
andcommitted
improve version-compare diff report
Report now includes: - Test result counts for both versions - List of failed test names - Actual snapshot diffs (response differences) - Schema validation failures (if any) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f7da9fc commit dfdafb5

1 file changed

Lines changed: 50 additions & 6 deletions

File tree

.github/workflows/version-compare.yml

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,20 +110,64 @@ jobs:
110110
run: |
111111
mkdir -p /tmp/report
112112
113-
cat > /tmp/report/summary.md << EOF
113+
cat > /tmp/report/summary.md << 'HEADER'
114114
# API Version Comparison Report
115+
HEADER
116+
117+
echo "" >> /tmp/report/summary.md
118+
echo "**Baseline:** ${{ inputs.old_image }}:${{ inputs.old_version }}" >> /tmp/report/summary.md
119+
echo "**Compared:** ${{ inputs.new_image }}:${{ inputs.new_version }}" >> /tmp/report/summary.md
120+
echo "" >> /tmp/report/summary.md
121+
122+
# Count results from baseline
123+
if [ -f /tmp/baseline-output.txt ]; then
124+
baseline_passed=$(grep -oP '\d+ passed' /tmp/baseline-output.txt | head -1 || echo "0 passed")
125+
baseline_failed=$(grep -oP '\d+ failed' /tmp/baseline-output.txt | head -1 || echo "0 failed")
126+
baseline_errors=$(grep -oP '\d+ error' /tmp/baseline-output.txt | head -1 || echo "0 errors")
127+
echo "**Baseline results:** ${baseline_passed}, ${baseline_failed}, ${baseline_errors}" >> /tmp/report/summary.md
128+
fi
115129
116-
**Baseline:** ${{ inputs.old_image }}:${{ inputs.old_version }}
117-
**Compared:** ${{ inputs.new_image }}:${{ inputs.new_version }}
130+
# Count results from comparison
131+
if [ -f /tmp/compare-output.txt ]; then
132+
compare_passed=$(grep -oP '\d+ passed' /tmp/compare-output.txt | head -1 || echo "0 passed")
133+
compare_failed=$(grep -oP '\d+ failed' /tmp/compare-output.txt | head -1 || echo "0 failed")
134+
compare_errors=$(grep -oP '\d+ error' /tmp/compare-output.txt | head -1 || echo "0 errors")
135+
echo "**Comparison results:** ${compare_passed}, ${compare_failed}, ${compare_errors}" >> /tmp/report/summary.md
136+
fi
118137
119-
EOF
138+
echo "" >> /tmp/report/summary.md
120139
121-
if [ -f /tmp/compare-output.txt ] && grep -q "FAILED" /tmp/compare-output.txt; then
140+
if [ -f /tmp/compare-output.txt ] && grep -qE "FAILED|ERROR" /tmp/compare-output.txt; then
122141
echo "## Differences Found" >> /tmp/report/summary.md
123142
echo "" >> /tmp/report/summary.md
143+
144+
# List failed tests
145+
echo "### Failed Tests" >> /tmp/report/summary.md
146+
echo "" >> /tmp/report/summary.md
147+
echo '```' >> /tmp/report/summary.md
148+
grep "FAILED" /tmp/compare-output.txt >> /tmp/report/summary.md || true
124149
echo '```' >> /tmp/report/summary.md
125-
grep -E "FAILED|AssertionError|snapshot" /tmp/compare-output.txt >> /tmp/report/summary.md || true
150+
echo "" >> /tmp/report/summary.md
151+
152+
# Extract snapshot diffs (the actual response differences)
153+
echo "### Response Differences" >> /tmp/report/summary.md
154+
echo "" >> /tmp/report/summary.md
155+
echo '```diff' >> /tmp/report/summary.md
156+
# syrupy outputs diffs between "snapshot" and "received" markers
157+
sed -n '/^____/,/^____\|^====\|^FAILED\|^---.*---$/p' /tmp/compare-output.txt \
158+
| head -500 >> /tmp/report/summary.md || true
126159
echo '```' >> /tmp/report/summary.md
160+
echo "" >> /tmp/report/summary.md
161+
162+
# Extract schema validation failures
163+
if grep -q "CheckFailed" /tmp/compare-output.txt; then
164+
echo "### Schema Validation Failures" >> /tmp/report/summary.md
165+
echo "" >> /tmp/report/summary.md
166+
echo '```' >> /tmp/report/summary.md
167+
grep -A 10 "Response violates schema" /tmp/compare-output.txt \
168+
| head -100 >> /tmp/report/summary.md || true
169+
echo '```' >> /tmp/report/summary.md
170+
fi
127171
else
128172
echo "## No Differences Found" >> /tmp/report/summary.md
129173
echo "" >> /tmp/report/summary.md

0 commit comments

Comments
 (0)