Skip to content

Commit 73cbf19

Browse files
hallvictoriahallvictoria
committed
comparison to baseline
1 parent 96562e4 commit 73cbf19

2 files changed

Lines changed: 189 additions & 0 deletions

File tree

‎eng/scripts/download-baseline-profile.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/usr/bin/env bash
2+
# Download baseline performance profile from dev branch builds
3+
# This script is used in CI to fetch baseline profiles for comparison
4+
5+
set -e
6+
7+
ORGANIZATION="${1:-}"
8+
PROJECT="${2:-}"
9+
PIPELINE_ID="${3:-}"
10+
ARTIFACT_NAME="${4:-}"
11+
OUTPUT_DIR="${5:-baseline_results}"
12+
ACCESS_TOKEN="${SYSTEM_ACCESSTOKEN:-}"
13+
14+
if [ -z "$ORGANIZATION" ] || [ -z "$PROJECT" ] || [ -z "$PIPELINE_ID" ] || [ -z "$ARTIFACT_NAME" ]; then
15+
echo "Usage: $0 <organization> <project> <pipeline_id> <artifact_name> [output_dir]"
16+
echo "Example: $0 myorg myproject 123 performance-profile-python-3.13"
17+
exit 1
18+
fi
19+
20+
echo "Searching for baseline artifacts from dev branch..."
21+
echo "Organization: $ORGANIZATION"
22+
echo "Project: $PROJECT"
23+
echo "Pipeline ID: $PIPELINE_ID"
24+
echo "Artifact: $ARTIFACT_NAME"
25+
26+
mkdir -p "$OUTPUT_DIR"
27+
28+
# Azure DevOps REST API endpoint
29+
API_VERSION="7.1-preview.7"
30+
BUILDS_API="https://dev.azure.com/${ORGANIZATION}/${PROJECT}/_apis/build/builds"
31+
32+
# Get the latest successful build from dev branch
33+
echo "Fetching latest successful build from dev branch..."
34+
35+
BUILD_RESPONSE=$(curl -s -u ":${ACCESS_TOKEN}" \
36+
"${BUILDS_API}?definitions=${PIPELINE_ID}&branchName=refs/heads/dev&statusFilter=completed&resultFilter=succeeded&\$top=1&api-version=${API_VERSION}")
37+
38+
BUILD_ID=$(echo "$BUILD_RESPONSE" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')
39+
40+
if [ -z "$BUILD_ID" ]; then
41+
echo "⚠ No successful builds found on dev branch"
42+
exit 1
43+
fi
44+
45+
echo "✓ Found build ID: $BUILD_ID"
46+
47+
# Get artifacts for this build
48+
echo "Fetching artifacts from build $BUILD_ID..."
49+
50+
ARTIFACTS_API="${BUILDS_API}/${BUILD_ID}/artifacts"
51+
ARTIFACTS_RESPONSE=$(curl -s -u ":${ACCESS_TOKEN}" \
52+
"${ARTIFACTS_API}?api-version=${API_VERSION}")
53+
54+
# Check if artifact exists
55+
if echo "$ARTIFACTS_RESPONSE" | grep -q "\"name\":\"${ARTIFACT_NAME}\""; then
56+
echo "✓ Found artifact: $ARTIFACT_NAME"
57+
58+
# Get download URL
59+
DOWNLOAD_URL=$(echo "$ARTIFACTS_RESPONSE" | grep -A 10 "\"name\":\"${ARTIFACT_NAME}\"" | grep -o '"downloadUrl":"[^"]*"' | cut -d'"' -f4)
60+
61+
if [ -n "$DOWNLOAD_URL" ]; then
62+
echo "Downloading artifact..."
63+
64+
# Download and extract artifact
65+
curl -s -u ":${ACCESS_TOKEN}" -o "${OUTPUT_DIR}/artifact.zip" "$DOWNLOAD_URL"
66+
67+
if [ -f "${OUTPUT_DIR}/artifact.zip" ]; then
68+
unzip -q -o "${OUTPUT_DIR}/artifact.zip" -d "$OUTPUT_DIR"
69+
rm -f "${OUTPUT_DIR}/artifact.zip"
70+
71+
echo "✓ Baseline artifact downloaded to $OUTPUT_DIR"
72+
73+
# List downloaded files
74+
echo "Downloaded files:"
75+
ls -lh "$OUTPUT_DIR"
76+
77+
exit 0
78+
else
79+
echo "✗ Failed to download artifact"
80+
exit 1
81+
fi
82+
else
83+
echo "✗ Could not find download URL for artifact"
84+
exit 1
85+
fi
86+
else
87+
echo "⚠ Artifact not found: $ARTIFACT_NAME"
88+
echo "This might be the first profiling run on dev branch"
89+
exit 1
90+
fi

‎eng/templates/jobs/ci-unit-tests.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,105 @@ jobs:
124124
PYTHON_VERSION: $(PYTHON_VERSION)
125125
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
126126
127+
- bash: |
128+
PY_VER="$(PYTHON_VERSION)"
129+
PY_MINOR="${PY_VER#*.}"
130+
131+
if [ "$PY_MINOR" -ge 13 ] && [ -f profile_results/dispatcher_session_profile.json ]; then
132+
echo "Checking for baseline profile to compare against..."
133+
134+
# Check if this is a PR build
135+
if [ -n "$(System.PullRequest.PullRequestId)" ]; then
136+
echo "=========================================="
137+
echo "PR Build Detected - Running Comparison"
138+
echo "PR #$(System.PullRequest.PullRequestId)"
139+
echo "=========================================="
140+
141+
# Download baseline from dev branch
142+
chmod +x eng/scripts/download-baseline-profile.sh
143+
144+
if eng/scripts/download-baseline-profile.sh \
145+
"$(System.TeamFoundationCollectionUri)" \
146+
"$(System.TeamProject)" \
147+
"$(System.DefinitionId)" \
148+
"performance-profile-python-$(PYTHON_VERSION)" \
149+
"baseline_results"; then
150+
151+
echo ""
152+
echo "✓ Baseline downloaded successfully"
153+
154+
# Find the baseline JSON file (might be in subdirectory)
155+
BASELINE_FILE=$(find baseline_results -name "dispatcher_session_profile.json" | head -1)
156+
157+
if [ -n "$BASELINE_FILE" ]; then
158+
echo "Running performance comparison..."
159+
160+
python tests/unittest_proxy/compare_profiles.py \
161+
"$BASELINE_FILE" \
162+
profile_results/dispatcher_session_profile.json \
163+
--output profile_results/comparison_report.txt \
164+
--show-all
165+
166+
echo ""
167+
echo "╔════════════════════════════════════════════════════════════╗"
168+
echo "║ PERFORMANCE COMPARISON REPORT (vs dev branch) ║"
169+
echo "╚════════════════════════════════════════════════════════════╝"
170+
cat profile_results/comparison_report.txt
171+
echo ""
172+
173+
# Check for regressions and create warnings
174+
REGRESSION_COUNT=$(grep -c "REGRESSION" profile_results/comparison_report.txt || echo "0")
175+
176+
if [ "$REGRESSION_COUNT" -gt 0 ]; then
177+
echo "##vso[task.logissue type=warning]⚠️ $REGRESSION_COUNT performance regression(s) detected!"
178+
echo "##vso[task.logissue type=warning]Review the comparison report in the artifacts for details."
179+
else
180+
echo "✓ No performance regressions detected"
181+
fi
182+
183+
# Upload comparison to build summary
184+
echo "##vso[task.uploadsummary]$(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/profile_results/comparison_report.txt"
185+
else
186+
echo "⚠ Could not find baseline JSON file in downloaded artifact"
187+
fi
188+
else
189+
echo ""
190+
echo "⚠ Could not download baseline - comparison skipped"
191+
echo "This might be the first profiling run, or the baseline is unavailable"
192+
fi
193+
else
194+
echo "=========================================="
195+
echo "Main/Dev Branch Build"
196+
echo "=========================================="
197+
echo "This profile will serve as baseline for future PR comparisons"
198+
echo "Artifact: performance-profile-python-$(PYTHON_VERSION)"
199+
fi
200+
else
201+
echo "Skipping baseline comparison (no profile results or Python < 3.13)"
202+
fi
203+
displayName: "Compare Performance Against Baseline"
204+
condition: succeededOrFailed()
205+
env:
206+
PYTHON_VERSION: $(PYTHON_VERSION)
207+
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
208+
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
209+
210+
- bash: |
211+
PY_VER="$(PYTHON_VERSION)"
212+
PY_MINOR="${PY_VER#*.}"
213+
214+
if [ "$PY_MINOR" -ge 13 ] && [ -d profile_results ]; then
215+
echo "Uploading performance profile artifact..."
216+
echo "##vso[artifact.upload artifactname=performance-profile-python-$(PYTHON_VERSION)]$(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}/profile_results"
217+
echo "Performance profile artifact uploaded successfully"
218+
else
219+
echo "Skipping artifact upload (no profile results)"
220+
fi
221+
displayName: "Upload Performance Profile Artifact"
222+
env:
223+
PYTHON_VERSION: $(PYTHON_VERSION)
224+
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PROJECT_DIRECTORY }}
225+
127226
- bash: |
128227
echo "Cleaning up after tests..."
129228
# Remove build artifacts and caches

0 commit comments

Comments
 (0)