Skip to content

Commit 95a930b

Browse files
yoshifuminakamurayoshifuminakamura
committed
refactor: package qws estimation flow [code:qws] [system:RC_GH200]
1 parent f5fb86c commit 95a930b

3 files changed

Lines changed: 246 additions & 108 deletions

File tree

‎programs/qws/estimate.sh‎

Lines changed: 20 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,28 @@
11
#!/bin/bash
2-
# estimate.sh — Dummy estimation script for qws application
3-
#
4-
# Usage: bash programs/qws/estimate.sh <result_json_path>
5-
# Output: results/estimate_<code>_0.json
6-
#
7-
# This script uses a simple scale-mock model for integration testing.
8-
# Replace the estimation logic section below with a real estimation tool
9-
# when available.
2+
# estimate.sh — Reference package-based estimation entrypoint for qws
103

114
source scripts/estimate_common.sh
12-
13-
# --- Read benchmark result ---
14-
read_values "$1"
15-
16-
# --- Future system benchmark: pass through from the benchmark run ---
17-
est_future_bench_system="$est_system"
18-
est_future_bench_fom="$est_fom"
19-
est_future_bench_nodes="$est_node_count"
20-
est_future_bench_numproc_node="$est_numproc_node"
21-
est_future_bench_timestamp="$est_timestamp"
22-
est_future_bench_uuid="$est_uuid"
23-
24-
# --- Current system: Fugaku — fetch real FOM from result_server ---
25-
est_current_system="Fugaku"
26-
CURRENT_EXP="CASE0" # Use CASE0 for Fugaku baseline
27-
fetch_current_fom "$est_code" "$CURRENT_EXP"
28-
# fetch_current_fom sets est_current_bench_* variables automatically
29-
est_current_target_nodes="$est_node_count"
30-
est_current_scaling_method="measured"
31-
32-
# --- Future system: FugakuNEXT — FOM scaled by 2x (dummy) ---
33-
est_future_system="FugakuNEXT"
34-
est_future_fom=$(awk -v fom="$est_fom" 'BEGIN {printf "%.3f", fom * 2}')
35-
est_future_target_nodes="$est_node_count"
36-
est_future_scaling_method="scale-mock"
37-
38-
# --- Optional metadata blocks ---
39-
est_estimation_id="estimate-${est_code}-${est_uuid:-unknown}"
40-
est_estimation_timestamp=$(date '+%Y-%m-%d %H:%M:%S')
41-
est_method_class="lightweight"
42-
est_detail_level="basic"
43-
est_measurement_json=$(jq -cn \
44-
--arg tool "benchmark-result-only" \
45-
--arg method "fom-plus-breakdown" \
46-
--arg annotation_method "none" \
47-
--arg counter_set "" \
48-
--arg interval_timing_method "measured-or-inherited" \
49-
'{
50-
tool: $tool,
51-
method: $method,
52-
annotation_method: $annotation_method,
53-
counter_set: ($counter_set | if . == "" then null else . end),
54-
interval_timing_method: $interval_timing_method
55-
}')
56-
est_assumptions_json=$(jq -cn \
57-
--arg future_system "FugakuNEXT" \
58-
--arg baseline_system "Fugaku" \
59-
'{
60-
future_system_assumption: $future_system,
61-
baseline_system: $baseline_system,
62-
future_fom_rule: "2x benchmark FOM when no detailed model is available"
63-
}')
64-
est_model_json=$(jq -cn \
65-
--arg type "scaling" \
66-
--arg name "scale-mock" \
67-
--arg version "0.1" \
68-
--arg implementation "programs/qws/estimate.sh" \
69-
'{
70-
type: $type,
71-
name: $name,
72-
version: $version,
73-
implementation: $implementation
74-
}')
75-
est_confidence_json='{"level":"experimental","score":0.30}'
76-
est_notes_json=$(jq -cn \
77-
--arg note "Reference implementation for lightweight estimation in BenchKit." \
78-
'{summary: $note}')
79-
80-
# --- fom_breakdown (extend with bench_time, scaling_method, time per section) ---
81-
# Read raw fom_breakdown from benchmark result
82-
raw_breakdown=$(jq -c '.fom_breakdown // empty' "$1")
83-
84-
if [[ -n "$raw_breakdown" ]]; then
85-
# Future system: scale each section/overlap time by 2x (dummy)
86-
est_future_fom_breakdown=$(echo "$raw_breakdown" | jq -c '{
87-
sections: [.sections[] | {name, bench_time: .time, scaling_method: "scale-mock", time: (.time * 2)}],
88-
overlaps: [(.overlaps // [])[] | {sections, bench_time: .time, scaling_method: "scale-mock", time: (.time * 2)}]
89-
}')
90-
91-
# Current system: measured, so time == bench_time (no scaling)
92-
est_current_fom_breakdown=$(echo "$raw_breakdown" | jq -c '{
93-
sections: [.sections[] | {name, bench_time: .time, scaling_method: "measured", time: .time}],
94-
overlaps: [(.overlaps // [])[] | {sections, bench_time: .time, scaling_method: "measured", time: .time}]
95-
}')
96-
97-
# Compute FOM from breakdown: Σsections.time - Σoverlaps.time
98-
est_future_fom=$(echo "$est_future_fom_breakdown" | jq '([.sections[].time] | add) - ([(.overlaps // [])[].time] | add // 0)' | awk '{printf "%.3f", $1}')
99-
est_current_fom=$(echo "$est_current_fom_breakdown" | jq '([.sections[].time] | add) - ([(.overlaps // [])[].time] | add // 0)' | awk '{printf "%.3f", $1}')
100-
else
101-
est_future_fom_breakdown=""
102-
est_current_fom_breakdown=""
5+
source scripts/estimation/packages/lightweight_fom_scaling.sh
6+
7+
BK_ESTIMATION_PACKAGE="lightweight_fom_scaling"
8+
BK_ESTIMATION_BASELINE_SYSTEM="Fugaku"
9+
BK_ESTIMATION_BASELINE_EXP="CASE0"
10+
BK_ESTIMATION_FUTURE_SYSTEM="FugakuNEXT"
11+
BK_ESTIMATION_SCALE_FACTOR="2"
12+
BK_ESTIMATION_MODEL_NAME="scale-mock"
13+
BK_ESTIMATION_MODEL_VERSION="0.1"
14+
BK_ESTIMATION_INPUT_JSON="$1"
15+
16+
read_values "$BK_ESTIMATION_INPUT_JSON"
17+
18+
if ! bk_estimation_package_check_applicability; then
19+
echo "ERROR: estimation package ${BK_ESTIMATION_PACKAGE} is not applicable for input ${BK_ESTIMATION_INPUT_JSON}" >&2
20+
exit 1
10321
fi
10422

105-
# --- Output ---
23+
bk_estimation_package_run
24+
bk_estimation_package_apply_metadata
25+
10626
mkdir -p results
10727
output_file="results/estimate_${est_code}_0.json"
10828
print_json > "$output_file"

‎scripts/estimate_common.sh‎

Lines changed: 79 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,14 @@ est_estimation_id=""
5959
est_estimation_timestamp=""
6060
est_method_class=""
6161
est_detail_level=""
62+
est_source_result_uuid=""
63+
est_estimation_package=""
64+
est_estimation_package_version=""
6265
est_measurement_json=""
6366
est_assumptions_json=""
6467
est_input_artifacts_json=""
6568
est_model_json=""
69+
est_applicability_json=""
6670
est_confidence_json=""
6771
est_notes_json=""
6872

@@ -119,6 +123,11 @@ read_values() {
119123
exit 1
120124
fi
121125
est_fom="$fom_raw"
126+
127+
# By default, keep the input benchmark result UUID as the source UUID
128+
# for the estimation result. Package-aware estimation scripts may override
129+
# this explicitly when needed.
130+
est_source_result_uuid="$est_uuid"
122131
}
123132

124133
# ---------------------------------------------------------------------------
@@ -132,6 +141,50 @@ performance_ratio() {
132141
'BEGIN { if (fut == 0) printf "0"; else printf "%.3f", cur / fut }'
133142
}
134143

144+
# ---------------------------------------------------------------------------
145+
# bk_estimation_set_package_metadata — Set package-oriented metadata fields
146+
#
147+
# Arguments:
148+
# $1 estimation package name
149+
# $2 estimation package version
150+
# $3 method_class
151+
# $4 detail_level
152+
# ---------------------------------------------------------------------------
153+
bk_estimation_set_package_metadata() {
154+
est_estimation_package="${1:-}"
155+
est_estimation_package_version="${2:-}"
156+
est_method_class="${3:-}"
157+
est_detail_level="${4:-}"
158+
}
159+
160+
# ---------------------------------------------------------------------------
161+
# bk_estimation_set_applicability — Build applicability JSON for Estimate JSON
162+
#
163+
# Arguments:
164+
# $1 status
165+
# $2 fallback_used (optional)
166+
# $3 missing_inputs_json (optional, JSON array)
167+
# $4 required_actions_json (optional, JSON array)
168+
# ---------------------------------------------------------------------------
169+
bk_estimation_set_applicability() {
170+
local status="${1:-}"
171+
local fallback_used="${2:-}"
172+
local missing_inputs_json="${3:-[]}"
173+
local required_actions_json="${4:-[]}"
174+
175+
est_applicability_json=$(jq -cn \
176+
--arg status "$status" \
177+
--arg fallback_used "$fallback_used" \
178+
--argjson missing_inputs "$missing_inputs_json" \
179+
--argjson required_actions "$required_actions_json" \
180+
'{
181+
status: $status
182+
}
183+
+ (if $fallback_used != "" then {fallback_used: $fallback_used} else {} end)
184+
+ (if ($missing_inputs | length) > 0 then {missing_inputs: $missing_inputs} else {} end)
185+
+ (if ($required_actions | length) > 0 then {required_actions: $required_actions} else {} end)')
186+
}
187+
135188
# ---------------------------------------------------------------------------
136189
# fetch_current_fom — Fetch Fugaku FOM from result_server API
137190
#
@@ -213,14 +266,26 @@ print_json() {
213266
fi
214267

215268
local estimate_metadata_block=""
216-
if [[ -n "$est_estimation_id" || -n "$est_estimation_timestamp" || -n "$est_method_class" || -n "$est_detail_level" ]]; then
269+
if [[ -n "$est_estimation_id" || -n "$est_estimation_timestamp" || -n "$est_method_class" || -n "$est_detail_level" || -n "$est_source_result_uuid" || -n "$est_estimation_package" || -n "$est_estimation_package_version" ]]; then
270+
local estimate_metadata_json=""
271+
estimate_metadata_json=$(jq -cn \
272+
--arg estimation_id "$est_estimation_id" \
273+
--arg timestamp "$est_estimation_timestamp" \
274+
--arg method_class "$est_method_class" \
275+
--arg detail_level "$est_detail_level" \
276+
--arg source_result_uuid "$est_source_result_uuid" \
277+
--arg estimation_package "$est_estimation_package" \
278+
--arg estimation_package_version "$est_estimation_package_version" \
279+
'{}
280+
+ (if $estimation_id != "" then {estimation_id: $estimation_id} else {} end)
281+
+ (if $timestamp != "" then {timestamp: $timestamp} else {} end)
282+
+ (if $method_class != "" then {method_class: $method_class} else {} end)
283+
+ (if $detail_level != "" then {detail_level: $detail_level} else {} end)
284+
+ (if $source_result_uuid != "" then {source_result_uuid: $source_result_uuid} else {} end)
285+
+ (if $estimation_package != "" then {estimation_package: $estimation_package} else {} end)
286+
+ (if $estimation_package_version != "" then {estimation_package_version: $estimation_package_version} else {} end)')
217287
estimate_metadata_block=",
218-
\"estimate_metadata\": {
219-
\"estimation_id\": \"${est_estimation_id}\",
220-
\"timestamp\": \"${est_estimation_timestamp}\",
221-
\"method_class\": \"${est_method_class}\",
222-
\"detail_level\": \"${est_detail_level}\"
223-
}"
288+
\"estimate_metadata\": $estimate_metadata_json"
224289
fi
225290

226291
local measurement_block=""
@@ -247,6 +312,12 @@ print_json() {
247312
\"model\": $est_model_json"
248313
fi
249314

315+
local applicability_block=""
316+
if [[ -n "$est_applicability_json" && "$est_applicability_json" != "null" ]]; then
317+
applicability_block=",
318+
\"applicability\": $est_applicability_json"
319+
fi
320+
250321
local confidence_block=""
251322
if [[ -n "$est_confidence_json" && "$est_confidence_json" != "null" ]]; then
252323
confidence_block=",
@@ -291,7 +362,7 @@ print_json() {
291362
"uuid": "$est_future_bench_uuid"
292363
}${future_breakdown_block}
293364
},
294-
"performance_ratio": $ratio${estimate_metadata_block}${measurement_block}${assumptions_block}${input_artifacts_block}${model_block}${confidence_block}${notes_block}
365+
"performance_ratio": $ratio${estimate_metadata_block}${measurement_block}${assumptions_block}${input_artifacts_block}${model_block}${applicability_block}${confidence_block}${notes_block}
295366
}
296367
EOF
297368
}

0 commit comments

Comments
 (0)