From cf666b8aa7705f2cc9a133f7896c783acb78dd49 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 3 Mar 2026 10:43:29 +0100 Subject: [PATCH 01/24] Add VM scheduler logic --- test/bin/vm_scheduler.sh | 1234 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1234 insertions(+) create mode 100644 test/bin/vm_scheduler.sh diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh new file mode 100644 index 0000000000..f1e75a1a03 --- /dev/null +++ b/test/bin/vm_scheduler.sh @@ -0,0 +1,1234 @@ +#!/bin/bash +# +# Dynamic VM Scheduler for MicroShift Scenario Testing +# +# This scheduler manages VM resources for parallel scenario execution, +# implementing VM reuse for compatible scenarios and queuing when +# host resources are exhausted. +# +# Usage: +# vm_scheduler.sh orchestrate - Schedule all scenarios in directory +# vm_scheduler.sh status - Show current scheduler state +# +# ============================================================================ +# LOCKING MECHANISMS +# ============================================================================ +# +# This scheduler uses directory-based locking with mkdir for atomic operations. +# Lock files are stored in: ${SCHEDULER_STATE_DIR}/locks/.lock +# +# Lock: vm_dispatch +# ----------------- +# Purpose: Protects all VM dispatch operations to prevent race conditions when +# multiple background scenarios complete simultaneously. +# +# Critical sections protected: +# 1. dispatch_dynamic_scenarios(): +# - Iterating queued scenarios +# - Finding compatible free VMs (find_compatible_vm) +# - Checking if resources are available (can_allocate) +# - Assigning VMs or creating new ones +# +# 2. run_scenario_on_vm(): +# - Releasing a VM after scenario completion +# - Searching for the next compatible queued scenario +# - Either reusing the VM for another scenario or destroying it +# +# Implicit Lock: VM Name Generation +# ---------------------------------- +# Function: generate_dynamic_vm_name() +# Mechanism: Uses mkdir "${VM_REGISTRY}/${vm_name}" as an atomic claim operation. +# Purpose: Ensures unique VM names (dynamic-vm-001, dynamic-vm-002, etc.) when +# multiple processes try to create VMs concurrently. +# +# ============================================================================ + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# shellcheck source=test/bin/common.sh +source "${SCRIPTDIR}/common.sh" + +# Scheduler state directory +SCHEDULER_STATE_DIR="${IMAGEDIR}/scheduler-state" +VM_REGISTRY="${SCHEDULER_STATE_DIR}/vms" +SCENARIO_QUEUE="${SCHEDULER_STATE_DIR}/queue" +SCENARIO_STATUS="${SCHEDULER_STATE_DIR}/scenarios" +LOCK_DIR="${SCHEDULER_STATE_DIR}/locks" +SCHEDULER_LOG="${SCHEDULER_STATE_DIR}/scheduler.log" + +# VM pool settings (from common.sh) +VM_POOL_BASENAME="vm-storage" +VM_DISK_BASEDIR="${IMAGEDIR}/${VM_POOL_BASENAME}" + +# Scenario info directory (also from common.sh) +SCENARIO_INFO_DIR="${SCENARIO_INFO_DIR:-${IMAGEDIR}/scenario-info}" + +# Host resource limits (configurable via environment, defaults from system) +# Detect system resources if not specified +_SYSTEM_VCPUS=$(nproc 2>/dev/null || echo 8) +_SYSTEM_MEMORY_KB=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 8388608) +_SYSTEM_MEMORY_MB=$((_SYSTEM_MEMORY_KB / 1024)) + +HOST_TOTAL_VCPUS="${HOST_TOTAL_VCPUS:-${_SYSTEM_VCPUS}}" +HOST_TOTAL_MEMORY="${HOST_TOTAL_MEMORY:-${_SYSTEM_MEMORY_MB}}" + +# System reserved resources (for host OS, hypervisor overhead, etc.) +SYSTEM_RESERVED_VCPUS="${SYSTEM_RESERVED_VCPUS:-2}" +SYSTEM_RESERVED_MEMORY="${SYSTEM_RESERVED_MEMORY:-4096}" + +# Available resources after system reservation +HOST_AVAILABLE_VCPUS=$((HOST_TOTAL_VCPUS - SYSTEM_RESERVED_VCPUS)) +HOST_AVAILABLE_MEMORY=$((HOST_TOTAL_MEMORY - SYSTEM_RESERVED_MEMORY)) + +# Timeouts for VM operations (in seconds) +# VM creation includes kickstart installation +VM_CREATE_TIMEOUT="${VM_CREATE_TIMEOUT:-600}" +# Default test timeout - scenarios can override via test_timeout in requirements +VM_TEST_TIMEOUT="${VM_TEST_TIMEOUT:-3600}" + +# Calculated resource requirements (populated during planning phase) +declare -i STATIC_TOTAL_VCPUS=0 +declare -i STATIC_TOTAL_MEMORY=0 +declare -i DYNAMIC_AVAILABLE_VCPUS=0 +declare -i DYNAMIC_AVAILABLE_MEMORY=0 +declare -i MAX_DYNAMIC_VCPUS=0 # Largest dynamic scenario vCPU requirement +declare -i MAX_DYNAMIC_MEMORY=0 # Largest dynamic scenario memory requirement + +# Current resource usage by dynamic VMs (tracked during execution) +declare -i current_vcpus=0 +declare -i current_memory=0 + + +log() { + local timestamp + timestamp="$(date '+%Y-%m-%d %H:%M:%S')" + echo "[${timestamp}] $*" >> "${SCHEDULER_LOG}" + echo "[${timestamp}] $*" >&2 +} + +init_scheduler() { + mkdir -p "${SCHEDULER_STATE_DIR}" + + log "Initializing scheduler state directory" + mkdir -p "${VM_REGISTRY}" "${SCENARIO_QUEUE}" "${SCENARIO_STATUS}" "${LOCK_DIR}" + + rm -rf "${VM_REGISTRY:?}"/* "${SCENARIO_QUEUE:?}"/* "${SCENARIO_STATUS:?}"/* "${LOCK_DIR:?}"/* + + current_vcpus=0 + current_memory=0 + STATIC_TOTAL_VCPUS=0 + STATIC_TOTAL_MEMORY=0 + DYNAMIC_AVAILABLE_VCPUS=0 + DYNAMIC_AVAILABLE_MEMORY=0 + MAX_DYNAMIC_VCPUS=0 + MAX_DYNAMIC_MEMORY=0 + + log "Scheduler state initialized" + log " Host total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB" + log " System reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB" + log " Available for VMs: vcpus=${HOST_AVAILABLE_VCPUS}, memory=${HOST_AVAILABLE_MEMORY}MB" +} + +# Default VM resources when not specified in launch_vm +DEFAULT_VM_VCPUS=2 +DEFAULT_VM_MEMORY=4096 + +parse_static_scenario_resources() { + local scenario_script="$1" + + local launch_vm_line + launch_vm_line=$(grep -E '^\s*launch_vm' "${scenario_script}" 2>/dev/null | head -1) || true + + local vcpus="${DEFAULT_VM_VCPUS}" + local memory="${DEFAULT_VM_MEMORY}" + + if [ -n "${launch_vm_line}" ]; then + if [[ "${launch_vm_line}" =~ --vm_vcpus[[:space:]]+([0-9]+) ]]; then + vcpus="${BASH_REMATCH[1]}" + fi + if [[ "${launch_vm_line}" =~ --vm_memory[[:space:]]+([0-9]+) ]]; then + memory="${BASH_REMATCH[1]}" + fi + fi + + echo "${vcpus} ${memory}" +} + +calculate_static_requirements() { + local -a static_scenarios=("$@") + + STATIC_TOTAL_VCPUS=0 + STATIC_TOTAL_MEMORY=0 + + for scenario_script in "${static_scenarios[@]}"; do + local resources + resources=$(parse_static_scenario_resources "${scenario_script}") + local vcpus memory + read -r vcpus memory <<< "${resources}" + + STATIC_TOTAL_VCPUS=$((STATIC_TOTAL_VCPUS + vcpus)) + STATIC_TOTAL_MEMORY=$((STATIC_TOTAL_MEMORY + memory)) + + local scenario_name + scenario_name=$(basename "${scenario_script}" .sh) + log " Static ${scenario_name}: vcpus=${vcpus}, memory=${memory}MB" + done + + log "Static total: vcpus=${STATIC_TOTAL_VCPUS}, memory=${STATIC_TOTAL_MEMORY}MB" +} + +calculate_max_dynamic_requirements() { + local -a dynamic_scenarios=("$@") + + MAX_DYNAMIC_VCPUS=0 + MAX_DYNAMIC_MEMORY=0 + + for scenario_script in "${dynamic_scenarios[@]}"; do + local scenario_name + scenario_name=$(basename "${scenario_script}" .sh) + + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + mkdir -p "$(dirname "${req_file}")" + + if ! get_scenario_requirements "${scenario_script}" "${req_file}"; then + log "WARNING: Failed to get requirements for ${scenario_name}" + continue + fi + + local vcpus memory + vcpus=$(get_req_value "${req_file}" "min_vcpus" "${DEFAULT_VM_VCPUS}") + memory=$(get_req_value "${req_file}" "min_memory" "${DEFAULT_VM_MEMORY}") + + log " Dynamic ${scenario_name}: vcpus=${vcpus}, memory=${memory}MB" + + if [ "${vcpus}" -gt "${MAX_DYNAMIC_VCPUS}" ]; then + MAX_DYNAMIC_VCPUS="${vcpus}" + fi + if [ "${memory}" -gt "${MAX_DYNAMIC_MEMORY}" ]; then + MAX_DYNAMIC_MEMORY="${memory}" + fi + done + + log "Max dynamic scenario: vcpus=${MAX_DYNAMIC_VCPUS}, memory=${MAX_DYNAMIC_MEMORY}MB" +} + +acquire_lock() { + local lock_name="$1" + local lock_file="${LOCK_DIR}/${lock_name}.lock" + + while ! mkdir "${lock_file}" 2>/dev/null; do + sleep 1 + done +} + +release_lock() { + local lock_name="$1" + local lock_file="${LOCK_DIR}/${lock_name}.lock" + rmdir "${lock_file}" 2>/dev/null || true +} + +create_static_vms() { + local -a static_scenarios=("$@") + + if [ ${#static_scenarios[@]} -eq 0 ]; then + log "No static VMs to create" + return 0 + fi + + log "Creating ${#static_scenarios[@]} static VMs in parallel" + + local static_create_log="${SCHEDULER_STATE_DIR}/static_create_jobs.txt" + + local progress="" + if [ -t 0 ]; then + progress="--progress" + fi + + if ! parallel \ + ${progress} \ + --results "${SCENARIO_INFO_DIR}/{/.}/boot.log" \ + --joblog "${static_create_log}" \ + --delay 5 \ + bash -x "${SCRIPTDIR}/scenario.sh" create ::: "${static_scenarios[@]}"; then + log "ERROR: Some static VMs failed to create" + cat "${static_create_log}" + return 1 + fi + + cat "${static_create_log}" + log "All static VMs created successfully" + return 0 +} + +run_static_tests() { + local -a static_scenarios=("$@") + + if [ ${#static_scenarios[@]} -eq 0 ]; then + log "No static tests to run" + return 0 + fi + + log "Running ${#static_scenarios[@]} static scenario tests in parallel" + + local static_run_log="${SCHEDULER_STATE_DIR}/static_run_jobs.txt" + + local progress="" + if [ -t 0 ]; then + progress="--progress" + fi + + local result=0 + if ! parallel \ + ${progress} \ + --results "${SCENARIO_INFO_DIR}/{/.}/run.log" \ + --joblog "${static_run_log}" \ + --delay 2 \ + bash -x "${SCRIPTDIR}/scenario.sh" run ::: "${static_scenarios[@]}"; then + result=1 + fi + + cat "${static_run_log}" + return ${result} +} + +generate_dynamic_vm_name() { + local count=1 + local vm_name + while true; do + vm_name="dynamic-vm-$(printf '%03d' "${count}")" + if mkdir "${VM_REGISTRY}/${vm_name}" 2>/dev/null; then + echo "${vm_name}" + return 0 + fi + ((count++)) + #TODO do i even need this? + if [ ${count} -gt 999 ]; then + echo "ERROR: Too many VMs" >&2 + return 1 + fi + done +} + +scenario_is_dynamic() { + local scenario_script="$1" + + if bash -c "source '${scenario_script}' 2>/dev/null && type dynamic_schedule_requirements &>/dev/null"; then + return 0 + fi + return 1 +} + +get_scenario_requirements() { + local scenario_script="$1" + local output_file="$2" + + # shellcheck disable=SC1090 + ( + source "${scenario_script}" + if type dynamic_schedule_requirements &>/dev/null; then + dynamic_schedule_requirements > "${output_file}" + else + # This shouldn't happen if scenario_is_dynamic was checked first + echo "ERROR: dynamic_schedule_requirements not found" >&2 + return 1 + fi + ) +} + +get_req_value() { + local req_file="$1" + local key="$2" + local default="${3:-}" + + local value + value=$(grep "^${key}=" "${req_file}" 2>/dev/null | cut -d= -f2 || true) + if [ -z "${value}" ]; then + echo "${default}" + else + echo "${value}" + fi +} + +boot_image_compatible() { + local vm_image="$1" + local req_image="$2" + + # Exact match always works + [ "${vm_image}" = "${req_image}" ] && return 0 + + # Special images require exact match (cannot substitute) + case "${req_image}" in + *-fips|*-tuned|*-isolated|*-ai-model-serving) + return 1 # Must be exact + ;; + esac + + # Check if VM image is a superset of required image + # Hierarchy: source < source-optionals, brew-lrel < brew-lrel-optional + case "${req_image}" in + *-source) + # source-optionals is superset of source + [[ "${vm_image}" == "${req_image}-optionals" ]] && return 0 + ;; + *-brew-lrel) + # brew-lrel-optional is superset of brew-lrel + [[ "${vm_image}" == "${req_image}-optional" ]] && return 0 + ;; + esac + + # ai-model-serving includes qemu-guest-agent, so can run isolated scenarios + if [[ "${req_image}" == *-isolated ]] && [[ "${vm_image}" == *-ai-model-serving ]]; then + return 0 + fi + + return 1 # Not compatible +} + +vm_satisfies_requirements() { + local vm_name="$1" + local scenario_reqs="$2" + + local vm_state="${VM_REGISTRY}/${vm_name}/state" + [ -f "${vm_state}" ] || return 1 + + # Get VM capabilities + local vm_vcpus vm_memory vm_disksize vm_networks vm_fips vm_boot_image + vm_vcpus=$(get_req_value "${vm_state}" "vcpus" "2") + vm_memory=$(get_req_value "${vm_state}" "memory" "4096") + vm_disksize=$(get_req_value "${vm_state}" "disksize" "20") + vm_networks=$(get_req_value "${vm_state}" "networks" "default") + vm_fips=$(get_req_value "${vm_state}" "fips" "false") + vm_boot_image=$(get_req_value "${vm_state}" "boot_image" "") + + # Get scenario requirements + local req_vcpus req_memory req_disksize req_networks req_fips req_boot_image + req_vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "2") + req_memory=$(get_req_value "${scenario_reqs}" "min_memory" "4096") + req_disksize=$(get_req_value "${scenario_reqs}" "min_disksize" "20") + req_networks=$(get_req_value "${scenario_reqs}" "networks" "default") + req_fips=$(get_req_value "${scenario_reqs}" "fips" "false") + req_boot_image=$(get_req_value "${scenario_reqs}" "boot_image" "") + + # Check minimums + [ "${vm_vcpus}" -ge "${req_vcpus}" ] || return 1 + [ "${vm_memory}" -ge "${req_memory}" ] || return 1 + [ "${vm_disksize}" -ge "${req_disksize}" ] || return 1 + + # Check networks (VM must have all required networks) + for net in ${req_networks//,/ }; do + echo ",${vm_networks}," | grep -q ",${net}," || return 1 + done + + # Check FIPS (if required, VM must have it) + if [ "${req_fips}" = "true" ] && [ "${vm_fips}" != "true" ]; then + return 1 + fi + + # Check boot image compatibility + if ! boot_image_compatible "${vm_boot_image}" "${req_boot_image}"; then + return 1 + fi + + return 0 # VM is compatible +} + +find_compatible_vm() { + local scenario_reqs="$1" + + for vm_dir in "${VM_REGISTRY}"/*; do + [ -d "${vm_dir}" ] || continue + local vm_name + vm_name=$(basename "${vm_dir}") + local vm_state="${vm_dir}/state" + [ -f "${vm_state}" ] || continue + + local vm_status + vm_status=$(get_req_value "${vm_state}" "status" "unknown") + + if [ "${vm_status}" = "available" ]; then + if vm_satisfies_requirements "${vm_name}" "${scenario_reqs}"; then + echo "${vm_name}" + return 0 + fi + fi + done + return 1 +} + +can_allocate() { + local vcpus="$1" + local memory="$2" + + # Recalculate current usage from active dynamic VMs + recalculate_resource_usage + + # Check against dynamic available resources (after system + static reservation) + [ $((current_vcpus + vcpus)) -le ${DYNAMIC_AVAILABLE_VCPUS} ] && \ + [ $((current_memory + memory)) -le ${DYNAMIC_AVAILABLE_MEMORY} ] +} + +recalculate_resource_usage() { + current_vcpus=0 + current_memory=0 + + for vm_dir in "${VM_REGISTRY}"/*; do + [ -d "${vm_dir}" ] || continue + local vm_state="${vm_dir}/state" + [ -f "${vm_state}" ] || continue + + local vm_status + vm_status=$(get_req_value "${vm_state}" "status" "unknown") + + #TODO do i have a "creating" status? if thats the case i need to take it too + if [ "${vm_status}" = "in_use" ] || [ "${vm_status}" = "available" ]; then + local vm_vcpus vm_memory + vm_vcpus=$(get_req_value "${vm_state}" "vcpus" "0") + vm_memory=$(get_req_value "${vm_state}" "memory" "0") + current_vcpus=$((current_vcpus + vm_vcpus)) + current_memory=$((current_memory + vm_memory)) + fi + done +} + +register_vm() { + local vm_name="$1" + local scenario_reqs="$2" + local scenario_name="$3" + + local vm_dir="${VM_REGISTRY}/${vm_name}" + mkdir -p "${vm_dir}" + + # Copy requirements as VM state with status + cp "${scenario_reqs}" "${vm_dir}/state" + + # Add vcpus/memory/disksize from min_* values and set status + #TODO this should use the defaults, right? + local vcpus memory disksize networks fips boot_image + vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "2") + memory=$(get_req_value "${scenario_reqs}" "min_memory" "4096") + disksize=$(get_req_value "${scenario_reqs}" "min_disksize" "20") + networks=$(get_req_value "${scenario_reqs}" "networks" "default") + fips=$(get_req_value "${scenario_reqs}" "fips" "false") + boot_image=$(get_req_value "${scenario_reqs}" "boot_image" "") + + cat > "${vm_dir}/state" <> "${vm_dir}/scenario_history.log" + + log "Registered VM ${vm_name} for scenario ${scenario_name}" +} + +assign_vm_to_scenario() { + local vm_name="$1" + local scenario_name="$2" + + local vm_dir="${VM_REGISTRY}/${vm_name}" + local vm_state="${vm_dir}/state" + + # Update status and current scenario + sed -i "s/^status=.*/status=in_use/" "${vm_state}" + sed -i "s/^current_scenario=.*/current_scenario=${scenario_name}/" "${vm_state}" + + # Record in scenario history + echo "$(date -Iseconds) START ${scenario_name}" >> "${vm_dir}/scenario_history.log" + + # Mark the scenario as running in the queue (prevents duplicate dispatch) + local queue_file="${SCENARIO_QUEUE}/${scenario_name}" + if [ -f "${queue_file}" ]; then + sed -i "s/^status=.*/status=running/" "${queue_file}" + echo "started_at=$(date -Iseconds)" >> "${queue_file}" + fi + + # Create assignment file for scenario + mkdir -p "${SCENARIO_STATUS}/${scenario_name}" + echo "${vm_name}" > "${SCENARIO_STATUS}/${scenario_name}/vm_assignment" + echo "true" > "${SCENARIO_STATUS}/${scenario_name}/vm_reused" + + log "Assigned VM ${vm_name} to scenario ${scenario_name} (reuse)" +} + +release_vm() { + local vm_name="$1" + local scenario_name="$2" + local result="$3" + + local vm_dir="${VM_REGISTRY}/${vm_name}" + local vm_state="${vm_dir}/state" + + # Record in scenario history + echo "$(date -Iseconds) END ${scenario_name} ${result}" >> "${vm_dir}/scenario_history.log" + + # Update status to available + sed -i "s/^status=.*/status=available/" "${vm_state}" + sed -i "s/^current_scenario=.*/current_scenario=/" "${vm_state}" + + log "Released VM ${vm_name} from scenario ${scenario_name} (${result})" +} + +destroy_vm() { + local vm_name="$1" + local reason="${2:-no compatible scenarios}" + + local vm_dir="${VM_REGISTRY}/${vm_name}" + + # Record destruction in history + echo "$(date -Iseconds) DESTROYED (${reason})" >> "${vm_dir}/scenario_history.log" + + # Mark as destroyed + sed -i "s/^status=.*/status=destroyed/" "${vm_dir}/state" 2>/dev/null || \ + echo "status=destroyed" >> "${vm_dir}/state" + + log "Destroying libvirt VM ${vm_name}: ${reason}" + + # Actually destroy the libvirt VM and clean up resources + if sudo virsh dumpxml "${vm_name}" &>/dev/null; then + if ! sudo virsh dominfo "${vm_name}" 2>/dev/null | grep '^State' | grep -q 'shut off'; then + sudo virsh destroy --graceful "${vm_name}" 2>/dev/null || true + fi + if ! sudo virsh dominfo "${vm_name}" 2>/dev/null | grep '^State' | grep -q 'shut off'; then + sudo virsh destroy "${vm_name}" 2>/dev/null || true + fi + sudo virsh undefine --nvram "${vm_name}" 2>/dev/null || true + fi + + # Clean up storage pool + local vm_pool_name="${VM_POOL_BASENAME}-${vm_name}" + if sudo virsh pool-info "${vm_pool_name}" &>/dev/null; then + sudo virsh pool-destroy "${vm_pool_name}" 2>/dev/null || true + sudo virsh pool-undefine "${vm_pool_name}" 2>/dev/null || true + fi + + # Remove pool directory + rm -rf "${VM_DISK_BASEDIR:?}/${vm_pool_name}" 2>/dev/null || true + + log "Destroyed VM ${vm_name}" +} + +queue_scenario() { + local scenario_script="$1" + local scenario_name="$2" + local req_file="$3" + + local queue_file="${SCENARIO_QUEUE}/${scenario_name}" + cat > "${queue_file}" <> "${queue_file}" +} + +mark_scenario_completed() { + local scenario_name="$1" + local result="$2" + local queue_file="${SCENARIO_QUEUE}/${scenario_name}" + + sed -i "s/^status=.*/status=completed/" "${queue_file}" + echo "completed_at=$(date -Iseconds)" >> "${queue_file}" + echo "result=${result}" >> "${queue_file}" +} + +has_pending_scenarios() { + for queue_file in "${SCENARIO_QUEUE}"/*; do + [ -f "${queue_file}" ] || continue + local status + status=$(get_req_value "${queue_file}" "status" "") + if [ "${status}" = "queued" ]; then + return 0 + fi + done + return 1 +} + +run_scenario_on_vm() { + local scenario_script="$1" + local scenario_name="$2" + local vm_name="$3" + local is_new_vm="$4" + + log "Starting scenario ${scenario_name} on VM ${vm_name} (new_vm=${is_new_vm})" + + mark_scenario_running "${scenario_name}" + + # Set up environment for scheduler-aware scenario execution + export SCHEDULER_ENABLED=true + export SCHEDULER_SCENARIO_NAME="${scenario_name}" + export SCHEDULER_STATE_DIR="${SCHEDULER_STATE_DIR}" + + # Always pass the VM name - scheduler controls naming for dynamic scenarios + export SCHEDULER_VM_NAME="${vm_name}" + if [ "${is_new_vm}" = "false" ]; then + export SCHEDULER_IS_NEW_VM="false" + else + export SCHEDULER_IS_NEW_VM="true" + fi + + local result="SUCCESS" + local exit_code=0 + + # Set up log directory for scenario execution + local scenario_log_dir="${SCENARIO_INFO_DIR}/${scenario_name}" + mkdir -p "${scenario_log_dir}" + + # Get scenario-specific timeouts if specified, otherwise use defaults + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + local create_timeout="${VM_CREATE_TIMEOUT}" + local test_timeout="${VM_TEST_TIMEOUT}" + if [ -f "${req_file}" ]; then + local custom_create_timeout + custom_create_timeout=$(get_req_value "${req_file}" "create_timeout" "") + if [ -n "${custom_create_timeout}" ]; then + create_timeout="${custom_create_timeout}" + fi + + local custom_test_timeout + custom_test_timeout=$(get_req_value "${req_file}" "test_timeout" "") + if [ -n "${custom_test_timeout}" ]; then + test_timeout="${custom_test_timeout}" + fi + fi + + # Phase 1: Create VM (only for new VMs) + if [ "${is_new_vm}" = "true" ]; then + local create_log="${scenario_log_dir}/create.log" + local vm_dir="${VM_REGISTRY}/${vm_name}" + mkdir -p "${vm_dir}" + ln -sf "${create_log}" "${vm_dir}/creation_log" + + log "Creating VM ${vm_name} (timeout: ${create_timeout}s) - logging to ${create_log}. Linking to ${vm_dir}/creation_log" + + local create_exit=0 + timeout --signal=TERM --kill-after=60 "${create_timeout}" \ + bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${create_log}" || create_exit=$? + + if [ ${create_exit} -ne 0 ]; then + result="FAILED" + exit_code=1 + if [ ${create_exit} -eq 124 ]; then + log "VM creation TIMED OUT for ${scenario_name} after ${create_timeout}s - see ${create_log}" + else + log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${create_log}" + fi + fi + fi + + # Phase 2: Run tests (only if creation succeeded or VM was reused) + if [ "${exit_code}" -eq 0 ]; then + local run_log="${scenario_log_dir}/run.log" + log "Running tests for ${scenario_name} (timeout: ${test_timeout}s) - logging to ${run_log}" + + local run_exit=0 + timeout --signal=TERM --kill-after=60 "${test_timeout}" \ + bash -x "${SCRIPTDIR}/scenario.sh" run "${scenario_script}" &> "${run_log}" || run_exit=$? + + if [ ${run_exit} -ne 0 ]; then + result="FAILED" + exit_code=1 + if [ ${run_exit} -eq 124 ]; then + log "Tests TIMED OUT for ${scenario_name} after ${test_timeout}s - see ${run_log}" + else + log "Tests failed for ${scenario_name} (exit ${run_exit}) - see ${run_log}" + fi + fi + fi + + mark_scenario_completed "${scenario_name}" "${result}" + + # Handle VM after scenario completion + acquire_lock "vm_dispatch" + + release_vm "${vm_name}" "${scenario_name}" "${result}" + + # If tests failed, destroy the VM immediately as there are no guarantees to its state + if [ "${result}" = "FAILED" ]; then + destroy_vm "${vm_name}" "test failure" + release_lock "vm_dispatch" + return ${exit_code} + fi + + # Check if any queued scenario can use this VM + local next_scenario="" + for queued in $(get_queued_scenarios); do + local queue_file="${SCENARIO_QUEUE}/${queued}" + local req_file + req_file=$(get_req_value "${queue_file}" "requirements" "") + + if [ -n "${req_file}" ] && [ -f "${req_file}" ]; then + if vm_satisfies_requirements "${vm_name}" "${req_file}"; then + next_scenario="${queued}" + break + fi + fi + done + + if [ -n "${next_scenario}" ]; then + # Reuse VM for next compatible scenario + local queue_file="${SCENARIO_QUEUE}/${next_scenario}" + local next_script + next_script=$(get_req_value "${queue_file}" "script" "") + + assign_vm_to_scenario "${vm_name}" "${next_scenario}" + + release_lock "vm_dispatch" + + # Run the next scenario (recursive call) + run_scenario_on_vm "${next_script}" "${next_scenario}" "${vm_name}" "false" + else + # No compatible scenario waiting - destroy VM + destroy_vm "${vm_name}" + release_lock "vm_dispatch" + fi + + return ${exit_code} +} + +create_vm_for_scenario() { + local scenario_script="$1" + local scenario_name="$2" + local req_file="$3" + + local vm_name + vm_name=$(generate_dynamic_vm_name) + + register_vm "${vm_name}" "${req_file}" "${scenario_name}" + + # Create assignment file for scenario + mkdir -p "${SCENARIO_STATUS}/${scenario_name}" + echo "${vm_name}" > "${SCENARIO_STATUS}/${scenario_name}/vm_assignment" + echo "false" > "${SCENARIO_STATUS}/${scenario_name}/vm_reused" + + echo "${vm_name}" +} + +dispatch_dynamic_scenarios() { + local -a pids=() + local -a pid_scenarios=() + local overall_result=0 + + while has_pending_scenarios || [ ${#pids[@]} -gt 0 ]; do + # Try to dispatch queued scenarios + acquire_lock "vm_dispatch" + + for scenario_name in $(get_queued_scenarios); do + local queue_file="${SCENARIO_QUEUE}/${scenario_name}" + local scenario_script req_file + scenario_script=$(get_req_value "${queue_file}" "script" "") + req_file=$(get_req_value "${queue_file}" "requirements" "") + + if [ -z "${scenario_script}" ] || [ -z "${req_file}" ]; then + continue + fi + + local min_vcpus min_memory + min_vcpus=$(get_req_value "${req_file}" "min_vcpus" "2") + min_memory=$(get_req_value "${req_file}" "min_memory" "4096") + + # Try to find compatible free VM + local vm_name="" + if vm_name=$(find_compatible_vm "${req_file}"); then + # Reuse existing VM + assign_vm_to_scenario "${vm_name}" "${scenario_name}" + log "Reusing VM ${vm_name} for ${scenario_name}" + + # Start scenario in background + run_scenario_on_vm "${scenario_script}" "${scenario_name}" "${vm_name}" "false" & + local pid=$! + pids+=("${pid}") + pid_scenarios+=("${scenario_name}") + + elif can_allocate "${min_vcpus}" "${min_memory}"; then + # Create new VM + vm_name=$(create_vm_for_scenario "${scenario_script}" "${scenario_name}" "${req_file}") + log "Creating new VM ${vm_name} for ${scenario_name}" + + # Start scenario in background + run_scenario_on_vm "${scenario_script}" "${scenario_name}" "${vm_name}" "true" & + local pid=$! + pids+=("${pid}") + pid_scenarios+=("${scenario_name}") + + else + log "Resources exhausted, keeping ${scenario_name} queued (need: vcpus=${min_vcpus}, mem=${min_memory})" + fi + done + + release_lock "vm_dispatch" + + # If we have running jobs, wait for any one to finish + if [ ${#pids[@]} -gt 0 ]; then + # Wait for any background job to complete (bash 4.3+) + # Returns immediately when any child exits + local finished_pid="" + wait -n -p finished_pid 2>/dev/null || true + + # Find which scenario finished using the captured PID + local -a new_pids=() + local -a new_pid_scenarios=() + for i in "${!pids[@]}"; do + local pid="${pids[${i}]}" + local scenario="${pid_scenarios[${i}]}" + if [ "${pid}" = "${finished_pid}" ]; then + # This is the process that finished - collect exit status + wait "${pid}" || overall_result=1 + log "Scenario ${scenario} finished (pid ${pid})" + else + new_pids+=("${pid}") + new_pid_scenarios+=("${scenario}") + fi + done + pids=("${new_pids[@]+"${new_pids[@]}"}") + pid_scenarios=("${new_pid_scenarios[@]+"${new_pid_scenarios[@]}"}") + fi + done + + return ${overall_result} +} + +orchestrate() { + local scenario_dir="$1" + + if [ ! -d "${scenario_dir}" ]; then + echo "ERROR: Scenario directory not found: ${scenario_dir}" >&2 + exit 1 + fi + + log "=== PHASE 0: Preparing ===" + init_scheduler + + log "=== PHASE 1: Classifying scenarios ===" + local -a dynamic_scenarios=() + local -a static_scenarios=() + + for scenario_script in "${scenario_dir}"/*.sh; do + [ -f "${scenario_script}" ] || continue + + if scenario_is_dynamic "${scenario_script}"; then + dynamic_scenarios+=("${scenario_script}") + log " Dynamic: $(basename "${scenario_script}")" + else + static_scenarios+=("${scenario_script}") + log " Static: $(basename "${scenario_script}")" + fi + done + + log "Found ${#dynamic_scenarios[@]} dynamic scenarios and ${#static_scenarios[@]} static scenarios" + + log "=== PHASE 2: Resource Planning and Validation ===" + + if [ ${#static_scenarios[@]} -gt 0 ]; then + log "Calculating static scenario requirements..." + calculate_static_requirements "${static_scenarios[@]}" + else + log "No static scenarios" + STATIC_TOTAL_VCPUS=0 + STATIC_TOTAL_MEMORY=0 + fi + + if [ ${#dynamic_scenarios[@]} -gt 0 ]; then + log "Calculating dynamic scenario requirements..." + calculate_max_dynamic_requirements "${dynamic_scenarios[@]}" + else + log "No dynamic scenarios" + MAX_DYNAMIC_VCPUS=0 + MAX_DYNAMIC_MEMORY=0 + fi + + DYNAMIC_AVAILABLE_VCPUS=$((HOST_AVAILABLE_VCPUS - STATIC_TOTAL_VCPUS)) + DYNAMIC_AVAILABLE_MEMORY=$((HOST_AVAILABLE_MEMORY - STATIC_TOTAL_MEMORY)) + + log "" + log "=== Resource Summary ===" + log " Host total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB" + log " System reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB" + log " Available for VMs: vcpus=${HOST_AVAILABLE_VCPUS}, memory=${HOST_AVAILABLE_MEMORY}MB" + log " Static requires: vcpus=${STATIC_TOTAL_VCPUS}, memory=${STATIC_TOTAL_MEMORY}MB" + log " Dynamic available: vcpus=${DYNAMIC_AVAILABLE_VCPUS}, memory=${DYNAMIC_AVAILABLE_MEMORY}MB" + log " Max dynamic needs: vcpus=${MAX_DYNAMIC_VCPUS}, memory=${MAX_DYNAMIC_MEMORY}MB" + log "" + + if [ ${STATIC_TOTAL_VCPUS} -gt ${HOST_AVAILABLE_VCPUS} ]; then + log "ERROR: Static scenarios require more vCPUs than available" + log " Required: ${STATIC_TOTAL_VCPUS}, Available: ${HOST_AVAILABLE_VCPUS}" + return 1 + fi + if [ ${STATIC_TOTAL_MEMORY} -gt ${HOST_AVAILABLE_MEMORY} ]; then + log "ERROR: Static scenarios require more memory than available" + log " Required: ${STATIC_TOTAL_MEMORY}MB, Available: ${HOST_AVAILABLE_MEMORY}MB" + return 1 + fi + + if [ ${#dynamic_scenarios[@]} -gt 0 ]; then + if [ ${MAX_DYNAMIC_VCPUS} -gt ${DYNAMIC_AVAILABLE_VCPUS} ]; then + log "ERROR: Largest dynamic scenario requires more vCPUs than available after static allocation" + log " Required: ${MAX_DYNAMIC_VCPUS}, Available: ${DYNAMIC_AVAILABLE_VCPUS}" + return 1 + fi + if [ ${MAX_DYNAMIC_MEMORY} -gt ${DYNAMIC_AVAILABLE_MEMORY} ]; then + log "ERROR: Largest dynamic scenario requires more memory than available after static allocation" + log " Required: ${MAX_DYNAMIC_MEMORY}MB, Available: ${DYNAMIC_AVAILABLE_MEMORY}MB" + return 1 + fi + fi + + log "Resource validation PASSED - all scenarios can run" + log "" + + if [ ${#static_scenarios[@]} -gt 0 ]; then + log "=== PHASE 3: Creating static VMs ===" + if ! create_static_vms "${static_scenarios[@]}"; then + log "ERROR: Failed to create static VMs, aborting" + return 1 + fi + fi + + for scenario_script in "${dynamic_scenarios[@]}"; do + local scenario_name + scenario_name=$(basename "${scenario_script}" .sh) + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + + queue_scenario "${scenario_script}" "${scenario_name}" "${req_file}" + done + + log "=== PHASE 4: Running tests ===" + + local overall_result=0 + local static_pid="" + local dynamic_result=0 + local static_result=0 + + if [ ${#static_scenarios[@]} -gt 0 ]; then + log "Starting static scenario tests in background" + run_static_tests "${static_scenarios[@]}" & + static_pid=$! + log "Static tests running in background (pid ${static_pid})" + fi + + if [ ${#dynamic_scenarios[@]} -gt 0 ]; then + log "Dispatching dynamic scenarios" + if ! dispatch_dynamic_scenarios; then + dynamic_result=1 + fi + fi + + if [ -n "${static_pid}" ]; then + log "Waiting for static tests to complete (pid ${static_pid})" + if ! wait "${static_pid}"; then + static_result=1 + fi + log "Static tests completed" + fi + + if [ ${dynamic_result} -ne 0 ] || [ ${static_result} -ne 0 ]; then + overall_result=1 + fi + + log "Orchestration complete (dynamic=${dynamic_result}, static=${static_result})" + + # Print final summary + show_status + + return ${overall_result} +} + +show_status() { + echo "============================================================================" + echo " SCHEDULER STATUS" + echo "============================================================================" + echo "" + echo "State directory: ${SCHEDULER_STATE_DIR}" + echo "" + + # --- Scenario Results --- + local total_scenarios=0 + local passed_scenarios=0 + local failed_scenarios=0 + + for queue_file in "${SCENARIO_QUEUE}"/*; do + [ -f "${queue_file}" ] || continue + ((total_scenarios++)) + + local result + result=$(get_req_value "${queue_file}" "result" "") + if [ "${result}" = "SUCCESS" ]; then + ((passed_scenarios++)) + elif [ "${result}" = "FAILED" ]; then + ((failed_scenarios++)) + fi + done + + # --- VM Metrics --- + local vms_created=0 + local vm_reuses=0 + local max_runs_per_vm=0 + + for scenario_dir in "${SCENARIO_STATUS}"/*; do + [ -d "${scenario_dir}" ] || continue + local reused_file="${scenario_dir}/vm_reused" + if [ -f "${reused_file}" ]; then + local reused + reused=$(cat "${reused_file}") + if [ "${reused}" = "true" ]; then + ((vm_reuses++)) + else + ((vms_created++)) + fi + fi + done + + for vm_dir in "${VM_REGISTRY}"/*; do + [ -d "${vm_dir}" ] || continue + local history_file="${vm_dir}/scenario_history.log" + if [ -f "${history_file}" ]; then + local runs + runs=$(grep -c "^.* START " "${history_file}" 2>/dev/null || echo 0) + if [ "${runs}" -gt "${max_runs_per_vm}" ]; then + max_runs_per_vm="${runs}" + fi + fi + done + + local reuse_rate=0 + if [ ${total_scenarios} -gt 0 ]; then + reuse_rate=$((vm_reuses * 100 / total_scenarios)) + fi + + echo "=== Scenario Results ===" + echo " Total scenarios: ${total_scenarios}" + echo " Passed: ${passed_scenarios}" + echo " Failed: ${failed_scenarios}" + echo "" + + echo "=== VM Efficiency ===" + echo " VMs created: ${vms_created}" + echo " VM reuses: ${vm_reuses}" + echo " Reuse rate: ${reuse_rate}%" + echo " Max scenarios per VM: ${max_runs_per_vm}" + echo "" + + echo "=== Resource Configuration ===" + echo " Host total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB" + echo " System reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB" + echo " Available for VMs: vcpus=${HOST_AVAILABLE_VCPUS}, memory=${HOST_AVAILABLE_MEMORY}MB" + echo "" + + echo "=== Resource Allocation ===" + echo " Static requires: vcpus=${STATIC_TOTAL_VCPUS}, memory=${STATIC_TOTAL_MEMORY}MB" + echo " Dynamic available: vcpus=${DYNAMIC_AVAILABLE_VCPUS}, memory=${DYNAMIC_AVAILABLE_MEMORY}MB" + echo " Max dynamic needs: vcpus=${MAX_DYNAMIC_VCPUS}, memory=${MAX_DYNAMIC_MEMORY}MB" + echo "" + + echo "=== Current Dynamic Usage ===" + recalculate_resource_usage + echo " Dynamic VMs using: vcpus=${current_vcpus}, memory=${current_memory}MB" + echo "" + + echo "=== VMs ===" + for vm_dir in "${VM_REGISTRY}"/*; do + [ -d "${vm_dir}" ] || continue + local vm_name + vm_name=$(basename "${vm_dir}") + local vm_state="${vm_dir}/state" + if [ -f "${vm_state}" ]; then + echo "${vm_name}:" + sed 's/^/ /' "${vm_state}" + fi + done + echo "" + + echo "=== Scenarios ===" + for queue_file in "${SCENARIO_QUEUE}"/*; do + [ -f "${queue_file}" ] || continue + local scenario_name + scenario_name=$(basename "${queue_file}") + echo "${scenario_name}:" + sed 's/^/ /' "${queue_file}" + done + echo "" + echo "============================================================================" +} + +usage() { + cat < [args] + +Commands: + orchestrate Schedule and run all scenarios in directory + status Show current scheduler state + +Environment Variables: + HOST_TOTAL_VCPUS Total host vCPUs (default: 48) + HOST_TOTAL_MEMORY Total host memory in MB (default: 98304) + SYSTEM_RESERVED_VCPUS vCPUs reserved for host OS (default: 2) + SYSTEM_RESERVED_MEMORY Memory reserved for host OS in MB (default: 4096) + SCHEDULER_ENABLED Enable scheduler mode (default: false) + +Execution Flow: + 1. Classify scenarios (dynamic vs static) + 2. Calculate resource requirements for ALL scenarios + - Parse static scenarios for VM requirements + - Parse dynamic scenarios for VM requirements + 3. Validate resources BEFORE creating any VMs: + - System reserved + static + max_dynamic <= host_total + 4. Create static VMs (in parallel) + 5. Run static tests and dynamic scenarios concurrently +EOF +} + +if [ $# -lt 1 ]; then + usage + exit 1 +fi + +command="$1" +shift + +case "${command}" in + orchestrate) + if [ $# -lt 1 ]; then + echo "ERROR: orchestrate requires scenario directory argument" >&2 + usage + exit 1 + fi + orchestrate "$1" + ;; + status) + show_status + ;; + *) + echo "ERROR: Unknown command: ${command}" >&2 + usage + exit 1 + ;; +esac From 5bb4fb1ea559cf27d25ca905168f335eb112fe36 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 3 Mar 2026 11:30:55 +0100 Subject: [PATCH 02/24] Add bin/scenario.sh changes to support dynamic scenarios --- test/bin/scenario.sh | 131 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 8f16e22d8f..8201999623 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -33,6 +33,12 @@ TEST_EXCLUDES="none" # may be overridden in scenario file TEST_EXECUTION_TIMEOUT="${TEST_EXECUTION_TIMEOUT:-30m}" # may be overriden in scenario file or CI config SUBSCRIPTION_MANAGER_PLUGIN="${SUBSCRIPTION_MANAGER_PLUGIN:-${SCRIPTDIR}/subscription_manager_register.sh}" # may be overridden in global settings file RUN_HOST_OVERRIDE="" # target any given VM for running scenarios +# Scheduler integration +SCHEDULER_ENABLED="${SCHEDULER_ENABLED:-false}" +SCHEDULER_VM_NAME="${SCHEDULER_VM_NAME:-}" +SCHEDULER_SCENARIO_NAME="${SCHEDULER_SCENARIO_NAME:-}" +SCHEDULER_IS_NEW_VM="${SCHEDULER_IS_NEW_VM:-true}" +SCHEDULER_STATE_DIR="${SCHEDULER_STATE_DIR:-${IMAGEDIR}/scheduler-state}" declare -i TESTCASES=0 declare -i FAILURES=0 @@ -41,6 +47,18 @@ TIMESTAMP="$(date --iso-8601=ns)" full_vm_name() { local -r base="${1}" + # In scheduler mode, use the scheduler-assigned VM name (e.g., dynamic-vm-001) + # This keeps VM names neutral and not tied to any specific scenario + if [ "${SCHEDULER_ENABLED}" = "true" ] && [ -n "${SCHEDULER_VM_NAME}" ]; then + if [ "${base}" = "host1" ]; then + echo "${SCHEDULER_VM_NAME}" + else + # For multiple VMs in a scenario, append the base name + echo "${SCHEDULER_VM_NAME}-${base}" + fi + return + fi + local -r type="$(get_scenario_type_from_path "${SCENARIO_SCRIPT}")" # Add a type suffix to the name to allow running scenarios from different # build types on the same hypervisor @@ -688,6 +706,96 @@ EOF EOF } +# Check if a scheduler-assigned VM exists for this scenario +get_scheduler_assigned_vm() { + if [ "${SCHEDULER_ENABLED}" != "true" ]; then + return 1 + fi + + if [ -n "${SCHEDULER_VM_NAME}" ]; then + echo "${SCHEDULER_VM_NAME}" + return 0 + fi + + # Check for assignment file + local assigned_vm="${SCHEDULER_STATE_DIR}/scenarios/${SCENARIO}/vm_assignment" + if [ -f "${assigned_vm}" ]; then + cat "${assigned_vm}" + return 0 + fi + + return 1 +} + +# Check if we should reuse an existing VM (scheduler mode) +should_reuse_vm() { + if [ "${SCHEDULER_ENABLED}" != "true" ]; then + return 1 + fi + + # If the scheduler assigned a VM and it's not a new VM, reuse it + if [ "${SCHEDULER_IS_NEW_VM}" = "false" ]; then + return 0 + fi + + local reused_file="${SCHEDULER_STATE_DIR}/scenarios/${SCENARIO}/vm_reused" + if [ -f "${reused_file}" ] && [ "$(cat "${reused_file}")" = "true" ]; then + return 0 + fi + + return 1 +} + +# Set up VM properties from an existing VM (for scheduler reuse) +setup_vm_properties_from_existing() { + local vmname="$1" + local scheduler_vm="$2" + + echo "Reusing scheduler-assigned VM: ${scheduler_vm}" + + # The VM already exists, so we need to set up properties for this scenario + local -r full_vmname="${scheduler_vm}" + + # Ensure the VM is running + if sudo virsh dominfo "${full_vmname}" 2>/dev/null | grep '^State' | grep -q 'shut off'; then + echo "Starting VM ${full_vmname}" + sudo virsh start "${full_vmname}" + fi + + # Wait for IP and set properties + echo "Waiting for VM ${full_vmname} to have an IP" + local ip + ip=$(get_vm_ip "${full_vmname}") + if [ -z "${ip}" ]; then + echo "VM ${full_vmname} has no IP" + record_junit "${vmname}" "ip-assignment" "FAILED" + return 1 + fi + + echo "VM ${full_vmname} has IP ${ip}" + record_junit "${vmname}" "ip-assignment" "OK" + + # Clear any previous SSH key info for this IP + if [ -f "${HOME}/.ssh/known_hosts" ]; then + ssh-keygen -R "${ip}" 2>/dev/null || true + fi + + # Set the VM properties for this scenario + set_vm_property "${vmname}" "ip" "${ip}" + set_vm_property "${vmname}" "ssh_port" "22" + set_vm_property "${vmname}" "api_port" "6443" + set_vm_property "${vmname}" "lb_port" "5678" + + if wait_for_ssh "${ip}"; then + record_junit "${vmname}" "ssh-access" "OK" + else + record_junit "${vmname}" "ssh-access" "FAILED" + return 1 + fi + + echo "${full_vmname} is up and ready (reused)" + return 0 +} # Public function to start a VM. # @@ -783,6 +891,19 @@ launch_vm() { record_junit "${vmname}" "vm-launch-args" "OK" + # Check if scheduler assigned a VM for reuse + local scheduler_vm="" + if scheduler_vm=$(get_scheduler_assigned_vm) && should_reuse_vm; then + # Scheduler has assigned an existing VM for reuse + if setup_vm_properties_from_existing "${vmname}" "${scheduler_vm}"; then + record_junit "${vmname}" "install_vm" "OK (reused)" + return 0 + else + record_junit "${vmname}" "install_vm" "FAILED (reuse failed)" + return 1 + fi + fi + local -r full_vmname="$(full_vm_name "${vmname}")" local -r kickstart_url="${WEB_SERVER_URL}/scenario-info/${SCENARIO}/vms/${vmname}/kickstart.ks" @@ -1005,6 +1126,16 @@ remove_vm() { local -r keep_pool="${2:-false}" local -r full_vmname="$(full_vm_name "${vmname}")" + # In scheduler mode, the scheduler handles VM lifecycle + # We only release the VM back to the scheduler, not destroy it + if [ "${SCHEDULER_ENABLED}" = "true" ]; then + echo "Scheduler mode: releasing VM ${full_vmname} to scheduler" + # The scheduler will handle destruction when no compatible scenarios remain + # Just clean up scenario-specific state + rm -rf "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" + return 0 + fi + # Remove the actual VM if sudo virsh dumpxml "${full_vmname}" >/dev/null; then if ! sudo virsh dominfo "${full_vmname}" | grep '^State' | grep -q 'shut off'; then From e2900babbbb84e9dc5683d6b701f9d265ace3505 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 3 Mar 2026 11:45:10 +0100 Subject: [PATCH 03/24] Add changes to ci_phase_boot_and_test --- test/bin/ci_phase_boot_and_test.sh | 59 ++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index a3b10b0dc6..13d1b28f1b 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -27,6 +27,24 @@ prepare_scenario_sources() { fi } +# VM Scheduler configuration +# Set SCHEDULER_ENABLED=true to use the dynamic VM scheduler +# The scheduler provides VM reuse and resource-aware queuing +SCHEDULER_ENABLED="${SCHEDULER_ENABLED:-false}" +export SCHEDULER_ENABLED + +# Host resource limits for the scheduler (defaults from system) +_SYSTEM_VCPUS=$(nproc 2>/dev/null || echo 8) +_SYSTEM_MEMORY_KB=$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2}' || echo 16777216) +_SYSTEM_MEMORY_MB=$((_SYSTEM_MEMORY_KB / 1024)) + +export HOST_TOTAL_VCPUS="${HOST_TOTAL_VCPUS:-${_SYSTEM_VCPUS}}" +export HOST_TOTAL_MEMORY="${HOST_TOTAL_MEMORY:-${_SYSTEM_MEMORY_MB}}" + +# System reserved resources (for host OS, hypervisor overhead, etc.) +export SYSTEM_RESERVED_VCPUS="${SYSTEM_RESERVED_VCPUS:-4}" +export SYSTEM_RESERVED_MEMORY="${SYSTEM_RESERVED_MEMORY:-8192}" + # Log output automatically LOGDIR="${ROOTDIR}/_output/ci-logs" LOGFILE="${LOGDIR}/$(basename "$0" .sh).log" @@ -77,24 +95,35 @@ fi # Tell scenario.sh to merge stderr into stdout export SCENARIO_MERGE_OUTPUT_STREAMS=true -# Show the summary of the output of the parallel jobs. -if [ -t 0 ]; then - progress="--progress" +if [ "${SCHEDULER_ENABLED}" = "true" ]; then + # Use the dynamic VM scheduler for resource-aware execution + echo "Using dynamic VM scheduler (total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB, reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB)" + + if ! bash -x ./bin/vm_scheduler.sh orchestrate "${SCENARIOS_TO_RUN}"; then + TEST_OK=false + fi else - progress="" -fi + # Legacy mode: use GNU parallel without resource awareness + echo "Using GNU parallel for scenario execution (legacy mode)" + + # Show the summary of the output of the parallel jobs. + if [ -t 0 ]; then + progress="--progress" + else + progress="" + fi -TEST_OK=true -if ! parallel \ - ${progress} \ - --results "${SCENARIO_INFO_DIR}/{/.}/boot_and_run.log" \ - --joblog "${BOOT_TEST_JOB_LOG}" \ - --delay 5 \ - bash -x ./bin/scenario.sh create-and-run ::: "${SCENARIOS_TO_RUN}"/*.sh ; then - TEST_OK=false -fi + if ! parallel \ + ${progress} \ + --results "${SCENARIO_INFO_DIR}/{/.}/boot_and_run.log" \ + --joblog "${BOOT_TEST_JOB_LOG}" \ + --delay 5 \ + bash -x ./bin/scenario.sh create-and-run ::: "${SCENARIOS_TO_RUN}"/*.sh ; then + TEST_OK=false + fi -cat "${BOOT_TEST_JOB_LOG}" + cat "${BOOT_TEST_JOB_LOG}" +fi echo "Boot and test phase complete" if ! "${TEST_OK}"; then From eeb202a2cc90f978ab35ba3d6ce4e149097dfeb6 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 3 Mar 2026 13:09:02 +0100 Subject: [PATCH 04/24] minor fixes --- test/bin/ci_phase_boot_and_test.sh | 7 +++++- test/bin/vm_scheduler.sh | 35 +++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 13d1b28f1b..8702f73302 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -30,7 +30,7 @@ prepare_scenario_sources() { # VM Scheduler configuration # Set SCHEDULER_ENABLED=true to use the dynamic VM scheduler # The scheduler provides VM reuse and resource-aware queuing -SCHEDULER_ENABLED="${SCHEDULER_ENABLED:-false}" +SCHEDULER_ENABLED="${SCHEDULER_ENABLED:-true}" export SCHEDULER_ENABLED # Host resource limits for the scheduler (defaults from system) @@ -95,6 +95,8 @@ fi # Tell scenario.sh to merge stderr into stdout export SCENARIO_MERGE_OUTPUT_STREAMS=true +TEST_OK=true + if [ "${SCHEDULER_ENABLED}" = "true" ]; then # Use the dynamic VM scheduler for resource-aware execution echo "Using dynamic VM scheduler (total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB, reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB)" @@ -102,6 +104,9 @@ if [ "${SCHEDULER_ENABLED}" = "true" ]; then if ! bash -x ./bin/vm_scheduler.sh orchestrate "${SCENARIOS_TO_RUN}"; then TEST_OK=false fi + + # Print scheduler summary + bash -x ./bin/vm_scheduler.sh status else # Legacy mode: use GNU parallel without resource awareness echo "Using GNU parallel for scenario execution (legacy mode)" diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index f1e75a1a03..e8b40616c2 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -924,8 +924,8 @@ orchestrate() { exit 1 fi - log "=== PHASE 0: Preparing ===" init_scheduler + log "=== PHASE 0: Preparing ===" log "=== PHASE 1: Classifying scenarios ===" local -a dynamic_scenarios=() @@ -1005,6 +1005,18 @@ orchestrate() { log "Resource validation PASSED - all scenarios can run" log "" + # Persist resource allocation for status command + cat > "${SCHEDULER_STATE_DIR}/resource_allocation" < Date: Wed, 4 Mar 2026 00:00:02 +0100 Subject: [PATCH 05/24] fixes --- test/bin/vm_scheduler.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index e8b40616c2..4ded4c124b 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -722,24 +722,24 @@ run_scenario_on_vm() { # Phase 1: Create VM (only for new VMs) if [ "${is_new_vm}" = "true" ]; then - local create_log="${scenario_log_dir}/create.log" + local boot_log="${scenario_log_dir}/boot.log" local vm_dir="${VM_REGISTRY}/${vm_name}" mkdir -p "${vm_dir}" - ln -sf "${create_log}" "${vm_dir}/creation_log" + ln -sf "${boot_log}" "${vm_dir}/creation_log" - log "Creating VM ${vm_name} (timeout: ${create_timeout}s) - logging to ${create_log}. Linking to ${vm_dir}/creation_log" + log "Creating VM ${vm_name} (timeout: ${create_timeout}s) - logging to ${boot_log}. Linking to ${vm_dir}/creation_log" local create_exit=0 timeout --signal=TERM --kill-after=60 "${create_timeout}" \ - bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${create_log}" || create_exit=$? + bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? if [ ${create_exit} -ne 0 ]; then result="FAILED" exit_code=1 if [ ${create_exit} -eq 124 ]; then - log "VM creation TIMED OUT for ${scenario_name} after ${create_timeout}s - see ${create_log}" + log "VM creation TIMED OUT for ${scenario_name} after ${create_timeout}s - see ${boot_log}" else - log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${create_log}" + log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" fi fi fi From 1a7b1fe642ae94549b9dd78e4dc523bc50d7acf7 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 3 Mar 2026 18:21:29 +0100 Subject: [PATCH 06/24] Opt-in to dynamic scheduling --- .../el98-src@ai-model-serving-online.sh | 13 +++++++++ .../presubmits/el98-src@auto-recovery.sh | 12 ++++++++ .../el98-src@backup-and-restore-on-reboot.sh | 12 ++++++++ .../presubmits/el98-src@backups.sh | 12 ++++++++ .../presubmits/el98-src@configuration.sh | 12 ++++++++ .../presubmits/el98-src@downgrade-block.sh | 12 ++++++++ .../presubmits/el98-src@dual-stack.sh | 12 ++++++++ .../presubmits/el98-src@ipv6.sh | 12 ++++++++ .../presubmits/el98-src@multi-nic.sh | 12 ++++++++ .../presubmits/el98-src@optional.sh | 23 +++++++++++---- .../presubmits/el98-src@router.sh | 12 ++++++++ .../presubmits/el98-src@standard-suite1.sh | 12 ++++++++ .../presubmits/el98-src@standard-suite2.sh | 12 ++++++++ .../presubmits/el98-src@storage.sh | 12 ++++++++ ...rc@upgrade-fails-then-recovers.sh.disabled | 12 ++++++++ .../presubmits/el98-src@upgrade-fails.sh | 12 ++++++++ .../presubmits/el96-src@auto-recovery.sh | 28 +++++++++++++++++++ .../el98-src@backup-and-restore-on-reboot.sh | 12 ++++++++ test/scenarios/presubmits/el98-src@backups.sh | 12 ++++++++ .../presubmits/el98-src@configuration.sh | 12 ++++++++ .../el98-src@downgrade-block.sh.disabled | 12 ++++++++ .../presubmits/el98-src@dual-stack.sh | 12 ++++++++ test/scenarios/presubmits/el98-src@ipv6.sh | 12 ++++++++ .../el98-src@low-latency.sh.disabled | 12 ++++++++ .../presubmits/el98-src@multi-nic.sh | 12 ++++++++ test/scenarios/presubmits/el98-src@router.sh | 12 ++++++++ .../presubmits/el98-src@standard-suite1.sh | 12 ++++++++ .../presubmits/el98-src@standard-suite2.sh | 12 ++++++++ test/scenarios/presubmits/el98-src@storage.sh | 12 ++++++++ ...rc@upgrade-fails-then-recovers.sh.disabled | 12 ++++++++ .../el98-src@upgrade-fails.sh.disabled | 12 ++++++++ 31 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 test/scenarios/presubmits/el96-src@auto-recovery.sh diff --git a/test/scenarios-bootc/presubmits/el98-src@ai-model-serving-online.sh b/test/scenarios-bootc/presubmits/el98-src@ai-model-serving-online.sh index 5bc51be78d..981f605dd4 100644 --- a/test/scenarios-bootc/presubmits/el98-src@ai-model-serving-online.sh +++ b/test/scenarios-bootc/presubmits/el98-src@ai-model-serving-online.sh @@ -6,6 +6,19 @@ # Increase greenboot timeout for optional packages (more services to start) GREENBOOT_TIMEOUT=1200 +# Opt-in to dynamic VM scheduling by declaring requirements +dynamic_schedule_requirements() { + cat < Date: Wed, 4 Mar 2026 09:35:56 +0100 Subject: [PATCH 07/24] add greenboot timeout override --- test/bin/scenario.sh | 4 ++-- test/bin/vm_scheduler.sh | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 8201999623..fac961c7a6 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -19,8 +19,8 @@ source "${SCRIPTDIR}/common_versions.sh" LVM_SYSROOT_SIZE="15360" PULL_SECRET="${PULL_SECRET:-${HOME}/.pull-secret.json}" PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")" -VM_BOOT_TIMEOUT=1200 # Overall total boot times are around 15m -VM_GREENBOOT_TIMEOUT=1800 # Greenboot readiness may take up to 15-30m depending on the load +VM_BOOT_TIMEOUT="${VM_BOOT_TIMEOUT:-1200}" # Overall total boot times are around 15m +VM_GREENBOOT_TIMEOUT="${VM_GREENBOOT_TIMEOUT:-1800}" # Greenboot readiness may take up to 15-30m depending on the load SKIP_SOS=${SKIP_SOS:-false} # may be overridden in global settings file SKIP_GREENBOOT=${SKIP_GREENBOOT:-false} # may be overridden in scenario file GREENBOOT_TIMEOUT=${GREENBOOT_TIMEOUT:-600} # may be overridden in scenario file diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 4ded4c124b..3b16560be7 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -718,6 +718,14 @@ run_scenario_on_vm() { if [ -n "${custom_test_timeout}" ]; then test_timeout="${custom_test_timeout}" fi + + # Export greenboot timeout for scenario.sh to use + local custom_greenboot_timeout + custom_greenboot_timeout=$(get_req_value "${req_file}" "greenboot_timeout" "") + if [ -n "${custom_greenboot_timeout}" ]; then + export VM_GREENBOOT_TIMEOUT="${custom_greenboot_timeout}" + log "Using custom greenboot timeout: ${custom_greenboot_timeout}s" + fi fi # Phase 1: Create VM (only for new VMs) From 6daad80d03ae5d20091f91d5be6dad5b99b1014c Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 09:38:29 +0100 Subject: [PATCH 08/24] Add destroy_after functionality to dynamic scheduler --- test/bin/vm_scheduler.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 3b16560be7..d48acecdd4 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -786,6 +786,15 @@ run_scenario_on_vm() { return ${exit_code} fi + # Check if scenario requested VM destruction after completion (e.g., system-level changes) + local destroy_after + destroy_after=$(get_req_value "${req_file}" "destroy_after" "false") + if [ "${destroy_after}" = "true" ]; then + destroy_vm "${vm_name}" "destroy_after flag set" + release_lock "vm_dispatch" + return ${exit_code} + fi + # Check if any queued scenario can use this VM local next_scenario="" for queued in $(get_queued_scenarios); do From dd2aea43b87f14d5cfc0a3ae8d31c7f0b0363614 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 11:14:33 +0100 Subject: [PATCH 09/24] fix false-evaluating calculation for summary --- test/bin/vm_scheduler.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index d48acecdd4..ce9d7a2664 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -301,8 +301,8 @@ generate_dynamic_vm_name() { echo "${vm_name}" return 0 fi - ((count++)) - #TODO do i even need this? + ((count++)) || true + # Safety limit to prevent infinite loop if [ ${count} -gt 999 ]; then echo "ERROR: Too many VMs" >&2 return 1 @@ -1114,14 +1114,14 @@ show_status() { for queue_file in "${SCENARIO_QUEUE}"/*; do [ -f "${queue_file}" ] || continue - ((total_scenarios++)) + ((total_scenarios++)) || true local result result=$(get_req_value "${queue_file}" "result" "") if [ "${result}" = "SUCCESS" ]; then - ((passed_scenarios++)) + ((passed_scenarios++)) || true elif [ "${result}" = "FAILED" ]; then - ((failed_scenarios++)) + ((failed_scenarios++)) || true fi done @@ -1137,9 +1137,9 @@ show_status() { local reused reused=$(cat "${reused_file}") if [ "${reused}" = "true" ]; then - ((vm_reuses++)) + ((vm_reuses++)) || true else - ((vms_created++)) + ((vms_created++)) || true fi fi done From af71cad294d222490e775c504737fe122fb37d5d Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 15:22:50 +0100 Subject: [PATCH 10/24] remove -x from vm_scheduler.sh call --- test/bin/ci_phase_boot_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 8702f73302..9fafd2c07f 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -101,7 +101,7 @@ if [ "${SCHEDULER_ENABLED}" = "true" ]; then # Use the dynamic VM scheduler for resource-aware execution echo "Using dynamic VM scheduler (total: vcpus=${HOST_TOTAL_VCPUS}, memory=${HOST_TOTAL_MEMORY}MB, reserved: vcpus=${SYSTEM_RESERVED_VCPUS}, memory=${SYSTEM_RESERVED_MEMORY}MB)" - if ! bash -x ./bin/vm_scheduler.sh orchestrate "${SCENARIOS_TO_RUN}"; then + if ! bash ./bin/vm_scheduler.sh orchestrate "${SCENARIOS_TO_RUN}"; then TEST_OK=false fi From 479873d3f07de13814223d17b5f582c1e3eba2f7 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 17:31:30 +0100 Subject: [PATCH 11/24] Increase system reserved resources for scheduler --- test/bin/ci_phase_boot_and_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 9fafd2c07f..a31a0b1048 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -41,9 +41,9 @@ _SYSTEM_MEMORY_MB=$((_SYSTEM_MEMORY_KB / 1024)) export HOST_TOTAL_VCPUS="${HOST_TOTAL_VCPUS:-${_SYSTEM_VCPUS}}" export HOST_TOTAL_MEMORY="${HOST_TOTAL_MEMORY:-${_SYSTEM_MEMORY_MB}}" -# System reserved resources (for host OS, hypervisor overhead, etc.) -export SYSTEM_RESERVED_VCPUS="${SYSTEM_RESERVED_VCPUS:-4}" -export SYSTEM_RESERVED_MEMORY="${SYSTEM_RESERVED_MEMORY:-8192}" +# System reserved resources (for host OS, hypervisor overhead, robot framework, etc.) +export SYSTEM_RESERVED_VCPUS="${SYSTEM_RESERVED_VCPUS:-12}" +export SYSTEM_RESERVED_MEMORY="${SYSTEM_RESERVED_MEMORY:-16384}" # Log output automatically LOGDIR="${ROOTDIR}/_output/ci-logs" From a7efaab0fb625f59aa5269776057ec781683efc8 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 22:50:50 +0100 Subject: [PATCH 12/24] Rely on existing vm timeout for creation --- test/bin/vm_scheduler.sh | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index ce9d7a2664..7180cb6532 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -81,10 +81,8 @@ SYSTEM_RESERVED_MEMORY="${SYSTEM_RESERVED_MEMORY:-4096}" HOST_AVAILABLE_VCPUS=$((HOST_TOTAL_VCPUS - SYSTEM_RESERVED_VCPUS)) HOST_AVAILABLE_MEMORY=$((HOST_TOTAL_MEMORY - SYSTEM_RESERVED_MEMORY)) -# Timeouts for VM operations (in seconds) -# VM creation includes kickstart installation -VM_CREATE_TIMEOUT="${VM_CREATE_TIMEOUT:-600}" -# Default test timeout - scenarios can override via test_timeout in requirements +# Timeout for test execution (in seconds) +# Scenarios can override via test_timeout in requirements VM_TEST_TIMEOUT="${VM_TEST_TIMEOUT:-3600}" # Calculated resource requirements (populated during planning phase) @@ -480,7 +478,6 @@ recalculate_resource_usage() { local vm_status vm_status=$(get_req_value "${vm_state}" "status" "unknown") - #TODO do i have a "creating" status? if thats the case i need to take it too if [ "${vm_status}" = "in_use" ] || [ "${vm_status}" = "available" ]; then local vm_vcpus vm_memory vm_vcpus=$(get_req_value "${vm_state}" "vcpus" "0") @@ -704,15 +701,8 @@ run_scenario_on_vm() { # Get scenario-specific timeouts if specified, otherwise use defaults local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" - local create_timeout="${VM_CREATE_TIMEOUT}" local test_timeout="${VM_TEST_TIMEOUT}" if [ -f "${req_file}" ]; then - local custom_create_timeout - custom_create_timeout=$(get_req_value "${req_file}" "create_timeout" "") - if [ -n "${custom_create_timeout}" ]; then - create_timeout="${custom_create_timeout}" - fi - local custom_test_timeout custom_test_timeout=$(get_req_value "${req_file}" "test_timeout" "") if [ -n "${custom_test_timeout}" ]; then @@ -729,26 +719,22 @@ run_scenario_on_vm() { fi # Phase 1: Create VM (only for new VMs) + # No timeout wrapper - scenario.sh has its own retry mechanism with timeouts if [ "${is_new_vm}" = "true" ]; then local boot_log="${scenario_log_dir}/boot.log" local vm_dir="${VM_REGISTRY}/${vm_name}" mkdir -p "${vm_dir}" ln -sf "${boot_log}" "${vm_dir}/creation_log" - log "Creating VM ${vm_name} (timeout: ${create_timeout}s) - logging to ${boot_log}. Linking to ${vm_dir}/creation_log" + log "Creating VM ${vm_name} - logging to ${boot_log}" local create_exit=0 - timeout --signal=TERM --kill-after=60 "${create_timeout}" \ - bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? + bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? if [ ${create_exit} -ne 0 ]; then result="FAILED" exit_code=1 - if [ ${create_exit} -eq 124 ]; then - log "VM creation TIMED OUT for ${scenario_name} after ${create_timeout}s - see ${boot_log}" - else - log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" - fi + log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" fi fi @@ -1224,8 +1210,8 @@ Commands: status Show current scheduler state Environment Variables: - HOST_TOTAL_VCPUS Total host vCPUs (default: 48) - HOST_TOTAL_MEMORY Total host memory in MB (default: 98304) + HOST_TOTAL_VCPUS Total host vCPUs (default: system detected) + HOST_TOTAL_MEMORY Total host memory in MB (default: system detected) SYSTEM_RESERVED_VCPUS vCPUs reserved for host OS (default: 2) SYSTEM_RESERVED_MEMORY Memory reserved for host OS in MB (default: 4096) SCHEDULER_ENABLED Enable scheduler mode (default: false) From 135a7e20f51d1172684be7c3bf557972bdc9b545 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 5 Mar 2026 09:50:48 +0100 Subject: [PATCH 13/24] Fix reuse logic for vm properties and exit codes --- test/bin/vm_scheduler.sh | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 7180cb6532..0530a88703 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -718,24 +718,28 @@ run_scenario_on_vm() { fi fi - # Phase 1: Create VM (only for new VMs) + # Phase 1: Create/setup VM + # For new VMs: creates the VM from scratch + # For reused VMs: launch_vm detects reuse via should_reuse_vm() and sets up properties # No timeout wrapper - scenario.sh has its own retry mechanism with timeouts - if [ "${is_new_vm}" = "true" ]; then - local boot_log="${scenario_log_dir}/boot.log" - local vm_dir="${VM_REGISTRY}/${vm_name}" - mkdir -p "${vm_dir}" - ln -sf "${boot_log}" "${vm_dir}/creation_log" + local boot_log="${scenario_log_dir}/boot.log" + local vm_dir="${VM_REGISTRY}/${vm_name}" + mkdir -p "${vm_dir}" + ln -sf "${boot_log}" "${vm_dir}/creation_log" + if [ "${is_new_vm}" = "true" ]; then log "Creating VM ${vm_name} - logging to ${boot_log}" + else + log "Setting up reused VM ${vm_name} - logging to ${boot_log}" + fi - local create_exit=0 - bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? + local create_exit=0 + bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? - if [ ${create_exit} -ne 0 ]; then - result="FAILED" - exit_code=1 - log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" - fi + if [ ${create_exit} -ne 0 ]; then + result="FAILED" + exit_code=1 + log "VM setup failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" fi # Phase 2: Run tests (only if creation succeeded or VM was reused) @@ -807,7 +811,12 @@ run_scenario_on_vm() { release_lock "vm_dispatch" # Run the next scenario (recursive call) - run_scenario_on_vm "${next_script}" "${next_scenario}" "${vm_name}" "false" + # Capture exit code to propagate failures from reused scenarios + local reuse_exit=0 + run_scenario_on_vm "${next_script}" "${next_scenario}" "${vm_name}" "false" || reuse_exit=$? + if [ ${reuse_exit} -ne 0 ]; then + exit_code=${reuse_exit} + fi else # No compatible scenario waiting - destroy VM destroy_vm "${vm_name}" From 708dfd0b300f00acd657e563adef1c685ad9333d Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 5 Mar 2026 11:48:12 +0100 Subject: [PATCH 14/24] Add vm retry creation metrics and speed up test turnaround --- test/bin/scenario.sh | 47 +++++++++++++++++------- test/bin/vm_scheduler.sh | 78 ++++++++++++++++++++++++++++------------ 2 files changed, 89 insertions(+), 36 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index fac961c7a6..25bcd17fd5 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -760,16 +760,24 @@ setup_vm_properties_from_existing() { if sudo virsh dominfo "${full_vmname}" 2>/dev/null | grep '^State' | grep -q 'shut off'; then echo "Starting VM ${full_vmname}" sudo virsh start "${full_vmname}" - fi - - # Wait for IP and set properties - echo "Waiting for VM ${full_vmname} to have an IP" - local ip - ip=$(get_vm_ip "${full_vmname}") - if [ -z "${ip}" ]; then - echo "VM ${full_vmname} has no IP" - record_junit "${vmname}" "ip-assignment" "FAILED" - return 1 + # For a VM that was shut off, use the full wait logic + local ip + ip=$(get_vm_ip "${full_vmname}") + if [ -z "${ip}" ]; then + echo "VM ${full_vmname} has no IP" + record_junit "${vmname}" "ip-assignment" "FAILED" + return 1 + fi + else + # VM is already running - get IP directly without ping loop + # The VM was just used by another scenario, so the IP is valid + local ip + ip=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${full_vmname}" | head -1) + if [ -z "${ip}" ]; then + echo "VM ${full_vmname} has no IP" + record_junit "${vmname}" "ip-assignment" "FAILED" + return 1 + fi fi echo "VM ${full_vmname} has IP ${ip}" @@ -786,11 +794,19 @@ setup_vm_properties_from_existing() { set_vm_property "${vmname}" "api_port" "6443" set_vm_property "${vmname}" "lb_port" "5678" - if wait_for_ssh "${ip}"; then + # For a reused VM, do a quick SSH check instead of the full wait loop + # The VM was just running tests, so SSH should be immediately available + if ssh -oConnectTimeout=5 -oBatchMode=yes -oStrictHostKeyChecking=accept-new "redhat@${ip}" "echo host is up" &>/dev/null; then record_junit "${vmname}" "ssh-access" "OK" else - record_junit "${vmname}" "ssh-access" "FAILED" - return 1 + # Fall back to full wait if quick check fails + echo "Quick SSH check failed, waiting for SSH..." + if wait_for_ssh "${ip}"; then + record_junit "${vmname}" "ssh-access" "OK" + else + record_junit "${vmname}" "ssh-access" "FAILED" + return 1 + fi fi echo "${full_vmname} is up and ready (reused)" @@ -1047,6 +1063,11 @@ launch_vm() { # Retry the operation on error local backoff=$(( attempt * 5 )) echo "Error running virt-install: retrying in ${backoff}s on attempt ${attempt}" + + # Record retry for metrics (scheduler will aggregate these) + local retry_file="${SCENARIO_INFO_DIR}/${SCENARIO}/vm_creation_retries" + echo "${attempt}" > "${retry_file}" + sleep "${backoff}" # Cleanup the failed VM before trying to recreate it diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 0530a88703..57888cc200 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -718,46 +718,65 @@ run_scenario_on_vm() { fi fi - # Phase 1: Create/setup VM - # For new VMs: creates the VM from scratch - # For reused VMs: launch_vm detects reuse via should_reuse_vm() and sets up properties - # No timeout wrapper - scenario.sh has its own retry mechanism with timeouts local boot_log="${scenario_log_dir}/boot.log" + local run_log="${scenario_log_dir}/run.log" local vm_dir="${VM_REGISTRY}/${vm_name}" mkdir -p "${vm_dir}" ln -sf "${boot_log}" "${vm_dir}/creation_log" if [ "${is_new_vm}" = "true" ]; then + # New VM: separate create and run phases + # Create phase has its own retry logic with timeouts log "Creating VM ${vm_name} - logging to ${boot_log}" - else - log "Setting up reused VM ${vm_name} - logging to ${boot_log}" - fi - local create_exit=0 - bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? + local create_exit=0 + bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? - if [ ${create_exit} -ne 0 ]; then - result="FAILED" - exit_code=1 - log "VM setup failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" - fi + if [ ${create_exit} -ne 0 ]; then + result="FAILED" + exit_code=1 + log "VM creation failed for ${scenario_name} (exit ${create_exit}) - see ${boot_log}" + fi + + # Run tests only if creation succeeded + if [ "${exit_code}" -eq 0 ]; then + log "Running tests for ${scenario_name} (timeout: ${test_timeout}s) - logging to ${run_log}" + + local run_exit=0 + timeout --signal=TERM --kill-after=60 "${test_timeout}" \ + bash -x "${SCRIPTDIR}/scenario.sh" run "${scenario_script}" &> "${run_log}" || run_exit=$? + + if [ ${run_exit} -ne 0 ]; then + result="FAILED" + exit_code=1 + if [ ${run_exit} -eq 124 ]; then + log "Tests TIMED OUT for ${scenario_name} after ${test_timeout}s - see ${run_log}" + else + log "Tests failed for ${scenario_name} (exit ${run_exit}) - see ${run_log}" + fi + fi + fi + else + # Reused VM: use create-and-run for faster turnaround + # The create phase is very quick (just IP lookup and SSH check) + # Combined output goes to run.log, with setup info in boot.log + log "Setting up reused VM ${vm_name} and running tests (timeout: ${test_timeout}s)" - # Phase 2: Run tests (only if creation succeeded or VM was reused) - if [ "${exit_code}" -eq 0 ]; then - local run_log="${scenario_log_dir}/run.log" - log "Running tests for ${scenario_name} (timeout: ${test_timeout}s) - logging to ${run_log}" + # Log reuse info to boot.log for consistency + echo "VM ${vm_name} reused for scenario ${scenario_name}" > "${boot_log}" + date >> "${boot_log}" - local run_exit=0 + local combined_exit=0 timeout --signal=TERM --kill-after=60 "${test_timeout}" \ - bash -x "${SCRIPTDIR}/scenario.sh" run "${scenario_script}" &> "${run_log}" || run_exit=$? + bash -x "${SCRIPTDIR}/scenario.sh" create-and-run "${scenario_script}" &> "${run_log}" || combined_exit=$? - if [ ${run_exit} -ne 0 ]; then + if [ ${combined_exit} -ne 0 ]; then result="FAILED" exit_code=1 - if [ ${run_exit} -eq 124 ]; then + if [ ${combined_exit} -eq 124 ]; then log "Tests TIMED OUT for ${scenario_name} after ${test_timeout}s - see ${run_log}" else - log "Tests failed for ${scenario_name} (exit ${run_exit}) - see ${run_log}" + log "Tests failed for ${scenario_name} (exit ${combined_exit}) - see ${run_log}" fi fi fi @@ -1123,6 +1142,7 @@ show_status() { # --- VM Metrics --- local vms_created=0 local vm_reuses=0 + local vm_creation_retries=0 local max_runs_per_vm=0 for scenario_dir in "${SCENARIO_STATUS}"/*; do @@ -1139,6 +1159,17 @@ show_status() { fi done + # Count VM creation retries from scenario info directories + for scenario_dir in "${SCENARIO_INFO_DIR}"/*; do + [ -d "${scenario_dir}" ] || continue + local retry_file="${scenario_dir}/vm_creation_retries" + if [ -f "${retry_file}" ]; then + local retries + retries=$(cat "${retry_file}") + ((vm_creation_retries += retries)) || true + fi + done + for vm_dir in "${VM_REGISTRY}"/*; do [ -d "${vm_dir}" ] || continue local history_file="${vm_dir}/scenario_history.log" @@ -1163,6 +1194,7 @@ show_status() { echo "=== Dynamic VM Efficiency ===" echo " VMs created: ${vms_created}" + echo " VM creation retries: ${vm_creation_retries}" echo " VM reuses: ${vm_reuses}" echo " Reuse rate: ${reuse_rate}%" echo " Max scenarios per VM: ${max_runs_per_vm}" From ad2b2e36b19744237a444bbc4fac76ff1aff57a9 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 5 Mar 2026 14:41:14 +0100 Subject: [PATCH 15/24] add queue ordering to prioritize bigger VMs first --- test/bin/vm_scheduler.sh | 53 +++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 57888cc200..d5ce8dc68d 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -612,31 +612,71 @@ destroy_vm() { log "Destroyed VM ${vm_name}" } +# Sort scenarios by requirements to maximize VM reuse +# Scenarios with same requirements end up adjacent, enabling reuse +sort_scenarios_for_reuse() { + local -a scenarios=("$@") + + # Sort scenarios to maximize VM reuse: + # 1. boot_image: more capable first (optionals before base) - descending + # 2. networks: more networks first - descending + # 3. vcpus: more vcpus first - descending + # + # This ensures larger/more capable VMs are created first. + # When they finish, they can run smaller/simpler scenarios. + + for scenario_script in "${scenarios[@]}"; do + local scenario_name + scenario_name=$(basename "${scenario_script}" .sh) + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + + if [ -f "${req_file}" ]; then + local boot_image networks vcpus + boot_image=$(get_req_value "${req_file}" "boot_image" "default") + networks=$(get_req_value "${req_file}" "networks" "default") + vcpus=$(get_req_value "${req_file}" "min_vcpus" "2") + # Sort key: boot_image, networks, vcpus - all descending via reverse sort + printf "%s\t%s\t%02d\t%s\n" "${boot_image}" "${networks}" "${vcpus}" "${scenario_script}" + else + printf "aaa\tdefault\t00\t%s\n" "${scenario_script}" + fi + done | sort -r | cut -f4 +} + +# Global sequence counter for queue ordering +QUEUE_SEQUENCE=0 + queue_scenario() { local scenario_script="$1" local scenario_name="$2" local req_file="$3" + QUEUE_SEQUENCE=$((QUEUE_SEQUENCE + 1)) local queue_file="${SCENARIO_QUEUE}/${scenario_name}" cat > "${queue_file}" < Date: Thu, 5 Mar 2026 22:36:40 +0100 Subject: [PATCH 16/24] Add reusability score and relax network requirements --- test/bin/vm_scheduler.sh | 78 ++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index d5ce8dc68d..00d6887001 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -414,9 +414,12 @@ vm_satisfies_requirements() { [ "${vm_disksize}" -ge "${req_disksize}" ] || return 1 # Check networks (VM must have all required networks) - for net in ${req_networks//,/ }; do - echo ",${vm_networks}," | grep -q ",${net}," || return 1 - done + # Empty network requirement means "any network is fine" (network-agnostic) + if [ -n "${req_networks}" ]; then + for net in ${req_networks//,/ }; do + echo ",${vm_networks}," | grep -q ",${net}," || return 1 + done + fi # Check FIPS (if required, VM must have it) if [ "${req_fips}" = "true" ] && [ "${vm_fips}" != "true" ]; then @@ -612,18 +615,60 @@ destroy_vm() { log "Destroyed VM ${vm_name}" } +# Calculate reusability score for a scenario +# Lower score = less reusable = should run first +# Higher score = more reusable = should stay queued longer for reuse +get_reusability_score() { + local boot_image="$1" + local networks="$2" + + # Score 0: Completely non-reusable (special boot_image that requires exact match) + case "${boot_image}" in + *-fips|*-tuned|*-isolated|*-ai-model-serving|*-fake-next-minor) + echo "0" + return + ;; + esac + + # Score 1: Non-reusable due to special network (not default or empty) + if [ -n "${networks}" ] && [ "${networks}" != "default" ]; then + echo "1" + return + fi + + # Score 2: Reusable - optionals can be reused by source scenarios + case "${boot_image}" in + *-optionals|*-optional|*-with-optional) + echo "2" + return + ;; + esac + + # Score 3: Flexible - source/base image with default network + if [ "${networks}" = "default" ]; then + echo "3" + return + fi + + # Score 4: Most flexible - network-agnostic (empty networks) + # Can run on ANY VM, stays queued longest for maximum reuse + echo "4" +} + # Sort scenarios by requirements to maximize VM reuse -# Scenarios with same requirements end up adjacent, enabling reuse sort_scenarios_for_reuse() { local -a scenarios=("$@") - # Sort scenarios to maximize VM reuse: - # 1. boot_image: more capable first (optionals before base) - descending - # 2. networks: more networks first - descending - # 3. vcpus: more vcpus first - descending + # Sort strategy for maximum VM reuse: + # 1. Reusability score (ascending): restrictive scenarios first + # - They can't be reused anyway, so get them out of the way + # - Their VMs are destroyed when done + # 2. Within same reusability: boot_image descending (optionals before source) + # 3. Then vcpus descending (larger VMs first) # - # This ensures larger/more capable VMs are created first. - # When they finish, they can run smaller/simpler scenarios. + # This ensures: + # - Restrictive scenarios (special networks/boot_images) start first + # - Flexible scenarios stay queued longer, maximizing reuse opportunities for scenario_script in "${scenarios[@]}"; do local scenario_name @@ -631,16 +676,19 @@ sort_scenarios_for_reuse() { local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" if [ -f "${req_file}" ]; then - local boot_image networks vcpus + local boot_image networks vcpus reuse_score boot_image=$(get_req_value "${req_file}" "boot_image" "default") networks=$(get_req_value "${req_file}" "networks" "default") vcpus=$(get_req_value "${req_file}" "min_vcpus" "2") - # Sort key: boot_image, networks, vcpus - all descending via reverse sort - printf "%s\t%s\t%02d\t%s\n" "${boot_image}" "${networks}" "${vcpus}" "${scenario_script}" + reuse_score=$(get_reusability_score "${boot_image}" "${networks}") + # Sort key: reuse_score (asc), boot_image (desc), vcpus (desc) + # Use inverse vcpus (100-vcpus) so ascending sort gives descending vcpus + printf "%d\t%s\t%02d\t%s\n" "${reuse_score}" "${boot_image}" "$((100 - vcpus))" "${scenario_script}" else - printf "aaa\tdefault\t00\t%s\n" "${scenario_script}" + # Unknown requirements - treat as flexible, put last + printf "9\taaa\t99\t%s\n" "${scenario_script}" fi - done | sort -r | cut -f4 + done | sort -t$'\t' -k1,1n -k2,2r -k3,3n | cut -f4 } # Global sequence counter for queue ordering From 1ba7cb924313cc92cd5069c03b47c2ae8a122a64 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Thu, 5 Mar 2026 17:16:34 +0100 Subject: [PATCH 17/24] Refine base image matching for reuse --- test/bin/vm_scheduler.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 00d6887001..d8a1eebd33 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -356,25 +356,29 @@ boot_image_compatible() { [ "${vm_image}" = "${req_image}" ] && return 0 # Special images require exact match (cannot substitute) + # These have unique configurations that can't be approximated case "${req_image}" in - *-fips|*-tuned|*-isolated|*-ai-model-serving) + *-fips|*-tuned|*-isolated|*-ai-model-serving|*-fake-next-minor) return 1 # Must be exact ;; esac - # Check if VM image is a superset of required image - # Hierarchy: source < source-optionals, brew-lrel < brew-lrel-optional - case "${req_image}" in - *-source) - # source-optionals is superset of source - [[ "${vm_image}" == "${req_image}-optionals" ]] && return 0 - ;; - *-brew-lrel) - # brew-lrel-optional is superset of brew-lrel - [[ "${vm_image}" == "${req_image}-optional" ]] && return 0 + # If VM has a special configuration, it can only run its exact scenarios + case "${vm_image}" in + *-fips|*-tuned|*-isolated|*-ai-model-serving|*-fake-next-minor) + return 1 # Special VM can't run generic scenarios ;; esac + # Check if VM image is a superset of required image + # A VM with "optionals" can run scenarios that need the base image + # This is like networks: having more capabilities is fine + if [[ "${vm_image}" == "${req_image}-optionals" ]] || \ + [[ "${vm_image}" == "${req_image}-optional" ]] || \ + [[ "${vm_image}" == "${req_image}-with-optional" ]]; then + return 0 + fi + # ai-model-serving includes qemu-guest-agent, so can run isolated scenarios if [[ "${req_image}" == *-isolated ]] && [[ "${vm_image}" == *-ai-model-serving ]]; then return 0 From d526bd92b8674d7f20c56141b75f9cef2435d22d Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 10 Mar 2026 16:13:13 +0100 Subject: [PATCH 18/24] Add test duration summary --- test/bin/vm_scheduler.sh | 86 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index d8a1eebd33..0c7efc61f6 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -834,10 +834,20 @@ run_scenario_on_vm() { if [ "${exit_code}" -eq 0 ]; then log "Running tests for ${scenario_name} (timeout: ${test_timeout}s) - logging to ${run_log}" + # Track test execution time + local test_start_time + test_start_time=$(date +%s) + local run_exit=0 timeout --signal=TERM --kill-after=60 "${test_timeout}" \ bash -x "${SCRIPTDIR}/scenario.sh" run "${scenario_script}" &> "${run_log}" || run_exit=$? + # Record test execution time + local test_end_time test_duration + test_end_time=$(date +%s) + test_duration=$((test_end_time - test_start_time)) + echo "${test_duration}" > "${scenario_log_dir}/test_time" + if [ ${run_exit} -ne 0 ]; then result="FAILED" exit_code=1 @@ -858,10 +868,20 @@ run_scenario_on_vm() { echo "VM ${vm_name} reused for scenario ${scenario_name}" > "${boot_log}" date >> "${boot_log}" + # Track test execution time (for reused VM, this includes quick setup) + local test_start_time + test_start_time=$(date +%s) + local combined_exit=0 timeout --signal=TERM --kill-after=60 "${test_timeout}" \ bash -x "${SCRIPTDIR}/scenario.sh" create-and-run "${scenario_script}" &> "${run_log}" || combined_exit=$? + # Record test execution time + local test_end_time test_duration + test_end_time=$(date +%s) + test_duration=$((test_end_time - test_start_time)) + echo "${test_duration}" > "${scenario_log_dir}/test_time" + if [ ${combined_exit} -ne 0 ]; then result="FAILED" exit_code=1 @@ -1323,6 +1343,72 @@ show_status() { done echo "" + echo "=== Test Durations ===" + echo " (Tests longer than avg boot time should be 'slow', shorter should be 'fast')" + echo "" + # Collect test durations and check for mislabeling + local mislabeled_count=0 + { + for scenario_dir in "${SCENARIO_INFO_DIR}"/*; do + [ -d "${scenario_dir}" ] || continue + local scenario_name + scenario_name=$(basename "${scenario_dir}") + local test_time_file="${scenario_dir}/test_time" + + if [ -f "${test_time_file}" ]; then + local test_time + test_time=$(cat "${test_time_file}") + local test_mins=$((test_time / 60)) + local test_secs=$((test_time % 60)) + local time_str + time_str=$(printf '%2d:%02d' "${test_mins}" "${test_secs}") + + # Get slow/fast flag from requirements + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + local slow_flag="false" + local fast_flag="false" + local current_label="-" + if [ -f "${req_file}" ]; then + slow_flag=$(get_req_value "${req_file}" "slow" "false") + fast_flag=$(get_req_value "${req_file}" "fast" "false") + if [ "${slow_flag}" = "true" ]; then + current_label="slow" + elif [ "${fast_flag}" = "true" ]; then + current_label="fast" + fi + fi + + # Determine if mislabeled (compare against avg boot time) + local status="" + if [ ${avg_boot_time} -gt 0 ]; then + if [ ${test_time} -gt ${avg_boot_time} ]; then + # Test is slower than boot - should be 'slow' + if [ "${fast_flag}" = "true" ]; then + status="<-- MISLABELED (should be slow)" + mislabeled_count=$((mislabeled_count + 1)) + fi + else + # Test is faster than boot - should be 'fast' + if [ "${slow_flag}" = "true" ]; then + status="<-- MISLABELED (should be fast)" + mislabeled_count=$((mislabeled_count + 1)) + fi + fi + fi + + # Output: time, scenario, label, status (tab-separated for sorting) + printf "%d\t%s\t%-40s\t%-6s\t%s\n" "${test_time}" "${time_str}" "${scenario_name}" "${current_label}" "${status}" + fi + done + } | sort -t$'\t' -k1,1rn | cut -f2- | while IFS=$'\t' read -r time_str scenario label status; do + printf " %s %-40s [%s] %s\n" "${time_str}" "${scenario}" "${label}" "${status}" + done + echo "" + if [ ${mislabeled_count} -gt 0 ]; then + echo " WARNING: ${mislabeled_count} test(s) appear to be mislabeled based on avg boot time (${avg_boot_time_str})" + fi + echo "" + echo "=== Scenarios ===" for queue_file in "${SCENARIO_QUEUE}"/*; do [ -f "${queue_file}" ] || continue From e22a81e8512d8ca2484a6137b072f9c49887748f Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Wed, 4 Mar 2026 13:14:26 +0100 Subject: [PATCH 19/24] Break down presubmit scenarios into robot suites --- .../el98-src@auto-recovery-extra.sh | 28 ++++++ .../presubmits/el98-src@auto-recovery.sh | 2 +- .../presubmits/el98-src@cert-manager.sh | 53 ++++++++++ .../presubmits/el98-src@configuration.sh | 2 +- .../presubmits/el98-src@containers-policy.sh | 28 ++++++ ...src@standard-suite1.sh => el98-src@dns.sh} | 2 +- .../presubmits/el98-src@etcd.sh | 99 +++++++++++++++++++ .../presubmits/el98-src@feature-gates.sh | 28 ++++++ .../presubmits/el98-src@gateway-api.sh | 53 ++++++++++ .../el98-src@generic-device-plugin.sh | 53 ++++++++++ .../el98-src@healthchecks-disabled-service.sh | 53 ++++++++++ .../presubmits/el98-src@hostname.sh | 99 +++++++++++++++++++ .../presubmits/el98-src@kustomize.sh | 99 +++++++++++++++++++ .../presubmits/el98-src@log-scan.sh | 28 ++++++ .../presubmits/el98-src@multus.sh | 53 ++++++++++ .../presubmits/el98-src@networking-smoke.sh | 99 +++++++++++++++++++ .../presubmits/el98-src@observability.sh | 53 ++++++++++ .../{el98-src@optional.sh => el98-src@olm.sh} | 2 +- .../presubmits/el98-src@pvc-resize.sh | 28 ++++++ ...@standard-suite2.sh => el98-src@reboot.sh} | 2 +- .../presubmits/el98-src@router.sh | 3 +- .../presubmits/el98-src@show-config.sh | 28 ++++++ ...98-src@storage.sh => el98-src@snapshot.sh} | 2 +- .../presubmits/el98-src@sriov.sh | 53 ++++++++++ .../el98-src@storage-version-migration.sh | 28 ++++++ .../presubmits/el98-src@tls-configuration.sh | 28 ++++++ .../el98-src@validate-certificate-rotation.sh | 28 ++++++ .../el98-src@validate-custom-certificates.sh | 28 ++++++ .../el98-src@validate-selinux-policy.sh | 28 ++++++ ...-src@validate-service-account-ca-bundle.sh | 28 ++++++ .../presubmits/el98-src@version.sh | 99 +++++++++++++++++++ .../presubmits/el96-src@containers-policy.sh | 33 +++++++ test/scenarios/presubmits/el96-src@dns.sh | 33 +++++++ test/scenarios/presubmits/el96-src@etcd.sh | 33 +++++++ .../presubmits/el96-src@feature-gates.sh | 33 +++++++ .../scenarios/presubmits/el96-src@hostname.sh | 33 +++++++ .../presubmits/el96-src@kustomize.sh | 33 +++++++ .../scenarios/presubmits/el96-src@log-scan.sh | 33 +++++++ .../presubmits/el96-src@networking-smoke.sh | 33 +++++++ .../presubmits/el96-src@pvc-resize.sh | 28 ++++++ test/scenarios/presubmits/el96-src@reboot.sh | 28 ++++++ .../presubmits/el96-src@show-config.sh | 28 ++++++ .../scenarios/presubmits/el96-src@snapshot.sh | 28 ++++++ .../el96-src@storage-version-migrator.sh | 28 ++++++ .../presubmits/el96-src@tls-configuration.sh | 28 ++++++ .../el96-src@validate-certificate-rotation.sh | 33 +++++++ .../el96-src@validate-custom-certificates.sh | 33 +++++++ .../el96-src@validate-selinux-policy.sh | 33 +++++++ ...-src@validate-service-account-ca-bundle.sh | 33 +++++++ test/scenarios/presubmits/el96-src@version.sh | 33 +++++++ .../presubmits/el98-src@configuration.sh | 2 +- test/scenarios/presubmits/el98-src@router.sh | 2 +- .../presubmits/el98-src@standard-suite1.sh | 2 +- .../presubmits/el98-src@standard-suite2.sh | 2 +- test/scenarios/presubmits/el98-src@storage.sh | 2 +- .../el98-src@upgrade-fails.sh.disabled | 6 ++ 56 files changed, 1818 insertions(+), 12 deletions(-) create mode 100644 test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@cert-manager.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@containers-policy.sh rename test/scenarios-bootc/presubmits/{el98-src@standard-suite1.sh => el98-src@dns.sh} (95%) create mode 100644 test/scenarios-bootc/presubmits/el98-src@etcd.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@feature-gates.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@gateway-api.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@generic-device-plugin.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@healthchecks-disabled-service.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@hostname.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@kustomize.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@log-scan.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@multus.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@networking-smoke.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@observability.sh rename test/scenarios-bootc/presubmits/{el98-src@optional.sh => el98-src@olm.sh} (97%) create mode 100644 test/scenarios-bootc/presubmits/el98-src@pvc-resize.sh rename test/scenarios-bootc/presubmits/{el98-src@standard-suite2.sh => el98-src@reboot.sh} (91%) create mode 100644 test/scenarios-bootc/presubmits/el98-src@show-config.sh rename test/scenarios-bootc/presubmits/{el98-src@storage.sh => el98-src@snapshot.sh} (90%) create mode 100644 test/scenarios-bootc/presubmits/el98-src@sriov.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@storage-version-migration.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@tls-configuration.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@validate-certificate-rotation.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@validate-custom-certificates.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@validate-selinux-policy.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@validate-service-account-ca-bundle.sh create mode 100644 test/scenarios-bootc/presubmits/el98-src@version.sh create mode 100644 test/scenarios/presubmits/el96-src@containers-policy.sh create mode 100644 test/scenarios/presubmits/el96-src@dns.sh create mode 100644 test/scenarios/presubmits/el96-src@etcd.sh create mode 100644 test/scenarios/presubmits/el96-src@feature-gates.sh create mode 100644 test/scenarios/presubmits/el96-src@hostname.sh create mode 100644 test/scenarios/presubmits/el96-src@kustomize.sh create mode 100644 test/scenarios/presubmits/el96-src@log-scan.sh create mode 100644 test/scenarios/presubmits/el96-src@networking-smoke.sh create mode 100644 test/scenarios/presubmits/el96-src@pvc-resize.sh create mode 100644 test/scenarios/presubmits/el96-src@reboot.sh create mode 100644 test/scenarios/presubmits/el96-src@show-config.sh create mode 100644 test/scenarios/presubmits/el96-src@snapshot.sh create mode 100644 test/scenarios/presubmits/el96-src@storage-version-migrator.sh create mode 100644 test/scenarios/presubmits/el96-src@tls-configuration.sh create mode 100644 test/scenarios/presubmits/el96-src@validate-certificate-rotation.sh create mode 100644 test/scenarios/presubmits/el96-src@validate-custom-certificates.sh create mode 100644 test/scenarios/presubmits/el96-src@validate-selinux-policy.sh create mode 100644 test/scenarios/presubmits/el96-src@validate-service-account-ca-bundle.sh create mode 100644 test/scenarios/presubmits/el96-src@version.sh diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh new file mode 100644 index 0000000000..2fe180e9aa --- /dev/null +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +# Opt-in to dynamic VM scheduling by declaring requirements +dynamic_schedule_requirements() { + cat < Date: Thu, 5 Mar 2026 07:43:34 +0100 Subject: [PATCH 20/24] Fix remove_vm logic when scheduler is enabled --- test/bin/scenario.sh | 21 +++++++++++-------- ... => el96-src@storage-version-migration.sh} | 0 2 files changed, 12 insertions(+), 9 deletions(-) rename test/scenarios/presubmits/{el96-src@storage-version-migrator.sh => el96-src@storage-version-migration.sh} (100%) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 25bcd17fd5..93d6a8ae70 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -1072,7 +1072,8 @@ launch_vm() { # Cleanup the failed VM before trying to recreate it # Keep the storage pool for the subsequent VM creation - remove_vm "${vmname}" true + # Force destroy even in scheduler mode so retry can work + remove_vm "${vmname}" true true done if ${vm_created} ; then @@ -1142,23 +1143,25 @@ launch_vm() { } # Clean up the resources for one VM, optionally skipping storage pool removal +# Parameters: +# vmname: name of the VM to remove +# keep_pool: if true, keep the storage pool (default: false) +# force_destroy: if true, destroy VM even in scheduler mode (default: false) +# Used by retry logic to clean up failed VM creation attempts remove_vm() { local -r vmname="${1}" local -r keep_pool="${2:-false}" + local -r force_destroy="${3:-false}" local -r full_vmname="$(full_vm_name "${vmname}")" - # In scheduler mode, the scheduler handles VM lifecycle - # We only release the VM back to the scheduler, not destroy it - if [ "${SCHEDULER_ENABLED}" = "true" ]; then - echo "Scheduler mode: releasing VM ${full_vmname} to scheduler" - # The scheduler will handle destruction when no compatible scenarios remain - # Just clean up scenario-specific state - rm -rf "${SCENARIO_INFO_DIR}/${SCENARIO}/vms/${vmname}" + # In scheduler mode, skip VM destruction unless forced (needed for retry logic) + # This allows VMs to be reused by subsequent scenarios + if [ "${SCHEDULER_ENABLED}" = "true" ] && [ "${force_destroy}" != "true" ]; then return 0 fi # Remove the actual VM - if sudo virsh dumpxml "${full_vmname}" >/dev/null; then + if sudo virsh dumpxml "${full_vmname}" >/dev/null 2>&1; then if ! sudo virsh dominfo "${full_vmname}" | grep '^State' | grep -q 'shut off'; then sudo virsh destroy --graceful "${full_vmname}" || true fi diff --git a/test/scenarios/presubmits/el96-src@storage-version-migrator.sh b/test/scenarios/presubmits/el96-src@storage-version-migration.sh similarity index 100% rename from test/scenarios/presubmits/el96-src@storage-version-migrator.sh rename to test/scenarios/presubmits/el96-src@storage-version-migration.sh From f7a7aeeed0352f701916bf18d4be39cda7e3573c Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 6 Mar 2026 11:01:03 +0100 Subject: [PATCH 21/24] Use 4 CPUs as default and relax network reqs in scenarios --- test/bin/vm_scheduler.sh | 20 ++++++++++--------- .../el98-src@auto-recovery-extra.sh | 2 +- .../presubmits/el98-src@auto-recovery.sh | 2 +- .../el98-src@backup-and-restore-on-reboot.sh | 2 +- .../presubmits/el98-src@backups.sh | 2 +- .../presubmits/el98-src@configuration.sh | 2 +- .../presubmits/el98-src@containers-policy.sh | 2 +- .../presubmits/el98-src@feature-gates.sh | 2 +- .../presubmits/el98-src@log-scan.sh | 2 +- .../presubmits/el98-src@pvc-resize.sh | 2 +- .../presubmits/el98-src@reboot.sh | 2 +- .../presubmits/el98-src@show-config.sh | 2 +- .../presubmits/el98-src@snapshot.sh | 2 +- .../el98-src@storage-version-migration.sh | 2 +- .../presubmits/el98-src@tls-configuration.sh | 2 +- .../el98-src@validate-certificate-rotation.sh | 2 +- .../el98-src@validate-custom-certificates.sh | 2 +- .../el98-src@validate-selinux-policy.sh | 2 +- ...-src@validate-service-account-ca-bundle.sh | 2 +- .../presubmits/el96-src@auto-recovery.sh | 2 +- .../presubmits/el96-src@containers-policy.sh | 2 +- .../presubmits/el96-src@feature-gates.sh | 2 +- .../scenarios/presubmits/el96-src@log-scan.sh | 2 +- .../presubmits/el96-src@pvc-resize.sh | 2 +- test/scenarios/presubmits/el96-src@reboot.sh | 2 +- .../presubmits/el96-src@show-config.sh | 2 +- .../scenarios/presubmits/el96-src@snapshot.sh | 2 +- .../el96-src@storage-version-migration.sh | 2 +- .../presubmits/el96-src@tls-configuration.sh | 2 +- .../el96-src@validate-certificate-rotation.sh | 2 +- .../el96-src@validate-custom-certificates.sh | 2 +- .../el96-src@validate-selinux-policy.sh | 2 +- ...-src@validate-service-account-ca-bundle.sh | 2 +- .../el98-src@backup-and-restore-on-reboot.sh | 2 +- test/scenarios/presubmits/el98-src@backups.sh | 2 +- .../presubmits/el98-src@configuration.sh | 2 +- 36 files changed, 46 insertions(+), 44 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 0c7efc61f6..f4df5a228d 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -129,7 +129,10 @@ init_scheduler() { } # Default VM resources when not specified in launch_vm -DEFAULT_VM_VCPUS=2 +# Using 4 vCPUs by default maximizes VM reuse potential: +# - All VMs can run any scenario (no vcpu mismatch) +# - More queuing = more reuse opportunities +DEFAULT_VM_VCPUS=4 DEFAULT_VM_MEMORY=4096 parse_static_scenario_resources() { @@ -405,8 +408,8 @@ vm_satisfies_requirements() { # Get scenario requirements local req_vcpus req_memory req_disksize req_networks req_fips req_boot_image - req_vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "2") - req_memory=$(get_req_value "${scenario_reqs}" "min_memory" "4096") + req_vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "${DEFAULT_VM_VCPUS}") + req_memory=$(get_req_value "${scenario_reqs}" "min_memory" "${DEFAULT_VM_MEMORY}") req_disksize=$(get_req_value "${scenario_reqs}" "min_disksize" "20") req_networks=$(get_req_value "${scenario_reqs}" "networks" "default") req_fips=$(get_req_value "${scenario_reqs}" "fips" "false") @@ -507,10 +510,9 @@ register_vm() { cp "${scenario_reqs}" "${vm_dir}/state" # Add vcpus/memory/disksize from min_* values and set status - #TODO this should use the defaults, right? local vcpus memory disksize networks fips boot_image - vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "2") - memory=$(get_req_value "${scenario_reqs}" "min_memory" "4096") + vcpus=$(get_req_value "${scenario_reqs}" "min_vcpus" "${DEFAULT_VM_VCPUS}") + memory=$(get_req_value "${scenario_reqs}" "min_memory" "${DEFAULT_VM_MEMORY}") disksize=$(get_req_value "${scenario_reqs}" "min_disksize" "20") networks=$(get_req_value "${scenario_reqs}" "networks" "default") fips=$(get_req_value "${scenario_reqs}" "fips" "false") @@ -683,7 +685,7 @@ sort_scenarios_for_reuse() { local boot_image networks vcpus reuse_score boot_image=$(get_req_value "${req_file}" "boot_image" "default") networks=$(get_req_value "${req_file}" "networks" "default") - vcpus=$(get_req_value "${req_file}" "min_vcpus" "2") + vcpus=$(get_req_value "${req_file}" "min_vcpus" "${DEFAULT_VM_VCPUS}") reuse_score=$(get_reusability_score "${boot_image}" "${networks}") # Sort key: reuse_score (asc), boot_image (desc), vcpus (desc) # Use inverse vcpus (100-vcpus) so ascending sort gives descending vcpus @@ -995,8 +997,8 @@ dispatch_dynamic_scenarios() { fi local min_vcpus min_memory - min_vcpus=$(get_req_value "${req_file}" "min_vcpus" "2") - min_memory=$(get_req_value "${req_file}" "min_memory" "4096") + min_vcpus=$(get_req_value "${req_file}" "min_vcpus" "${DEFAULT_VM_VCPUS}") + min_memory=$(get_req_value "${req_file}" "min_memory" "${DEFAULT_VM_MEMORY}") # Try to find compatible free VM local vm_name="" diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh index 2fe180e9aa..839edc5f32 100644 --- a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh index 50065a7d47..002c45f162 100644 --- a/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh b/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh index 7bcd2a21b0..4a952bf206 100644 --- a/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh +++ b/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@backups.sh b/test/scenarios-bootc/presubmits/el98-src@backups.sh index cf05a806bb..8903f21941 100644 --- a/test/scenarios-bootc/presubmits/el98-src@backups.sh +++ b/test/scenarios-bootc/presubmits/el98-src@backups.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@configuration.sh b/test/scenarios-bootc/presubmits/el98-src@configuration.sh index 7970e6bf21..5862d46162 100644 --- a/test/scenarios-bootc/presubmits/el98-src@configuration.sh +++ b/test/scenarios-bootc/presubmits/el98-src@configuration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh index 49787bb6e4..6a44c6857f 100644 --- a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh +++ b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@feature-gates.sh b/test/scenarios-bootc/presubmits/el98-src@feature-gates.sh index ec7eb553d4..e9d3754c1b 100644 --- a/test/scenarios-bootc/presubmits/el98-src@feature-gates.sh +++ b/test/scenarios-bootc/presubmits/el98-src@feature-gates.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@log-scan.sh b/test/scenarios-bootc/presubmits/el98-src@log-scan.sh index 235bbea4ac..abb57fd5ca 100644 --- a/test/scenarios-bootc/presubmits/el98-src@log-scan.sh +++ b/test/scenarios-bootc/presubmits/el98-src@log-scan.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@pvc-resize.sh b/test/scenarios-bootc/presubmits/el98-src@pvc-resize.sh index a24b44364e..52228c6a43 100644 --- a/test/scenarios-bootc/presubmits/el98-src@pvc-resize.sh +++ b/test/scenarios-bootc/presubmits/el98-src@pvc-resize.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@reboot.sh b/test/scenarios-bootc/presubmits/el98-src@reboot.sh index 1d17c849b1..5e4c21a795 100644 --- a/test/scenarios-bootc/presubmits/el98-src@reboot.sh +++ b/test/scenarios-bootc/presubmits/el98-src@reboot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@show-config.sh b/test/scenarios-bootc/presubmits/el98-src@show-config.sh index e130578c14..4410438abb 100644 --- a/test/scenarios-bootc/presubmits/el98-src@show-config.sh +++ b/test/scenarios-bootc/presubmits/el98-src@show-config.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@snapshot.sh b/test/scenarios-bootc/presubmits/el98-src@snapshot.sh index c32816e638..78a3752332 100644 --- a/test/scenarios-bootc/presubmits/el98-src@snapshot.sh +++ b/test/scenarios-bootc/presubmits/el98-src@snapshot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@storage-version-migration.sh b/test/scenarios-bootc/presubmits/el98-src@storage-version-migration.sh index 6f01c4e7cc..61d996aa38 100644 --- a/test/scenarios-bootc/presubmits/el98-src@storage-version-migration.sh +++ b/test/scenarios-bootc/presubmits/el98-src@storage-version-migration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@tls-configuration.sh b/test/scenarios-bootc/presubmits/el98-src@tls-configuration.sh index 0b786e98d4..33202bf62a 100644 --- a/test/scenarios-bootc/presubmits/el98-src@tls-configuration.sh +++ b/test/scenarios-bootc/presubmits/el98-src@tls-configuration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@validate-certificate-rotation.sh b/test/scenarios-bootc/presubmits/el98-src@validate-certificate-rotation.sh index b57130c247..e75de8b99a 100644 --- a/test/scenarios-bootc/presubmits/el98-src@validate-certificate-rotation.sh +++ b/test/scenarios-bootc/presubmits/el98-src@validate-certificate-rotation.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@validate-custom-certificates.sh b/test/scenarios-bootc/presubmits/el98-src@validate-custom-certificates.sh index 960a1fdb09..ac77b6f45b 100644 --- a/test/scenarios-bootc/presubmits/el98-src@validate-custom-certificates.sh +++ b/test/scenarios-bootc/presubmits/el98-src@validate-custom-certificates.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@validate-selinux-policy.sh b/test/scenarios-bootc/presubmits/el98-src@validate-selinux-policy.sh index 3af4f08862..34ce54c337 100644 --- a/test/scenarios-bootc/presubmits/el98-src@validate-selinux-policy.sh +++ b/test/scenarios-bootc/presubmits/el98-src@validate-selinux-policy.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios-bootc/presubmits/el98-src@validate-service-account-ca-bundle.sh b/test/scenarios-bootc/presubmits/el98-src@validate-service-account-ca-bundle.sh index 1107526bd2..7087fbd523 100644 --- a/test/scenarios-bootc/presubmits/el98-src@validate-service-account-ca-bundle.sh +++ b/test/scenarios-bootc/presubmits/el98-src@validate-service-account-ca-bundle.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@auto-recovery.sh b/test/scenarios/presubmits/el96-src@auto-recovery.sh index f19c5041cb..66ecdb1e8c 100644 --- a/test/scenarios/presubmits/el96-src@auto-recovery.sh +++ b/test/scenarios/presubmits/el96-src@auto-recovery.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@containers-policy.sh b/test/scenarios/presubmits/el96-src@containers-policy.sh index c7e6125e91..29f6476963 100644 --- a/test/scenarios/presubmits/el96-src@containers-policy.sh +++ b/test/scenarios/presubmits/el96-src@containers-policy.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@feature-gates.sh b/test/scenarios/presubmits/el96-src@feature-gates.sh index f53d0def00..202aec24ca 100644 --- a/test/scenarios/presubmits/el96-src@feature-gates.sh +++ b/test/scenarios/presubmits/el96-src@feature-gates.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@log-scan.sh b/test/scenarios/presubmits/el96-src@log-scan.sh index 87a01d7ed8..168a104857 100644 --- a/test/scenarios/presubmits/el96-src@log-scan.sh +++ b/test/scenarios/presubmits/el96-src@log-scan.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@pvc-resize.sh b/test/scenarios/presubmits/el96-src@pvc-resize.sh index 93fae0d57e..d2c33b0f39 100644 --- a/test/scenarios/presubmits/el96-src@pvc-resize.sh +++ b/test/scenarios/presubmits/el96-src@pvc-resize.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@reboot.sh b/test/scenarios/presubmits/el96-src@reboot.sh index 201006febc..d2b6db3a37 100644 --- a/test/scenarios/presubmits/el96-src@reboot.sh +++ b/test/scenarios/presubmits/el96-src@reboot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@show-config.sh b/test/scenarios/presubmits/el96-src@show-config.sh index a72653c789..1f273bf948 100644 --- a/test/scenarios/presubmits/el96-src@show-config.sh +++ b/test/scenarios/presubmits/el96-src@show-config.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@snapshot.sh b/test/scenarios/presubmits/el96-src@snapshot.sh index eedc0cdbc9..73543d2db2 100644 --- a/test/scenarios/presubmits/el96-src@snapshot.sh +++ b/test/scenarios/presubmits/el96-src@snapshot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@storage-version-migration.sh b/test/scenarios/presubmits/el96-src@storage-version-migration.sh index 9a70a1cf56..421758cf0f 100644 --- a/test/scenarios/presubmits/el96-src@storage-version-migration.sh +++ b/test/scenarios/presubmits/el96-src@storage-version-migration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@tls-configuration.sh b/test/scenarios/presubmits/el96-src@tls-configuration.sh index 763ab0ae67..74ae75c39d 100644 --- a/test/scenarios/presubmits/el96-src@tls-configuration.sh +++ b/test/scenarios/presubmits/el96-src@tls-configuration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@validate-certificate-rotation.sh b/test/scenarios/presubmits/el96-src@validate-certificate-rotation.sh index 7927b4ca50..d0ca7e785b 100644 --- a/test/scenarios/presubmits/el96-src@validate-certificate-rotation.sh +++ b/test/scenarios/presubmits/el96-src@validate-certificate-rotation.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@validate-custom-certificates.sh b/test/scenarios/presubmits/el96-src@validate-custom-certificates.sh index 7927b4ca50..d0ca7e785b 100644 --- a/test/scenarios/presubmits/el96-src@validate-custom-certificates.sh +++ b/test/scenarios/presubmits/el96-src@validate-custom-certificates.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@validate-selinux-policy.sh b/test/scenarios/presubmits/el96-src@validate-selinux-policy.sh index 7f33eaa78f..d511143e7a 100644 --- a/test/scenarios/presubmits/el96-src@validate-selinux-policy.sh +++ b/test/scenarios/presubmits/el96-src@validate-selinux-policy.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el96-src@validate-service-account-ca-bundle.sh b/test/scenarios/presubmits/el96-src@validate-service-account-ca-bundle.sh index 0f6ff65916..fc9618477e 100644 --- a/test/scenarios/presubmits/el96-src@validate-service-account-ca-bundle.sh +++ b/test/scenarios/presubmits/el96-src@validate-service-account-ca-bundle.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el98-src@backup-and-restore-on-reboot.sh b/test/scenarios/presubmits/el98-src@backup-and-restore-on-reboot.sh index 695c9574bd..30e161beb7 100644 --- a/test/scenarios/presubmits/el98-src@backup-and-restore-on-reboot.sh +++ b/test/scenarios/presubmits/el98-src@backup-and-restore-on-reboot.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el98-src@backups.sh b/test/scenarios/presubmits/el98-src@backups.sh index e3a9fe8b34..9bd6866556 100644 --- a/test/scenarios/presubmits/el98-src@backups.sh +++ b/test/scenarios/presubmits/el98-src@backups.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF diff --git a/test/scenarios/presubmits/el98-src@configuration.sh b/test/scenarios/presubmits/el98-src@configuration.sh index 42cb887933..dcb3eee9df 100644 --- a/test/scenarios/presubmits/el98-src@configuration.sh +++ b/test/scenarios/presubmits/el98-src@configuration.sh @@ -8,7 +8,7 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false EOF From 9320593607c5694c69180601d7f8fd2ad5d404f0 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Fri, 6 Mar 2026 14:57:39 +0100 Subject: [PATCH 22/24] Add "slow" to scenario requirements for better reuse --- test/bin/vm_scheduler.sh | 39 +++++++++++++++---- .../presubmits/el98-src@cert-manager.sh | 1 + .../presubmits/el98-src@gateway-api.sh | 1 + .../el98-src@generic-device-plugin.sh | 1 + .../presubmits/el98-src@multi-nic.sh | 1 + .../presubmits/el98-src@olm.sh | 1 + .../presubmits/el98-src@router.sh | 3 +- .../presubmits/el98-src@upgrade-fails.sh | 3 +- .../presubmits/el98-src@multi-nic.sh | 1 + test/scenarios/presubmits/el98-src@router.sh | 3 +- .../el98-src@upgrade-fails.sh.disabled | 3 +- 11 files changed, 46 insertions(+), 11 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index f4df5a228d..f5b0fb1a4b 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -42,6 +42,22 @@ # multiple processes try to create VMs concurrently. # # ============================================================================ +# SCENARIO REQUIREMENTS +# ============================================================================ +# +# Scenarios opt-in to dynamic scheduling by defining dynamic_schedule_requirements(). +# Supported fields: +# - min_vcpus: Minimum vCPUs required (default: DEFAULT_VM_VCPUS) +# - min_memory: Minimum memory in MB (default: DEFAULT_VM_MEMORY) +# - min_disksize: Minimum disk size in GB +# - networks: Required networks (empty = network-agnostic, can run on any VM) +# - boot_image: Required boot image +# - fips: Whether FIPS mode is required +# - slow: Set to "true" for long-running tests (e.g., router, conformance) +# Slow tests get priority within their reusability tier to avoid +# becoming the critical path. +# +# ============================================================================ set -euo pipefail @@ -669,11 +685,14 @@ sort_scenarios_for_reuse() { # 1. Reusability score (ascending): restrictive scenarios first # - They can't be reused anyway, so get them out of the way # - Their VMs are destroyed when done - # 2. Within same reusability: boot_image descending (optionals before source) - # 3. Then vcpus descending (larger VMs first) + # 2. Within same reusability: slow tests first (ascending 0=slow, 1=normal) + # - Long-running tests start early to avoid becoming critical path + # 3. Then boot_image descending (optionals before source) + # 4. Then vcpus descending (larger VMs first) # # This ensures: # - Restrictive scenarios (special networks/boot_images) start first + # - Slow tests start early within their reusability tier # - Flexible scenarios stay queued longer, maximizing reuse opportunities for scenario_script in "${scenarios[@]}"; do @@ -682,19 +701,25 @@ sort_scenarios_for_reuse() { local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" if [ -f "${req_file}" ]; then - local boot_image networks vcpus reuse_score + local boot_image networks vcpus reuse_score slow_flag slow_sort boot_image=$(get_req_value "${req_file}" "boot_image" "default") networks=$(get_req_value "${req_file}" "networks" "default") vcpus=$(get_req_value "${req_file}" "min_vcpus" "${DEFAULT_VM_VCPUS}") + slow_flag=$(get_req_value "${req_file}" "slow" "false") reuse_score=$(get_reusability_score "${boot_image}" "${networks}") - # Sort key: reuse_score (asc), boot_image (desc), vcpus (desc) + # slow=true -> sort value 0 (first), slow=false -> sort value 1 (after) + slow_sort=1 + if [ "${slow_flag}" = "true" ]; then + slow_sort=0 + fi + # Sort key: reuse_score (asc), slow_sort (asc), boot_image (desc), vcpus (desc) # Use inverse vcpus (100-vcpus) so ascending sort gives descending vcpus - printf "%d\t%s\t%02d\t%s\n" "${reuse_score}" "${boot_image}" "$((100 - vcpus))" "${scenario_script}" + printf "%d\t%d\t%s\t%02d\t%s\n" "${reuse_score}" "${slow_sort}" "${boot_image}" "$((100 - vcpus))" "${scenario_script}" else # Unknown requirements - treat as flexible, put last - printf "9\taaa\t99\t%s\n" "${scenario_script}" + printf "9\t1\taaa\t99\t%s\n" "${scenario_script}" fi - done | sort -t$'\t' -k1,1n -k2,2r -k3,3n | cut -f4 + done | sort -t$'\t' -k1,1n -k2,2n -k3,3r -k4,4n | cut -f5 } # Global sequence counter for queue ordering diff --git a/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh b/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh index df0bfeea9f..3fe68e949c 100644 --- a/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh +++ b/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh @@ -26,6 +26,7 @@ min_disksize=25 networks="${NETWORKS}" boot_image=rhel98-bootc-source-optionals fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@gateway-api.sh b/test/scenarios-bootc/presubmits/el98-src@gateway-api.sh index aaef64f790..2e9210bfe4 100644 --- a/test/scenarios-bootc/presubmits/el98-src@gateway-api.sh +++ b/test/scenarios-bootc/presubmits/el98-src@gateway-api.sh @@ -26,6 +26,7 @@ min_disksize=25 networks="${NETWORKS}" boot_image=rhel98-bootc-source-optionals fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@generic-device-plugin.sh b/test/scenarios-bootc/presubmits/el98-src@generic-device-plugin.sh index 70fd303eec..8c29126863 100644 --- a/test/scenarios-bootc/presubmits/el98-src@generic-device-plugin.sh +++ b/test/scenarios-bootc/presubmits/el98-src@generic-device-plugin.sh @@ -26,6 +26,7 @@ min_disksize=25 networks="${NETWORKS}" boot_image=rhel98-bootc-source-optionals fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@multi-nic.sh b/test/scenarios-bootc/presubmits/el98-src@multi-nic.sh index 39b688ba1d..e162d2d486 100644 --- a/test/scenarios-bootc/presubmits/el98-src@multi-nic.sh +++ b/test/scenarios-bootc/presubmits/el98-src@multi-nic.sh @@ -11,6 +11,7 @@ min_disksize=20 networks=default,"${VM_MULTUS_NETWORK}" boot_image=rhel98-bootc-source fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@olm.sh b/test/scenarios-bootc/presubmits/el98-src@olm.sh index 1fa4fa030e..9032f99c22 100644 --- a/test/scenarios-bootc/presubmits/el98-src@olm.sh +++ b/test/scenarios-bootc/presubmits/el98-src@olm.sh @@ -26,6 +26,7 @@ min_disksize=25 networks="${NETWORKS}" boot_image=rhel98-bootc-source-optionals fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@router.sh b/test/scenarios-bootc/presubmits/el98-src@router.sh index db4c2d4460..9f5de38b7e 100644 --- a/test/scenarios-bootc/presubmits/el98-src@router.sh +++ b/test/scenarios-bootc/presubmits/el98-src@router.sh @@ -8,9 +8,10 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@upgrade-fails.sh b/test/scenarios-bootc/presubmits/el98-src@upgrade-fails.sh index 42125fe9e8..d91de76dce 100644 --- a/test/scenarios-bootc/presubmits/el98-src@upgrade-fails.sh +++ b/test/scenarios-bootc/presubmits/el98-src@upgrade-fails.sh @@ -8,9 +8,10 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel98-bootc-source fips=false +slow=true EOF } diff --git a/test/scenarios/presubmits/el98-src@multi-nic.sh b/test/scenarios/presubmits/el98-src@multi-nic.sh index 3f4d5d0c5c..3db0e6debd 100644 --- a/test/scenarios/presubmits/el98-src@multi-nic.sh +++ b/test/scenarios/presubmits/el98-src@multi-nic.sh @@ -11,6 +11,7 @@ min_disksize=20 networks=default,"${VM_MULTUS_NETWORK}" boot_image=rhel-9.6-microshift-source fips=false +slow=true EOF } diff --git a/test/scenarios/presubmits/el98-src@router.sh b/test/scenarios/presubmits/el98-src@router.sh index f27e9c0f08..c0d276bb2c 100644 --- a/test/scenarios/presubmits/el98-src@router.sh +++ b/test/scenarios/presubmits/el98-src@router.sh @@ -8,9 +8,10 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false +slow=true EOF } diff --git a/test/scenarios/presubmits/el98-src@upgrade-fails.sh.disabled b/test/scenarios/presubmits/el98-src@upgrade-fails.sh.disabled index 4e41dca8a9..037af92360 100644 --- a/test/scenarios/presubmits/el98-src@upgrade-fails.sh.disabled +++ b/test/scenarios/presubmits/el98-src@upgrade-fails.sh.disabled @@ -8,9 +8,10 @@ dynamic_schedule_requirements() { min_vcpus=2 min_memory=4096 min_disksize=20 -networks=default +networks= boot_image=rhel-9.6-microshift-source fips=false +slow=true EOF } From 3aef00086753d88fe5575304840bbadea02e07fd Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 10 Mar 2026 11:49:43 +0100 Subject: [PATCH 23/24] Add fast tag for scheduling plus avg vm boot time Adjust fast tags for better scheduling --- test/bin/vm_scheduler.sh | 57 ++++++++++++++++--- .../el98-src@auto-recovery-extra.sh | 1 + .../presubmits/el98-src@auto-recovery.sh | 1 + .../el98-src@backup-and-restore-on-reboot.sh | 1 + .../presubmits/el98-src@backups.sh | 1 + .../presubmits/el98-src@configuration.sh | 1 + .../presubmits/el98-src@containers-policy.sh | 1 + .../presubmits/el98-src@ipv6.sh | 1 + .../presubmits/el98-src@multus.sh | 6 +- .../presubmits/el98-src@pvc-resize.sh | 1 + .../presubmits/el98-src@show-config.sh | 1 + .../presubmits/el98-src@sriov.sh | 2 +- .../el98-src@storage-version-migration.sh | 1 + .../el98-src@validate-certificate-rotation.sh | 1 + .../el98-src@validate-selinux-policy.sh | 1 + ...-src@validate-service-account-ca-bundle.sh | 1 + .../presubmits/el98-src@version.sh | 1 + 17 files changed, 66 insertions(+), 13 deletions(-) diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index f5b0fb1a4b..56c9257f81 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -53,9 +53,12 @@ # - networks: Required networks (empty = network-agnostic, can run on any VM) # - boot_image: Required boot image # - fips: Whether FIPS mode is required -# - slow: Set to "true" for long-running tests (e.g., router, conformance) +# - slow: Set to "true" for tests longer than VM boot time (~4 min) # Slow tests get priority within their reusability tier to avoid # becoming the critical path. +# - fast: Set to "true" for tests shorter than VM boot time (~4 min) +# Fast tests stay queued longer to maximize VM reuse opportunities +# (reusing a VM saves more time than the test itself takes). # # ============================================================================ @@ -685,14 +688,16 @@ sort_scenarios_for_reuse() { # 1. Reusability score (ascending): restrictive scenarios first # - They can't be reused anyway, so get them out of the way # - Their VMs are destroyed when done - # 2. Within same reusability: slow tests first (ascending 0=slow, 1=normal) - # - Long-running tests start early to avoid becoming critical path + # 2. Within same reusability: duration priority (0=slow, 1=normal, 2=fast) + # - Slow tests (longer than VM boot) start early to avoid critical path + # - Fast tests (shorter than VM boot) stay queued for reuse opportunities # 3. Then boot_image descending (optionals before source) # 4. Then vcpus descending (larger VMs first) # # This ensures: # - Restrictive scenarios (special networks/boot_images) start first # - Slow tests start early within their reusability tier + # - Fast tests act as "gap fillers", maximizing VM reuse # - Flexible scenarios stay queued longer, maximizing reuse opportunities for scenario_script in "${scenarios[@]}"; do @@ -701,20 +706,23 @@ sort_scenarios_for_reuse() { local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" if [ -f "${req_file}" ]; then - local boot_image networks vcpus reuse_score slow_flag slow_sort + local boot_image networks vcpus reuse_score slow_flag fast_flag duration_sort boot_image=$(get_req_value "${req_file}" "boot_image" "default") networks=$(get_req_value "${req_file}" "networks" "default") vcpus=$(get_req_value "${req_file}" "min_vcpus" "${DEFAULT_VM_VCPUS}") slow_flag=$(get_req_value "${req_file}" "slow" "false") + fast_flag=$(get_req_value "${req_file}" "fast" "false") reuse_score=$(get_reusability_score "${boot_image}" "${networks}") - # slow=true -> sort value 0 (first), slow=false -> sort value 1 (after) - slow_sort=1 + # Duration priority: slow=0 (first), normal=1, fast=2 (last, best for reuse) + duration_sort=1 if [ "${slow_flag}" = "true" ]; then - slow_sort=0 + duration_sort=0 + elif [ "${fast_flag}" = "true" ]; then + duration_sort=2 fi - # Sort key: reuse_score (asc), slow_sort (asc), boot_image (desc), vcpus (desc) + # Sort key: reuse_score (asc), duration_sort (asc), boot_image (desc), vcpus (desc) # Use inverse vcpus (100-vcpus) so ascending sort gives descending vcpus - printf "%d\t%d\t%s\t%02d\t%s\n" "${reuse_score}" "${slow_sort}" "${boot_image}" "$((100 - vcpus))" "${scenario_script}" + printf "%d\t%d\t%s\t%02d\t%s\n" "${reuse_score}" "${duration_sort}" "${boot_image}" "$((100 - vcpus))" "${scenario_script}" else # Unknown requirements - treat as flexible, put last printf "9\t1\taaa\t99\t%s\n" "${scenario_script}" @@ -848,9 +856,19 @@ run_scenario_on_vm() { # Create phase has its own retry logic with timeouts log "Creating VM ${vm_name} - logging to ${boot_log}" + # Track boot time for metrics + local boot_start_time + boot_start_time=$(date +%s) + local create_exit=0 bash -x "${SCRIPTDIR}/scenario.sh" create "${scenario_script}" &> "${boot_log}" || create_exit=$? + # Record boot time + local boot_end_time boot_duration + boot_end_time=$(date +%s) + boot_duration=$((boot_end_time - boot_start_time)) + echo "${boot_duration}" > "${vm_dir}/boot_time" + if [ ${create_exit} -ne 0 ]; then result="FAILED" exit_code=1 @@ -1332,12 +1350,33 @@ show_status() { echo " Dynamic scenarios: ${dynamic_scenario_count} (passed: ${passed_scenarios}, failed: ${failed_scenarios})" echo "" + # Calculate average boot time + local total_boot_time=0 + local boot_time_count=0 + for vm_dir in "${VM_REGISTRY}"/*; do + [ -d "${vm_dir}" ] || continue + local boot_time_file="${vm_dir}/boot_time" + if [ -f "${boot_time_file}" ]; then + local bt + bt=$(cat "${boot_time_file}") + total_boot_time=$((total_boot_time + bt)) + boot_time_count=$((boot_time_count + 1)) + fi + done + local avg_boot_time=0 + local avg_boot_time_str="N/A" + if [ ${boot_time_count} -gt 0 ]; then + avg_boot_time=$((total_boot_time / boot_time_count)) + avg_boot_time_str="$(printf '%d:%02d' $((avg_boot_time / 60)) $((avg_boot_time % 60)))" + fi + echo "=== Dynamic VM Efficiency ===" echo " VMs created: ${vms_created}" echo " VM creation retries: ${vm_creation_retries}" echo " VM reuses: ${vm_reuses}" echo " Reuse rate: ${reuse_rate}%" echo " Max scenarios per VM: ${max_runs_per_vm}" + echo " Avg VM boot time: ${avg_boot_time_str} (${boot_time_count} VMs)" echo "" echo "=== Resource Configuration ===" diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh index 839edc5f32..2ada1c59b4 100644 --- a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh index 002c45f162..7322e37001 100644 --- a/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +fast=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh b/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh index 4a952bf206..5c9835689c 100644 --- a/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh +++ b/test/scenarios-bootc/presubmits/el98-src@backup-and-restore-on-reboot.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +fast=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@backups.sh b/test/scenarios-bootc/presubmits/el98-src@backups.sh index 8903f21941..bda2033818 100644 --- a/test/scenarios-bootc/presubmits/el98-src@backups.sh +++ b/test/scenarios-bootc/presubmits/el98-src@backups.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +fast=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@configuration.sh b/test/scenarios-bootc/presubmits/el98-src@configuration.sh index 5862d46162..0b7b2a492e 100644 --- a/test/scenarios-bootc/presubmits/el98-src@configuration.sh +++ b/test/scenarios-bootc/presubmits/el98-src@configuration.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +slow=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh index 6a44c6857f..c4717d214d 100644 --- a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh +++ b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh @@ -11,6 +11,7 @@ min_disksize=20 networks= boot_image=rhel98-bootc-source fips=false +fast=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@ipv6.sh b/test/scenarios-bootc/presubmits/el98-src@ipv6.sh index 39ab951c93..6f105d8fa2 100644 --- a/test/scenarios-bootc/presubmits/el98-src@ipv6.sh +++ b/test/scenarios-bootc/presubmits/el98-src@ipv6.sh @@ -23,6 +23,7 @@ min_disksize=20 networks="${VM_IPV6_NETWORK}" boot_image=rhel98-bootc-source fips=false +fast=true EOF } diff --git a/test/scenarios-bootc/presubmits/el98-src@multus.sh b/test/scenarios-bootc/presubmits/el98-src@multus.sh index 601b31a646..da78b58ac7 100644 --- a/test/scenarios-bootc/presubmits/el98-src@multus.sh +++ b/test/scenarios-bootc/presubmits/el98-src@multus.sh @@ -20,10 +20,10 @@ fi # Opt-in to dynamic VM scheduling by declaring requirements dynamic_schedule_requirements() { cat < Date: Wed, 18 Mar 2026 13:46:28 +0100 Subject: [PATCH 24/24] verify nits --- test/bin/scenario.sh | 2 +- test/bin/vm_scheduler.sh | 111 ++++++++++-------- .../el98-src@auto-recovery-extra.sh | 2 +- .../presubmits/el98-src@cert-manager.sh | 2 +- .../presubmits/el98-src@containers-policy.sh | 2 +- .../presubmits/el98-src@etcd.sh | 72 +----------- .../presubmits/el98-src@feature-gates.sh | 2 +- .../presubmits/el98-src@gateway-api.sh | 2 +- .../el98-src@generic-device-plugin.sh | 2 +- .../el98-src@healthchecks-disabled-service.sh | 2 +- .../presubmits/el98-src@hostname.sh | 74 +----------- .../presubmits/el98-src@kustomize.sh | 74 +----------- .../presubmits/el98-src@log-scan.sh | 2 +- .../presubmits/el98-src@multus.sh | 2 +- .../presubmits/el98-src@networking-smoke.sh | 74 +----------- .../presubmits/el98-src@observability.sh | 2 +- .../presubmits/el98-src@olm.sh | 2 +- .../presubmits/el98-src@pvc-resize.sh | 2 +- .../presubmits/el98-src@show-config.sh | 2 +- .../presubmits/el98-src@sriov.sh | 2 +- .../el98-src@storage-version-migration.sh | 2 +- .../presubmits/el98-src@tls-configuration.sh | 2 +- .../el98-src@validate-certificate-rotation.sh | 2 +- .../el98-src@validate-custom-certificates.sh | 2 +- .../el98-src@validate-selinux-policy.sh | 2 +- ...-src@validate-service-account-ca-bundle.sh | 2 +- .../presubmits/el98-src@version.sh | 73 +----------- ...98-zprel@el98-lrel@rpm-upgrade.sh.disabled | 2 +- 28 files changed, 90 insertions(+), 432 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 93d6a8ae70..49f9bb5e9e 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -559,7 +559,7 @@ function get_vm_ip { ip=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${vmname}" | head -1) while true; do now=$(date +%s) - if [ $(( now - start )) -ge ${VM_BOOT_TIMEOUT} ]; then + if [ $(( now - start )) -ge "${VM_BOOT_TIMEOUT}" ]; then echo "Timed out while waiting for IP retrieval" >&2 return 1 fi diff --git a/test/bin/vm_scheduler.sh b/test/bin/vm_scheduler.sh index 56c9257f81..52b5707929 100644 --- a/test/bin/vm_scheduler.sh +++ b/test/bin/vm_scheduler.sh @@ -663,7 +663,7 @@ get_reusability_score() { # Score 2: Reusable - optionals can be reused by source scenarios case "${boot_image}" in - *-optionals|*-optional|*-with-optional) + *-optionals|*-with-optional) echo "2" return ;; @@ -1412,65 +1412,72 @@ show_status() { echo "=== Test Durations ===" echo " (Tests longer than avg boot time should be 'slow', shorter should be 'fast')" echo "" - # Collect test durations and check for mislabeling - local mislabeled_count=0 - { - for scenario_dir in "${SCENARIO_INFO_DIR}"/*; do - [ -d "${scenario_dir}" ] || continue - local scenario_name - scenario_name=$(basename "${scenario_dir}") - local test_time_file="${scenario_dir}/test_time" - - if [ -f "${test_time_file}" ]; then - local test_time - test_time=$(cat "${test_time_file}") - local test_mins=$((test_time / 60)) - local test_secs=$((test_time % 60)) - local time_str - time_str=$(printf '%2d:%02d' "${test_mins}" "${test_secs}") - - # Get slow/fast flag from requirements - local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" - local slow_flag="false" - local fast_flag="false" - local current_label="-" - if [ -f "${req_file}" ]; then - slow_flag=$(get_req_value "${req_file}" "slow" "false") - fast_flag=$(get_req_value "${req_file}" "fast" "false") - if [ "${slow_flag}" = "true" ]; then - current_label="slow" - elif [ "${fast_flag}" = "true" ]; then - current_label="fast" - fi + # Collect test durations to a temp file for sorting and counting + local tmp_durations + tmp_durations=$(mktemp) + # shellcheck disable=SC2064 + trap "rm -f ${tmp_durations}" RETURN + + for scenario_dir in "${SCENARIO_INFO_DIR}"/*; do + [ -d "${scenario_dir}" ] || continue + local scenario_name + scenario_name=$(basename "${scenario_dir}") + local test_time_file="${scenario_dir}/test_time" + + if [ -f "${test_time_file}" ]; then + local test_time + test_time=$(cat "${test_time_file}") + local test_mins=$((test_time / 60)) + local test_secs=$((test_time % 60)) + local time_str + time_str=$(printf '%2d:%02d' "${test_mins}" "${test_secs}") + + # Get slow/fast flag from requirements + local req_file="${SCENARIO_STATUS}/${scenario_name}/requirements" + local slow_flag="false" + local fast_flag="false" + local current_label="-" + if [ -f "${req_file}" ]; then + slow_flag=$(get_req_value "${req_file}" "slow" "false") + fast_flag=$(get_req_value "${req_file}" "fast" "false") + if [ "${slow_flag}" = "true" ]; then + current_label="slow" + elif [ "${fast_flag}" = "true" ]; then + current_label="fast" fi + fi - # Determine if mislabeled (compare against avg boot time) - local status="" - if [ ${avg_boot_time} -gt 0 ]; then - if [ ${test_time} -gt ${avg_boot_time} ]; then - # Test is slower than boot - should be 'slow' - if [ "${fast_flag}" = "true" ]; then - status="<-- MISLABELED (should be slow)" - mislabeled_count=$((mislabeled_count + 1)) - fi - else - # Test is faster than boot - should be 'fast' - if [ "${slow_flag}" = "true" ]; then - status="<-- MISLABELED (should be fast)" - mislabeled_count=$((mislabeled_count + 1)) - fi + # Determine if mislabeled (compare against avg boot time) + local status="" + if [ "${avg_boot_time}" -gt 0 ]; then + if [ "${test_time}" -gt "${avg_boot_time}" ]; then + # Test is slower than boot - should be 'slow' + if [ "${fast_flag}" = "true" ]; then + status="<-- MISLABELED (should be slow)" + fi + else + # Test is faster than boot - should be 'fast' + if [ "${slow_flag}" = "true" ]; then + status="<-- MISLABELED (should be fast)" fi fi - - # Output: time, scenario, label, status (tab-separated for sorting) - printf "%d\t%s\t%-40s\t%-6s\t%s\n" "${test_time}" "${time_str}" "${scenario_name}" "${current_label}" "${status}" fi - done - } | sort -t$'\t' -k1,1rn | cut -f2- | while IFS=$'\t' read -r time_str scenario label status; do + + # Output: time, scenario, label, status (tab-separated for sorting) + printf "%d\t%s\t%-40s\t%-6s\t%s\n" "${test_time}" "${time_str}" "${scenario_name}" "${current_label}" "${status}" >> "${tmp_durations}" + fi + done + + # Sort and display + sort -t$'\t' -k1,1rn "${tmp_durations}" | cut -f2- | while IFS=$'\t' read -r time_str scenario label status; do printf " %s %-40s [%s] %s\n" "${time_str}" "${scenario}" "${label}" "${status}" done + + # Count mislabeled + local mislabeled_count + mislabeled_count=$(grep -c "MISLABELED" "${tmp_durations}" 2>/dev/null || echo 0) echo "" - if [ ${mislabeled_count} -gt 0 ]; then + if [ "${mislabeled_count}" -gt 0 ]; then echo " WARNING: ${mislabeled_count} test(s) appear to be mislabeled based on avg boot time (${avg_boot_time_str})" fi echo "" diff --git a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh index 2ada1c59b4..ca6982524f 100644 --- a/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh +++ b/test/scenarios-bootc/presubmits/el98-src@auto-recovery-extra.sh @@ -17,7 +17,7 @@ EOF scenario_create_vms() { prepare_kickstart host1 kickstart-bootc.ks.template rhel98-bootc-source - launch_vm --boot_blueprint rhel98-bootc + launch_vm rhel98-bootc } scenario_remove_vms() { diff --git a/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh b/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh index 3fe68e949c..0aa69e3e0a 100644 --- a/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh +++ b/test/scenarios-bootc/presubmits/el98-src@cert-manager.sh @@ -33,7 +33,7 @@ EOF scenario_create_vms() { LVM_SYSROOT_SIZE=20480 prepare_kickstart host1 kickstart-bootc.ks.template rhel98-bootc-source-optionals # Three nics - one for sriov, one for macvlan, another for ipvlan (they cannot enslave the same interface) - launch_vm --boot_blueprint rhel98-bootc --network "${NETWORKS}" --vm_disksize 25 --vm_vcpus 4 + launch_vm rhel98-bootc --network "${NETWORKS}" --vm_disksize 25 --vm_vcpus 4 } scenario_remove_vms() { diff --git a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh index c4717d214d..569cdc54f4 100644 --- a/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh +++ b/test/scenarios-bootc/presubmits/el98-src@containers-policy.sh @@ -17,7 +17,7 @@ EOF scenario_create_vms() { prepare_kickstart host1 kickstart-bootc.ks.template rhel98-bootc-source - launch_vm --boot_blueprint rhel98-bootc + launch_vm rhel98-bootc } scenario_remove_vms() { diff --git a/test/scenarios-bootc/presubmits/el98-src@etcd.sh b/test/scenarios-bootc/presubmits/el98-src@etcd.sh index cd5ab1278a..bf4b64f66e 100644 --- a/test/scenarios-bootc/presubmits/el98-src@etcd.sh +++ b/test/scenarios-bootc/presubmits/el98-src@etcd.sh @@ -16,7 +16,7 @@ EOF scenario_create_vms() { prepare_kickstart host1 kickstart-bootc.ks.template rhel98-bootc-source - launch_vm --boot_blueprint rhel98-bootc + launch_vm rhel98-bootc } scenario_remove_vms() { @@ -24,76 +24,6 @@ scenario_remove_vms() { } scenario_run_tests() { - # The SYNC_FREQUENCY is set to a shorter-than-default value to speed up - # pre-submit scenario completion time in DNS tests. run_tests host1 \ - --variable "EXPECTED_OS_VERSION:9.8" \ - --variable "SYNC_FREQUENCY:5s" \ - suites/standard1/dns.robot -} -#!/bin/bash - -# Sourced from scenario.sh and uses functions defined there. - -# Opt-in to dynamic VM scheduling by declaring requirements -dynamic_schedule_requirements() { - cat <