ci: DH-19408: PNAP Integration

.github/resources/adhoc-scale-benchmark.properties

@@ -39,7 +39,7 @@ timestamp.test.results=
 
 # Experimental: Docker compose file (e.g. /mypath/docker-compose.yml)
 # Empty means no docker restart attempt will be made
-docker.compose.file=/root/deephaven/docker-compose.yml
+docker.compose.file=${userHome}/deephaven/docker-compose.yml
 
 # The url used for posting messages to slack
 slack.token=

.github/resources/adhoc-server-deploy.json

@@ -0,0 +1,14 @@
+{
+  "hostname": "my-server",
+  "description": "Ephemeral adhoc benchmark server",
+  "os": "ubuntu/noble",
+  "type": "s2.c2.small",
+  "location": "CHI",
+  "installDefaultSshKeys": true,
+  "publicIpAddresses": [
+    {
+      "type": "PRIMARY"
+    }
+  ]
+}
+

.github/resources/compare-scale-benchmark.properties

@@ -39,5 +39,5 @@ timestamp.test.results=
 
 # Experimental: Docker compose file (e.g. /mypath/docker-compose.yml)
 # Empty means no docker restart attempt will be made
-docker.compose.file=/root/deephaven/docker-compose.yml
+docker.compose.file=${userHome}/deephaven/docker-compose.yml
 

.github/resources/nightly-scale-benchmark.properties

@@ -39,7 +39,7 @@ timestamp.test.results=
 
 # Experimental: Docker compose file (e.g. /mypath/docker-compose.yml)
 # Empty means no docker restart attempt will be made
-docker.compose.file=/root/deephaven/docker-compose.yml
+docker.compose.file=${userHome}/deephaven/docker-compose.yml
 
 # The url used for posting messages to slack
 slack.token=${slackToken}

.github/resources/release-scale-benchmark.properties

@@ -39,4 +39,4 @@ timestamp.test.results=
 
 # Experimental: Docker compose file (e.g. /mypath/docker-compose.yml)
 # Empty means no docker restart attempt will be made
-docker.compose.file=/root/deephaven/docker-compose.yml
+docker.compose.file=${userHome}/deephaven/docker-compose.yml

.github/scripts/adhoc.sh

@@ -3,37 +3,34 @@
 set -o errexit
 set -o pipefail
 
-# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
 
 # Provides what is needed to set up an adhoc benchmark run, including bare metal and labels
 # ex. adhoc.sh make-labels "where" "0.36.0" "user123:branch-name-123"
-# ex. adhoc.sh metal-deploy api-key project-id c3.small.x86 server-name "2 days" 
-# ex. adhoc.sh metal-delete api-key device-id service-name
+# ex. adhoc.sh scale-nums 10 5
+# ex. adhoc.sh deploy-metal api-key project-id s2.c2.small server-name
+# ex. adhoc.sh delete-metal api-key project-id device-id server-name
+# ex. adhoc.sh purge-metal api-key project-id
 
-if [[ $# < 2 ]]; then
+if [[ $# -lt 2 ]]; then
   echo "$0: Missing action or its arguments"
   exit 1
 fi
 
 ACTION=$1
 SCRIPT_DIR=$(dirname "$0")
 OUTPUT_NAME=adhoc-${ACTION}.out
+SERVER_NAME_PREFIX="adhoc-"
 
 rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}
 
-# Get metal device info including ip address
-getDeviceInfo() {
-  curl --no-progress-meter --max-time 10 -X GET -H "X-Auth-Token: $1" \
-    "https://api.equinix.com/metal/v1/devices/$2?include=ip_addresses,state&exclude=root_password,ssh_keys" \
-    | jq | tee get-device-response.json | jq -r "$3"
-}
-
 # Get the label part of an image/branch name
 # ex. edge@sha256:15ab331629805076cdf5ed6666186c6b578298ab493a980779338d153214640e
 # ex. user123:1111-my-pull-request
 # ex. 0.36.0 or edge
 getSetLabel() {
-  SUFFIX=$2
+  PREFIX="$1"
+  SUFFIX="$2"
   if [[ $2 == *"@sha"*":"* ]]; then
     SUFFIX=$(echo "$2" | sed 's/@sha.*:/_/g' | head -c 20)
   elif [[ $2 == *":"* ]]; then
@@ -42,75 +39,113 @@ getSetLabel() {
   echo "${PREFIX}_${SUFFIX}" | sed -E 's/(^[0-9])/_\1/g' | sed 's/[^0-9a-zA-Z_]/_/g'
 }
 
+getApiToken() {
+  curl -s -X POST "https://auth.phoenixnap.com/auth/realms/BMC/protocol/openid-connect/token" \
+    -H "Content-Type: application/x-www-form-urlencoded" --data-urlencode "grant_type=client_credentials" \
+    --data-urlencode "client_id=$1" --data-urlencode "client_secret=$2" | jq -r '.access_token'
+}
+
+deleteMetal() {
+    TOKEN="$1"
+    DEVICE_ID="$2"
+    DEVICE_NAME="$3"
+    echo "Deleting server ${DEVICE_NAME}"
+    RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer ${TOKEN}" "https://api.phoenixnap.com/bmc/v1/servers/${DEVICE_ID}")
+    CURL_EXIT=$?
+    if (( ${CURL_EXIT} != 0 )); then
+        echo "Failed deleting server ${DEVICE_NAME} with exit code ${CURL_EXIT}"
+        exit 1
+    fi
+    if (( ${RESP_CODE} != 404 && (${RESP_CODE} < 200 || ${RESP_CODE} >= 300) )); then
+        echo "Failed deleting server ${DEVICE_NAME} with http code ${RESP_CODE}"
+        exit 1
+    fi
+    echo "Successfully deleted server ${DEVICE_NAME}"
+}
+
+
 # Make set labels from a prefix and image/branch names
 if [[ ${ACTION} == "make-labels" ]]; then
   PREFIX=$2
   IMAGE1=$3
   IMAGE2=$4
   echo "Making Labels: ${PREFIX}"
-  
+
   LABEL1=$(getSetLabel ${PREFIX} ${IMAGE1})
   LABEL2=$(getSetLabel ${PREFIX} ${IMAGE2})
-  
+
   echo "PREFIX=${PREFIX}" | tee -a ${OUTPUT_NAME}
   echo "SET_LABEL_1=${LABEL1}" | tee -a ${OUTPUT_NAME}
   echo "SET_LABEL_2=${LABEL2}" | tee -a ${OUTPUT_NAME}
 fi
 
-# Format some number used for scaling the tests
+# Format some numbers used for scaling the tests
 if [[ ${ACTION} == "scale-nums" ]]; then
   INPUT_ROW_COUNT=$2
   INPUT_ITERATIONS=$3
   echo "Scaling Numbers"
-  
+
   TEST_ROW_COUNT=$((${INPUT_ROW_COUNT} * 1000000))
   TEST_ITERATIONS=${INPUT_ITERATIONS}
   if [ $((${INPUT_ITERATIONS} % 2)) == 0 ]; then
 	TEST_ITERATIONS=$((${INPUT_ITERATIONS} + 1))
   fi
-  
+
   echo "INPUT_ROW_COUNT=${INPUT_ROW_COUNT}" | tee -a ${OUTPUT_NAME}
   echo "INPUT_ITERATIONS=${INPUT_ITERATIONS}" | tee -a ${OUTPUT_NAME}
   echo "TEST_ROW_COUNT=${TEST_ROW_COUNT}" | tee -a ${OUTPUT_NAME}
   echo "TEST_ITERATIONS=${TEST_ITERATIONS}" | tee -a ${OUTPUT_NAME}
 fi
 
-# Deploy a bare metal server using the Equinix ReST API
+# Deploy a bare metal server using the Phoenix NAP REST API
 if [[ ${ACTION} == "deploy-metal" ]]; then
   API_KEY=$2
   PROJECT_ID=$3
   PLAN=$4
-  ACTOR=$(echo "adhoc-$5-"$(${SCRIPT_DIR}/base.sh $(date +%s%03N) 36) | tr '[:upper:]' '[:lower:]')
-  EXPIRE_WHEN=$6
+  ACTOR=$(echo "${SERVER_NAME_PREFIX}$5-"$(${SCRIPT_DIR}/base.sh $(date +%s%03N) 36) | tr '[:upper:]' '[:lower:]')
   echo "Deploying Server: ${ACTOR}"
-
   BEGIN_SECS=$(date +%s)
-  DEVICE_ID=$(curl --fail-with-body -X POST \
-    -H "Content-Type: application/json" -H "X-Auth-Token: ${API_KEY}" \
-    "https://api.equinix.com/metal/v1/projects/${PROJECT_ID}/devices?exclude=plan,ssh_keys,provisioning_events,network_ports,operating_system" \
-    -d '{
-      "metro": "sv",
-      "plan": "'${PLAN}'",
-      "operating_system": "ubuntu_22_04",
-      "hostname": "'${ACTOR}'",
-      "termination_time": "'$(date --iso-8601=seconds -d "+${EXPIRE_WHEN}")'"
-    }' | jq | tee create-device-response.json | jq -r '.id')
-
-  IP_ADDRESS="null"
-  for i in $(seq 100); do
-    echo -n "$i) Device Status: "
-    STATE=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".state")
-    echo "${STATE}"
-    if [[ "${STATE}" == "active" ]]; then break; fi
-    sleep 6
+
+  pushd ${SCRIPT_DIR}/../resources
+  TOKEN=$(getApiToken "${PROJECT_ID}" "${API_KEY}")
+
+  echo "Making Deploy POST"
+  jq --arg hostname "${ACTOR}" --arg plan "${PLAN}" \
+   '.hostname = $hostname | .type = $plan' adhoc-server-deploy.json > adhoc-server-deploy-final.json
+  echo "Finished Deploy POST"
+
+  echo "Running Deploy API"
+  RESPONSE=$(curl -s -X POST "https://api.phoenixnap.com/bmc/v1/servers" -H "Authorization: Bearer ${TOKEN}" \
+    -H "Content-Type: application/json" -d @adhoc-server-deploy-final.json)
+
+  IP_ADDRESS=$(jq -r '.publicIpAddresses[0] // empty' <<< "${RESPONSE}")
+  DEVICE_ID=$(jq -r '.id // empty' <<< "${RESPONSE}")
+  if [[ -z "$IP_ADDRESS" || -z "$DEVICE_ID" ]]; then
+    echo "Failed Getting IP Address and Device ID"
+    exit 1
+  else
+    echo "Got Address ${IP_ADDRESS} and Device Id ${DEVICE_ID}"
+  fi
+  popd
+
+  echo "Waiting for Server Ready"
+  STATUS=0
+  for i in {1..60}; do
+    STATUS_RESPONSE=$(curl -s -H "Authorization: Bearer ${TOKEN}" "https://api.phoenixnap.com/bmc/v1/servers/${DEVICE_ID}")
+    SERVER_STATUS=$(echo "${STATUS_RESPONSE}" | jq -r '.status')
+    if [[ "${SERVER_STATUS}" == "powered-on" ]]; then
+      if nc -z -w 2 "${IP_ADDRESS}" 22 >/dev/null 2>&1; then
+        STATUS=1
+        break
+      fi
+    fi
+    sleep 10
   done
-
+  
   DURATION=$(($(date +%s) - ${BEGIN_SECS}))
-
-  IP_ADDRESS=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".ip_addresses[0].address")
-  STATE=$(getDeviceInfo ${API_KEY} ${DEVICE_ID} ".state")
-  if [[ "${IP_ADDRESS}" == "null" ]] || [[ "${STATE}" != "active" ]]; then
-    echo "Failed to provision device after ${DURATION} seconds"
+  if [[ ${STATUS} -eq 0 ]]; then
+    echo "Failed to provision device ${ACTOR} after ${DURATION} seconds"
+    echo "Last provision attempt status: ${SERVER_STATUS}"
     exit 1
   fi
 
@@ -119,23 +154,50 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
   echo "DEVICE_NAME=${ACTOR}" | tee -a ${OUTPUT_NAME}
   echo "DEVICE_ID=${DEVICE_ID}" | tee -a ${OUTPUT_NAME}
   echo "DEVICE_ADDR=${IP_ADDRESS}" | tee -a ${OUTPUT_NAME}
-  echo "DEVICE_EXPIRE=${EXPIRE_WHEN}" | tee -a ${OUTPUT_NAME}
 fi
 
-# Delete a bare metal server using the Equlinix ReST API
+# Delete a bare metal server with the given id
 if [[ ${ACTION} == "delete-metal" ]]; then
   API_KEY=$2
-  DEVICE_ID=$3
-  DEVICE_NAME=$4
-
-  curl --no-progress-meter --max-time 10 --fail-with-body -X DELETE -H "X-Auth-Token: ${API_KEY}" \
-    "https://api.equinix.com/metal/v1/devices/${DEVICE_ID}" \
-    | jq | tee delete-device-response.json 
+  PROJECT_ID=$3
+  DEVICE_ID=$4
+  DEVICE_NAME=$5
 
+  TOKEN=$(getApiToken "${PROJECT_ID}" "${API_KEY}")
+  echo "Deleting Server ${DEVICE_NAME}"
+  deleteMetal ${TOKEN} ${DEVICE_ID} ${DEVICE_NAME}
+
   echo "ACTION=${ACTION}" | tee -a ${OUTPUT_NAME}
   echo "DEVICE_NAME=${DEVICE_NAME}" | tee -a ${OUTPUT_NAME}
   echo "DEVICE_ID=${DEVICE_ID}" | tee -a ${OUTPUT_NAME}
 fi
 
-
+# Purge all ephemeral metal that's past its expiration date
+if [[ ${ACTION} == "purge-metal" ]]; then
+  API_KEY=$2
+  PROJECT_ID=$3
+  EXPIRATION_HOURS=24
+
+  echo "Starting Ephemeral Server Cleanup"
+  echo "Max Hours to Expiration: ${EXPIRATION_HOURS}"
+  TOKEN=$(getApiToken "${PROJECT_ID}" "${API_KEY}")
+  THRESHOLD=$(( EXPIRATION_HOURS * 3600 ))
+  NOW=$(date +%s)
+
+  servers=$(curl -s -H "Authorization: Bearer ${TOKEN}" https://api.phoenixnap.com/bmc/v1/servers)
+  echo "$servers" | jq -c '.[]' | while read server; do
+    id=$(echo "$server" | jq -r '.id')
+    name=$(echo "$server" | jq -r '.hostname')
+    created=$(echo "$server" | jq -r '.provisionedOn')
+    [ "$created" = "null" ] && continue
+    created_epoch=$(date -d "$created" +%s)
+    age_seconds=$(( NOW - created_epoch ))
+    echo "Found Server $name Aged $(( age_seconds / 3600 )) Hours"
+
+    if [[ "$name" == "${SERVER_NAME_PREFIX}"* ]] && (( age_seconds > THRESHOLD )); then
+      deleteMetal ${TOKEN} $id $name
+    fi
+  done
+  echo "ACTION=${ACTION}" | tee -a ${OUTPUT_NAME}
+fi
 

.github/scripts/build-benchmark-artifact-remote.sh

@@ -4,13 +4,15 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
+
 # Build benchmark artifact on the remote side
 # Assumes git branch is available and docker is running
 
 HOST=`hostname`
-GIT_DIR=/root/git/benchmark
-RUN_DIR=/root/run
-DEEPHAVEN_DIR=/root/deephaven
+GIT_DIR=${HOME}/git/benchmark
+RUN_DIR=${HOME}/run
+DEEPHAVEN_DIR=${HOME}/deephaven
 
 if [ ! -d "${GIT_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"
@@ -28,10 +30,10 @@ mvn verify
 title "-- Cleanup After Build  --"
 cd ${DEEPHAVEN_DIR};
 docker compose down
-rm -f data/*.*
+sudo rm -f data/*.*
 
 title "-- Copying Artifact and Tests to Run Directory --"
-rm -rf ${RUN_DIR}
+sudo rm -rf ${RUN_DIR}
 mkdir -p ${RUN_DIR}/
 cp ${GIT_DIR}/target/deephaven-benchmark-*.jar ${RUN_DIR}/
 mv ${RUN_DIR}/deephaven-benchmark-*-tests.jar ${RUN_DIR}/standard-tests.jar

.github/scripts/build-docker-image-remote.sh

@@ -3,12 +3,14 @@
 set -o errexit
 set -o pipefail
 
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
+
 # Build a local docker image on the remote side
 # Ensure the docker image is running in the Deephaven directory
 
 HOST=`hostname`
-GIT_DIR=/root/git
-DEEPHAVEN_DIR=/root/deephaven
+GIT_DIR=${HOME}/git
+DEEPHAVEN_DIR=${HOME}/deephaven
 DEEPHAVEN_VERSION_FILE=${GIT_DIR}/deephaven-core/build/version
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then

.github/scripts/build-server-distribution-remote.sh

@@ -3,7 +3,7 @@
 set -o errexit
 set -o pipefail
 
-# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
 
 # Assemble the Deephaven server artifacts on the remote side if needed
 # The supplied argument can be an image name or <owner>::<branch>
@@ -15,8 +15,8 @@ if [[ $# != 1 ]]; then
 fi
 
 HOST=`hostname`
-GIT_DIR=/root/git
-DEEPHAVEN_DIR=/root/deephaven
+GIT_DIR=${HOME}/git
+DEEPHAVEN_DIR=${HOME}/deephaven
 DOCKER_IMG=$1
 BRANCH_DELIM=":"
 BUILD_JAVA=temurin-17-jdk-amd64

.github/scripts/fetch-results-local.sh

@@ -22,7 +22,7 @@ RUN_TYPE=$4
 ACTOR=$5
 SET_LABEL=${6:-$(echo -n "set-"; ${SCRIPT_DIR}/base.sh $(date +%s%03N) 62)}
 DOCKER_IMG=$7
-RUN_DIR=/root/run
+RUN_DIR=/home/${USER}/run
 OUTPUT_NAME=fetch-results-local.out
 
 rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}

.github/scripts/manage-deephaven-remote.sh

@@ -3,7 +3,7 @@
 set -o errexit
 set -o pipefail
 
-# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
 
 # Start or Stop a Deephaven image based on the given directive and image/branch name
 # The directives argument can be start or stop
@@ -18,7 +18,7 @@ DIRECTIVE=$1
 DOCKER_IMG=$2
 CONFIG_OPTS="${@:3}"
 HOST=`hostname`
-DEEPHAVEN_DIR=/root/deephaven
+DEEPHAVEN_DIR=${HOME}/deephaven
 
 if [ ! -d "${DEEPHAVEN_DIR}" ]; then
   echo "$0: Missing one or more Benchmark setup directories"