deploy.sh

#!/usr/bin/env bash
# Deploy server-monitor to remote servers over SSH.

set -Eeuo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOCAL_ENV_FILE="${SCRIPT_DIR}/.env"
SERVER_ENV_EXAMPLE="${SCRIPT_DIR}/.env.server.example"
SERVER_ENV_DEFAULTS="${SCRIPT_DIR}/.env.server"
GENERATED_ENV_DIR="${SCRIPT_DIR}/.deploy"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BOLD='\033[1m'
RESET='\033[0m'

log_info()  { echo -e "${CYAN}[INFO]${RESET}  $*"; }
log_ok()    { echo -e "${GREEN}[OK]${RESET}    $*"; }
log_warn()  { echo -e "${YELLOW}[WARN]${RESET}  $*"; }
log_error() { echo -e "${RED}[ERR]${RESET}   $*"; }
log_step()  { echo -e "  -> $*"; }
print_heading() { printf '\n%b\n' "${BOLD}$*${RESET}"; }

ssh_run() {
  local target="$1"
  local command="$2"
  ssh "$target" "bash -lc '${command//\'/\'\\\'\'}'"
}

ssh_sudo() {
  local target="$1"
  local password="$2"
  local command="$3"

  if [ -n "$password" ]; then
    ssh "$target" "printf '%s\\n' '${password//\'/\'\\\'\'}' | sudo -S bash -lc '${command//\'/\'\\\'\'}' 2>&1 | sed '/^\\[sudo\\]/d'"
  else
    ssh "$target" "sudo -n bash -lc '${command//\'/\'\\\'\'}'"
  fi
}

source_if_present() {
  local file="$1"
  if [ -f "$file" ]; then
    # shellcheck disable=SC1090
    source "$file"
  fi
}

env_escape() {
  printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
}

format_prompt_default() {
  local value="${1:-}"
  if [ -n "$value" ]; then
    printf 'current: %s' "$value"
  else
    printf 'empty'
  fi
}

bool_normalize() {
  case "${1:-false}" in
    true|TRUE|yes|YES|y|Y|1|on|ON) echo true ;;
    *) echo false ;;
  esac
}

prompt_text() {
  local __var="$1"
  local label="$2"
  local default_value="${3:-}"
  local secret="${4:-false}"
  local reply=""

  if [ "$INTERACTIVE_MODE" = false ]; then
    printf -v "$__var" '%s' "$default_value"
    return
  fi

  if [ "$secret" = true ]; then
    if [ -n "$default_value" ]; then
      read -rsp "${label} [configured]: " reply
    else
      read -rsp "${label} [empty]: " reply
    fi
    echo ""
  else
    read -rp "${label} [$(format_prompt_default "$default_value")]: " reply
  fi

  if [ -z "$reply" ]; then
    reply="$default_value"
  fi
  printf -v "$__var" '%s' "$reply"
}

prompt_bool() {
  local __var="$1"
  local label="$2"
  local default_value
  local reply=""
  local normalized=""

  default_value="$(bool_normalize "${3:-false}")"
  if [ "$INTERACTIVE_MODE" = false ]; then
    printf -v "$__var" '%s' "$default_value"
    return
  fi

  while true; do
    read -rp "${label} [current: ${default_value}] (true/false): " reply
    reply="${reply:-$default_value}"
    case "$reply" in
      true|TRUE|yes|YES|y|Y|1|on|ON|false|FALSE|no|NO|n|N|0|off|OFF)
        normalized="$(bool_normalize "$reply")"
        printf -v "$__var" '%s' "$normalized"
        return
        ;;
    esac
    log_warn "Enter true or false."
  done
}

prompt_number() {
  local __var="$1"
  local label="$2"
  local default_value="$3"
  local reply=""

  if [ "$INTERACTIVE_MODE" = false ]; then
    printf -v "$__var" '%s' "$default_value"
    return
  fi

  while true; do
    read -rp "${label} [current: ${default_value}]: " reply
    reply="${reply:-$default_value}"
    if [[ "$reply" =~ ^[0-9]+$ ]]; then
      printf -v "$__var" '%s' "$reply"
      return
    fi
    log_warn "Enter a whole number."
  done
}

prompt_services_selection() {
  local ssh_target="$1"
  local label="$2"
  local existing="$3"
  local default_services reply token
  local -a services default_array selected available_array

  mapfile -t services < <(ssh_run "$ssh_target" "systemctl list-unit-files --type=service --state=enabled --no-legend | awk '{print \$1}' | sed 's/\.service\$//' | grep -Ev '^(getty|autovt|systemd-|dbus|polkit|rsyslog|cron|atd|udev|snapd|apparmor|lvm2|mdadm|open-iscsi|multipathd|plymouth|cloud-init|e2scrub|ua-|ubuntu-advantage|pollinate|secureboot-db|vgauth|networkd-dispatcher|open-vm-tools|dmesg)' | grep -v '@' | sort -u" 2>/dev/null || true)

  if [ "${#services[@]}" -eq 0 ]; then
    if [ "$INTERACTIVE_MODE" = true ]; then
      log_warn "No services found on remote (or all filtered). Enter names manually below."
    fi
    SERVICES_TO_MONITOR="${existing:-}"
    if [ "$INTERACTIVE_MODE" = false ]; then
      return
    fi
  fi

  mapfile -t default_array < <(printf '%s\n' "${services[@]}" | grep -E '^(nginx|apache2|caddy|mysql|mariadb|postgres|postgresql|redis|memcached|php.*-fpm|docker|containerd|supervisord|supervisor|pm2|queue|worker|laravel|node|gunicorn|uvicorn)$' || true)
  if [ "${#default_array[@]}" -eq 0 ]; then
    default_array=("${services[@]:0:5}")
  fi
  default_services="${default_array[*]}"
  [ -n "$existing" ] && default_services="$existing"

  if [ "$INTERACTIVE_MODE" = false ]; then
    SERVICES_TO_MONITOR="$default_services"
    return
  fi

  print_heading "$label"
  for i in "${!services[@]}"; do
    printf '  %2d. %s\n' "$((i + 1))" "${services[$i]}"
  done
  echo "  all. monitor every listed service"
  echo "  none. disable service tracking targets"
  echo ""
  read -rp "Select services to monitor [$(format_prompt_default "$default_services")]: " reply
  reply="${reply:-$default_services}"

  case "$reply" in
    all)
      SERVICES_TO_MONITOR="${services[*]}"
      return
      ;;
    none)
      SERVICES_TO_MONITOR=""
      return
      ;;
  esac

  selected=()
  for token in ${reply//,/ }; do
    if [[ "$token" =~ ^[0-9]+$ ]] && [ "$token" -ge 1 ] && [ "$token" -le "${#services[@]}" ]; then
      selected+=("${services[$((token - 1))]}")
    else
      token="${token%.service}"
      selected+=("$token")
    fi
  done

  mapfile -t available_array < <(printf '%s\n' "${selected[@]}" | sed '/^$/d' | sort -u)
  SERVICES_TO_MONITOR="${available_array[*]}"
}

render_server_env() {
  local destination="$1"
  cat > "$destination" <<ENVEOF
# Generated by deploy.sh on $(date '+%Y-%m-%d %H:%M:%S %Z')
SERVER_NAME="$(env_escape "$SERVER_NAME")"

TELEGRAM_ENABLED=${TELEGRAM_ENABLED}
TELEGRAM_BOT_TOKEN="$(env_escape "$TELEGRAM_BOT_TOKEN")"
TELEGRAM_CHAT_ID="$(env_escape "$TELEGRAM_CHAT_ID")"

DISCORD_ENABLED=${DISCORD_ENABLED}
DISCORD_WEBHOOK_URL="$(env_escape "$DISCORD_WEBHOOK_URL")"

WEBHOOK_ENABLED=${WEBHOOK_ENABLED}
WEBHOOK_URL="$(env_escape "$WEBHOOK_URL")"
WEBHOOK_SECRET="$(env_escape "$WEBHOOK_SECRET")"

MONITOR_CPU=${MONITOR_CPU}
MONITOR_RAM=${MONITOR_RAM}
MONITOR_DISK=${MONITOR_DISK}
MONITOR_IOWAIT=${MONITOR_IOWAIT}
MONITOR_NETWORK=${MONITOR_NETWORK}
MONITOR_SERVICES=${MONITOR_SERVICES}
MONITOR_BRUTEFORCE=${MONITOR_BRUTEFORCE}
MONITOR_SSH_LOGIN=${MONITOR_SSH_LOGIN}
MONITOR_LOG_SIZE=${MONITOR_LOG_SIZE}

CPU_THRESHOLD=${CPU_THRESHOLD}
RAM_THRESHOLD=${RAM_THRESHOLD}
DISK_THRESHOLD=${DISK_THRESHOLD}
IO_THRESHOLD=${IO_THRESHOLD}
NET_THRESHOLD_MBPS=${NET_THRESHOLD_MBPS}
BRUTE_FORCE_THRESHOLD=${BRUTE_FORCE_THRESHOLD}
LOG_SIZE_THRESHOLD_MB=${LOG_SIZE_THRESHOLD_MB}
LOG_DIR_THRESHOLD_MB=${LOG_DIR_THRESHOLD_MB}
LOG_DIRS="$(env_escape "$LOG_DIRS")"
SERVICES_TO_MONITOR="$(env_escape "$SERVICES_TO_MONITOR")"

CHECK_INTERVAL_RESOURCES_MINUTES=${CHECK_INTERVAL_RESOURCES_MINUTES}
CHECK_INTERVAL_SERVICES_MINUTES=${CHECK_INTERVAL_SERVICES_MINUTES}
CHECK_INTERVAL_BRUTEFORCE_MINUTES=${CHECK_INTERVAL_BRUTEFORCE_MINUTES}
CHECK_INTERVAL_LOG_SIZE_MINUTES=${CHECK_INTERVAL_LOG_SIZE_MINUTES}
RESOURCE_SUSTAIN_MINUTES=${RESOURCE_SUSTAIN_MINUTES}

NOTIFY_RECOVERY=${NOTIFY_RECOVERY}
ENVEOF
}

validate_server_env() {
  local errors=()
  local any_notifier=false

  if [ "$TELEGRAM_ENABLED" = true ]; then
    any_notifier=true
    [ -n "$TELEGRAM_BOT_TOKEN" ] || errors+=("Telegram is enabled but TELEGRAM_BOT_TOKEN is empty.")
    [ -n "$TELEGRAM_CHAT_ID" ] || errors+=("Telegram is enabled but TELEGRAM_CHAT_ID is empty.")
  fi

  if [ "$DISCORD_ENABLED" = true ]; then
    any_notifier=true
    [ -n "$DISCORD_WEBHOOK_URL" ] || errors+=("Discord is enabled but DISCORD_WEBHOOK_URL is empty.")
  fi

  if [ "$WEBHOOK_ENABLED" = true ]; then
    any_notifier=true
    [ -n "$WEBHOOK_URL" ] || errors+=("Webhook is enabled but WEBHOOK_URL is empty.")
  fi

  if [ "$MONITOR_CPU" = true ] || [ "$MONITOR_RAM" = true ] || [ "$MONITOR_DISK" = true ] || \
     [ "$MONITOR_IOWAIT" = true ] || [ "$MONITOR_NETWORK" = true ] || [ "$MONITOR_SERVICES" = true ] || \
     [ "$MONITOR_BRUTEFORCE" = true ] || [ "$MONITOR_SSH_LOGIN" = true ] || [ "$MONITOR_LOG_SIZE" = true ]; then
    [ "$any_notifier" = true ] || errors+=("At least one alert platform must be enabled when monitors are enabled.")
  fi

  if [ "${#errors[@]}" -gt 0 ]; then
    printf '%s\n' "${errors[@]}" >&2
    return 1
  fi
}

show_current_config_summary() {
  local _mask
  _mask() { [ -n "$1" ] && echo "(configured)" || echo "(empty)"; }

  printf '\n'
  printf '  %-38s %s\n' "SERVER_NAME:"                  "${SERVER_NAME:-my-server}"
  printf '\n'
  printf '  %-38s %s\n' "TELEGRAM_ENABLED:"             "$TELEGRAM_ENABLED"
  [ "$TELEGRAM_ENABLED" = true ] && {
    printf '  %-38s %s\n' "TELEGRAM_BOT_TOKEN:"         "$(_mask "$TELEGRAM_BOT_TOKEN")"
    printf '  %-38s %s\n' "TELEGRAM_CHAT_ID:"           "${TELEGRAM_CHAT_ID:-(empty)}"
  }
  printf '  %-38s %s\n' "DISCORD_ENABLED:"              "$DISCORD_ENABLED"
  [ "$DISCORD_ENABLED" = true ] && \
    printf '  %-38s %s\n' "DISCORD_WEBHOOK_URL:"        "$(_mask "$DISCORD_WEBHOOK_URL")"
  printf '  %-38s %s\n' "WEBHOOK_ENABLED:"              "$WEBHOOK_ENABLED"
  [ "$WEBHOOK_ENABLED" = true ] && {
    printf '  %-38s %s\n' "WEBHOOK_URL:"                "$(_mask "$WEBHOOK_URL")"
    printf '  %-38s %s\n' "WEBHOOK_SECRET:"             "$(_mask "$WEBHOOK_SECRET")"
  }
  printf '\n'
  printf '  %-38s %s\n' "CPU / RAM / Disk monitors:"   "${MONITOR_CPU} / ${MONITOR_RAM} / ${MONITOR_DISK}"
  printf '  %-38s %s\n' "IOWait / Network monitors:"   "${MONITOR_IOWAIT} / ${MONITOR_NETWORK}"
  printf '  %-38s %s\n' "Services / Brute-force:"      "${MONITOR_SERVICES} / ${MONITOR_BRUTEFORCE}"
  printf '  %-38s %s\n' "SSH login / Log size:"        "${MONITOR_SSH_LOGIN} / ${MONITOR_LOG_SIZE}"
  printf '\n'
  printf '  %-38s %s\n' "CPU / RAM / Disk threshold:"  "${CPU_THRESHOLD}% / ${RAM_THRESHOLD}% / ${DISK_THRESHOLD}%"
  printf '  %-38s %s\n' "IO / Network threshold:"      "${IO_THRESHOLD}% / ${NET_THRESHOLD_MBPS} MB/s"
  printf '  %-38s %s\n' "Resource sustain:"            "${RESOURCE_SUSTAIN_MINUTES} min"
  printf '  %-38s %s\n' "Brute-force threshold:"       "${BRUTE_FORCE_THRESHOLD} attempts/5min"
  printf '  %-38s %s\n' "Services to monitor:"         "${SERVICES_TO_MONITOR:-(auto-detect)}"
  printf '\n'
}

confirm_reuse_config() {
  local label="$1"
  # Only offer reuse when a previously saved config exists with real values
  [ -f "$SERVER_ENV_DEFAULTS" ] || return 1
  [ -n "${SERVER_NAME:-}" ] && [ "$SERVER_NAME" != "my-server" ] || return 1
  local any_notifier=false
  { [ "$TELEGRAM_ENABLED" = true ] && [ -n "${TELEGRAM_BOT_TOKEN:-}" ]; } && any_notifier=true
  { [ "$DISCORD_ENABLED" = true ] && [ -n "${DISCORD_WEBHOOK_URL:-}" ]; } && any_notifier=true
  { [ "$WEBHOOK_ENABLED" = true ] && [ -n "${WEBHOOK_URL:-}" ]; } && any_notifier=true
  [ "$any_notifier" = true ] || return 1

  if [ "$INTERACTIVE_MODE" = false ]; then
    return 0  # non-interactive: always reuse existing config
  fi

  print_heading "Existing configuration found for ${label}"
  show_current_config_summary
  local reply=""
  read -rp "Re-deploy with this config? [Y/n] (n = go through each setting): " reply
  reply="${reply:-Y}"
  case "$reply" in
    Y|y|yes|YES) return 0 ;;
    *) return 1 ;;
  esac
}

report_deploy_changes() {
  local ssh_target="$1"
  local remote_stage="$2"

  local report
  report="$(ssh_sudo "$ssh_target" "$SUDO_PASS" "
    updated=(); added=()
    while IFS= read -r staged; do
      rel=\"\${staged#${remote_stage}/}\"
      deployed=\"${REMOTE_DIR}/\${rel}\"
      if [ ! -f \"\$deployed\" ]; then
        added+=(\"\$rel\")
      elif ! cmp -s \"\$staged\" \"\$deployed\" 2>/dev/null; then
        updated+=(\"\$rel\")
      fi
    done < <(find '${remote_stage}' -type f | sort)
    for f in \"\${updated[@]:-}\"; do [ -n \"\$f\" ] && printf '[updated] %s\n' \"\$f\"; done
    for f in \"\${added[@]:-}\";   do [ -n \"\$f\" ] && printf '[new]     %s\n' \"\$f\"; done
  " 2>/dev/null || true)"

  if [ -z "$report" ]; then
    log_info "No file changes detected from previous deployment"
  else
    print_heading "Files changed in this deployment"
    while IFS= read -r line; do
      [ -n "$line" ] && log_step "$line"
    done <<< "$report"
  fi
}

configure_scheduler_mode() {

  if [ "$INTERACTIVE_MODE" = false ] || [ "$SCHEDULER_MODE_EXPLICIT" = true ]; then
    SETUP_TIMER="$([ "$SCHEDULER_MODE" = timer ] && echo true || echo false)"
    return
  fi

  print_heading "Server scheduling"
  log_info "Choose whether deploy should install the systemd timer or leave execution to your own cron."

  while true; do
    read -rp "Run monitors with [current: ${SCHEDULER_MODE}] (timer/cron): " reply
    reply="${reply:-$SCHEDULER_MODE}"
    case "$reply" in
      timer|cron)
        SCHEDULER_MODE="$reply"
        SETUP_TIMER="$([ "$SCHEDULER_MODE" = timer ] && echo true || echo false)"
        return
        ;;
    esac
    log_warn "Enter timer or cron."
  done
}

load_server_defaults() {
  source_if_present "$SERVER_ENV_EXAMPLE"
  source_if_present "$SERVER_ENV_DEFAULTS"

  TELEGRAM_ENABLED="$(bool_normalize "${TELEGRAM_ENABLED:-false}")"
  DISCORD_ENABLED="$(bool_normalize "${DISCORD_ENABLED:-false}")"
  WEBHOOK_ENABLED="$(bool_normalize "${WEBHOOK_ENABLED:-false}")"
  MONITOR_CPU="$(bool_normalize "${MONITOR_CPU:-true}")"
  MONITOR_RAM="$(bool_normalize "${MONITOR_RAM:-true}")"
  MONITOR_DISK="$(bool_normalize "${MONITOR_DISK:-true}")"
  MONITOR_IOWAIT="$(bool_normalize "${MONITOR_IOWAIT:-true}")"
  MONITOR_NETWORK="$(bool_normalize "${MONITOR_NETWORK:-true}")"
  MONITOR_SERVICES="$(bool_normalize "${MONITOR_SERVICES:-true}")"
  MONITOR_BRUTEFORCE="$(bool_normalize "${MONITOR_BRUTEFORCE:-true}")"
  MONITOR_SSH_LOGIN="$(bool_normalize "${MONITOR_SSH_LOGIN:-true}")"
  MONITOR_LOG_SIZE="$(bool_normalize "${MONITOR_LOG_SIZE:-true}")"
  NOTIFY_RECOVERY="$(bool_normalize "${NOTIFY_RECOVERY:-true}")"

  SERVER_NAME="${SERVER_NAME:-my-server}"
  TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}"
  TELEGRAM_CHAT_ID="${TELEGRAM_CHAT_ID:-}"
  DISCORD_WEBHOOK_URL="${DISCORD_WEBHOOK_URL:-}"
  WEBHOOK_URL="${WEBHOOK_URL:-}"
  WEBHOOK_SECRET="${WEBHOOK_SECRET:-}"
  CPU_THRESHOLD="${CPU_THRESHOLD:-80}"
  RAM_THRESHOLD="${RAM_THRESHOLD:-85}"
  DISK_THRESHOLD="${DISK_THRESHOLD:-85}"
  IO_THRESHOLD="${IO_THRESHOLD:-80}"
  NET_THRESHOLD_MBPS="${NET_THRESHOLD_MBPS:-100}"
  BRUTE_FORCE_THRESHOLD="${BRUTE_FORCE_THRESHOLD:-10}"
  LOG_SIZE_THRESHOLD_MB="${LOG_SIZE_THRESHOLD_MB:-500}"
  LOG_DIR_THRESHOLD_MB="${LOG_DIR_THRESHOLD_MB:-2048}"
  LOG_DIRS="${LOG_DIRS:-/var/log}"
  SERVICES_TO_MONITOR="${SERVICES_TO_MONITOR:-}"
  CHECK_INTERVAL_RESOURCES_MINUTES="${CHECK_INTERVAL_RESOURCES_MINUTES:-1}"
  CHECK_INTERVAL_SERVICES_MINUTES="${CHECK_INTERVAL_SERVICES_MINUTES:-1}"
  CHECK_INTERVAL_BRUTEFORCE_MINUTES="${CHECK_INTERVAL_BRUTEFORCE_MINUTES:-1}"
  CHECK_INTERVAL_LOG_SIZE_MINUTES="${CHECK_INTERVAL_LOG_SIZE_MINUTES:-1440}"
  RESOURCE_SUSTAIN_MINUTES="${RESOURCE_SUSTAIN_MINUTES:-5}"
}

configure_server_env() {
  local label="$1"
  local ssh_target="$2"
  local default_server_name="$label"

  if [ "$TARGET_COUNT" -eq 1 ] && [ -n "${SERVER_NAME:-}" ] && [ "$SERVER_NAME" != "my-server" ]; then
    default_server_name="$SERVER_NAME"
  fi

  if [ "$INTERACTIVE_MODE" = true ] && [ -f "$SERVER_ENV_DEFAULTS" ]; then
    log_info "Going through each setting — press Enter to keep the current value."
    echo ""
  fi

  print_heading "Server alert configuration for ${label}"
  prompt_text SERVER_NAME "Server name shown in alerts" "$default_server_name"

  print_heading "Alert platforms"
  prompt_bool TELEGRAM_ENABLED "Enable Telegram alerts" "$TELEGRAM_ENABLED"
  if [ "$TELEGRAM_ENABLED" = true ]; then
    prompt_text TELEGRAM_BOT_TOKEN "Telegram bot token" "$TELEGRAM_BOT_TOKEN" true
    prompt_text TELEGRAM_CHAT_ID "Telegram chat ID" "$TELEGRAM_CHAT_ID"
  else
    TELEGRAM_BOT_TOKEN=""
    TELEGRAM_CHAT_ID=""
  fi

  prompt_bool DISCORD_ENABLED "Enable Discord alerts" "$DISCORD_ENABLED"
  if [ "$DISCORD_ENABLED" = true ]; then
    prompt_text DISCORD_WEBHOOK_URL "Discord webhook URL" "$DISCORD_WEBHOOK_URL" true
  else
    DISCORD_WEBHOOK_URL=""
  fi

  prompt_bool WEBHOOK_ENABLED "Enable generic webhook alerts" "$WEBHOOK_ENABLED"
  if [ "$WEBHOOK_ENABLED" = true ]; then
    prompt_text WEBHOOK_URL "Webhook URL" "$WEBHOOK_URL" true
    prompt_text WEBHOOK_SECRET "Webhook secret header value" "$WEBHOOK_SECRET" true
  else
    WEBHOOK_URL=""
    WEBHOOK_SECRET=""
  fi

  print_heading "Monitors"
  prompt_bool MONITOR_CPU "Enable CPU alerts" "$MONITOR_CPU"
  prompt_bool MONITOR_RAM "Enable RAM alerts" "$MONITOR_RAM"
  prompt_bool MONITOR_DISK "Enable disk space alerts" "$MONITOR_DISK"
  prompt_bool MONITOR_IOWAIT "Enable disk I/O wait alerts" "$MONITOR_IOWAIT"
  prompt_bool MONITOR_NETWORK "Enable network spike alerts" "$MONITOR_NETWORK"
  prompt_bool MONITOR_SERVICES "Enable service status alerts" "$MONITOR_SERVICES"
  prompt_bool MONITOR_BRUTEFORCE "Enable brute-force alerts" "$MONITOR_BRUTEFORCE"
  prompt_bool MONITOR_SSH_LOGIN "Enable SSH login alerts (fires immediately via PAM)" "$MONITOR_SSH_LOGIN"
  prompt_bool MONITOR_LOG_SIZE "Enable log storage alerts" "$MONITOR_LOG_SIZE"
  prompt_bool NOTIFY_RECOVERY "Send recovery alerts when issues clear" "$NOTIFY_RECOVERY"

  print_heading "Cadence and thresholds"
  prompt_number CHECK_INTERVAL_RESOURCES_MINUTES "Resource checks every N minutes" "$CHECK_INTERVAL_RESOURCES_MINUTES"
  prompt_number RESOURCE_SUSTAIN_MINUTES "Resource alert after sustained N minutes" "$RESOURCE_SUSTAIN_MINUTES"
  prompt_number CHECK_INTERVAL_SERVICES_MINUTES "Service checks every N minutes" "$CHECK_INTERVAL_SERVICES_MINUTES"
  prompt_number CHECK_INTERVAL_BRUTEFORCE_MINUTES "Brute-force checks every N minutes" "$CHECK_INTERVAL_BRUTEFORCE_MINUTES"
  prompt_number CHECK_INTERVAL_LOG_SIZE_MINUTES "Storage/log checks every N minutes" "$CHECK_INTERVAL_LOG_SIZE_MINUTES"
  log_info "SSH login alerts are event-driven and fire immediately when enabled."

  if [ "$MONITOR_CPU" = true ]; then
    prompt_number CPU_THRESHOLD "CPU threshold percent" "$CPU_THRESHOLD"
  fi
  if [ "$MONITOR_RAM" = true ]; then
    prompt_number RAM_THRESHOLD "RAM threshold percent" "$RAM_THRESHOLD"
  fi
  if [ "$MONITOR_DISK" = true ]; then
    prompt_number DISK_THRESHOLD "Disk threshold percent" "$DISK_THRESHOLD"
  fi
  if [ "$MONITOR_IOWAIT" = true ]; then
    prompt_number IO_THRESHOLD "I/O wait threshold percent" "$IO_THRESHOLD"
  fi
  if [ "$MONITOR_NETWORK" = true ]; then
    prompt_number NET_THRESHOLD_MBPS "Network spike threshold MB/s" "$NET_THRESHOLD_MBPS"
  fi
  if [ "$MONITOR_BRUTEFORCE" = true ]; then
    prompt_number BRUTE_FORCE_THRESHOLD "Failed SSH attempts per 5 minutes" "$BRUTE_FORCE_THRESHOLD"
  fi
  if [ "$MONITOR_LOG_SIZE" = true ]; then
    prompt_number LOG_SIZE_THRESHOLD_MB "Single log file threshold in MB" "$LOG_SIZE_THRESHOLD_MB"
    prompt_number LOG_DIR_THRESHOLD_MB "Log directory threshold in MB" "$LOG_DIR_THRESHOLD_MB"
    prompt_text LOG_DIRS "Log directories to scan (space separated)" "$LOG_DIRS"
  fi

  if [ "$MONITOR_SERVICES" = true ]; then
    prompt_services_selection "$ssh_target" "Select important services to monitor" "$SERVICES_TO_MONITOR"
  else
    SERVICES_TO_MONITOR=""
  fi

  validate_server_env || {
    log_error "Configuration is incomplete. Re-run deploy and provide the missing alert values."
    return 1
  }
}

prepare_local_package() {
  local env_file="$1"
  local package_dir="$2"

  mkdir -p "$package_dir"
  cp -R "${SCRIPT_DIR}/scripts" "$package_dir/"
  cp -R "${SCRIPT_DIR}/systemd" "$package_dir/"
  cp "$env_file" "$package_dir/.env"
}

write_cron_artifacts() {
  local host="$1"
  local safe_host="$2"
  local cron_md="${GENERATED_ENV_DIR}/${safe_host}.cron.setup.md"
  local cron_sh="${GENERATED_ENV_DIR}/${safe_host}.cron.setup.sh"
  local cron_line="* * * * * /bin/bash ${REMOTE_DIR}/scripts/run-monitors.sh"

  cat > "$cron_sh" <<EOF
#!/usr/bin/env bash
set -Eeuo pipefail

CRON_LINE='${cron_line}'

if sudo crontab -l 2>/dev/null | grep -Fq "${REMOTE_DIR}/scripts/run-monitors.sh"; then
  echo "server-monitor cron entry already exists"
  exit 0
fi

(sudo crontab -l 2>/dev/null; printf '%s\n' "\${CRON_LINE}") | sudo crontab -
echo "installed: \${CRON_LINE}"
EOF
  chmod 700 "$cron_sh"

  cat > "$cron_md" <<EOF
# Cron Setup for ${host}

Deploy was configured to skip the systemd timer and let you manage execution with cron.

Recommended cron entry:

\`\`\`cron
${cron_line}
\`\`\`

Why a single every-minute cron works:

- \`run-monitors.sh\` already respects the cadence values from \`${REMOTE_DIR}/.env\`
- resource, service, brute-force, and storage intervals still follow the values chosen during deploy
- SSH login alerts remain event-driven through PAM when enabled

Install it with the generated helper:

\`\`\`bash
bash ${cron_sh}
\`\`\`

Or install it manually:

\`\`\`bash
(sudo crontab -l 2>/dev/null; printf '%s\n' '${cron_line}') | sudo crontab -
\`\`\`
EOF

  if [ "$TARGET_COUNT" -eq 1 ]; then
    cp "$cron_md" "${SCRIPT_DIR}/cron.setup.md"
    log_ok "Updated local cron.setup.md"
  fi

  log_ok "Generated cron helper: ${cron_sh}"
  log_ok "Generated cron instructions: ${cron_md}"
  log_info "Run this after deployment if you want cron instead of the systemd timer:"
  log_step "bash ${cron_sh}"
}

verify_remote_access() {
  local ssh_target="$1"
  local local_stage="$2"
  local remote_stage="$3"

  log_info "Checking remote temp staging access"
  ssh_run "$ssh_target" "probe=\$(mktemp -d /tmp/server-monitor-probe.XXXXXX) && touch \"\$probe/probe\" && rm -rf \"\$probe\""
  log_ok "Remote temp directory is writable"

  log_info "Checking sudo command access"
  ssh_sudo "$ssh_target" "$SUDO_PASS" "probe=\$(mktemp -d /tmp/server-monitor-root-probe.XXXXXX) && rmdir \"\$probe\""
  log_ok "sudo access verified"

  log_info "Checking file copy path to remote temp staging"
  ssh_run "$ssh_target" "rm -rf '${remote_stage}' && mkdir -p '${remote_stage}'"
  rsync -az --dry-run "$local_stage/" "${ssh_target}:${remote_stage}/" >/dev/null
  ssh_run "$ssh_target" "rm -rf '${remote_stage}'"
  log_ok "Rsync dry-run to remote temp staging works"
}

ensure_sudo_access() {
  local ssh_target="$1"
  if ssh_run "$ssh_target" "sudo -n true" >/dev/null 2>&1; then
    SUDO_PASS=""
    log_ok "Passwordless sudo available"
    return
  fi

  while true; do
    echo -ne "  sudo password for ${ssh_target}: "
    read -rs SUDO_PASS
    echo ""
    if ssh "$ssh_target" "printf '%s\\n' '${SUDO_PASS//\'/\'\\\'\'}' | sudo -S true 2>/dev/null"; then
      log_ok "sudo credentials verified"
      return
    fi
    log_warn "Password rejected. Try again."
  done
}

install_dependencies() {
  local ssh_target="$1"
  local missing

  missing="$(ssh_run "$ssh_target" "missing=(); command -v curl >/dev/null || missing+=(curl); command -v bc >/dev/null || missing+=(bc); command -v iostat >/dev/null || missing+=(sysstat); printf '%s ' \"\${missing[@]}\"" 2>/dev/null || true)"
  missing="${missing% }"
  [ -z "$missing" ] && return

  if ssh_run "$ssh_target" "command -v apt-get >/dev/null" >/dev/null 2>&1; then
    log_info "Installing missing packages: ${missing}"
    ssh_sudo "$ssh_target" "$SUDO_PASS" "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ${missing}"
    log_ok "Dependencies installed"
  else
    log_warn "Missing packages (${missing}) but apt-get is unavailable. Install them manually."
  fi
}

deploy_to_host() {
  local host="$1"
  local label="$2"
  local ssh_target="${SSH_USER}@${host}"
  local remote_stage="/tmp/server-monitor-deploy.$$"
  local local_stage env_output safe_host
  local timer_preexisting=false
  local pam_preexisting=false
  local env_preexisting=false

  print_heading "Deploying to ${label} (${host})"

  if ! ssh -o ConnectTimeout=8 -o BatchMode=yes "$ssh_target" "echo connected" >/dev/null 2>&1; then
    log_error "Cannot connect to ${ssh_target}."
    return 1
  fi
  log_ok "SSH connection OK"

  ensure_sudo_access "$ssh_target"
  load_server_defaults

  local skip_configure=false
  confirm_reuse_config "$label" && skip_configure=true || true

  if [ "$skip_configure" = false ]; then
    configure_server_env "$label" "$ssh_target" || return 1
  else
    validate_server_env || {
      log_error "Existing configuration is incomplete. Re-run and fill in the missing values."
      return 1
    }
    log_ok "Using existing configuration"
    # Always re-prompt service selection so the user can review/change monitored services
    if [ "$MONITOR_SERVICES" = true ] && [ "$INTERACTIVE_MODE" = true ]; then
      prompt_services_selection "$ssh_target" "Select services to monitor (current: ${SERVICES_TO_MONITOR:-auto-detect})" "$SERVICES_TO_MONITOR"
    fi
  fi

  mkdir -p "$GENERATED_ENV_DIR"
  safe_host="$(printf '%s' "$host" | tr '/:@ ' '_')"
  env_output="${GENERATED_ENV_DIR}/${safe_host}.env.server"
  render_server_env "$env_output"

  if [ "$SCHEDULER_MODE" = cron ]; then
    write_cron_artifacts "$host" "$safe_host"
  fi

  if [ "$TARGET_COUNT" -eq 1 ]; then
    cp "$env_output" "$SERVER_ENV_DEFAULTS"
    chmod 600 "$SERVER_ENV_DEFAULTS"
    log_ok "Updated local ${SERVER_ENV_DEFAULTS##*/} with the latest answers"
  fi

  local_stage="$(mktemp -d /tmp/server-monitor-package.XXXXXX)"
  trap 'rm -rf "$local_stage"' RETURN
  prepare_local_package "$env_output" "$local_stage"
  verify_remote_access "$ssh_target" "$local_stage" "$remote_stage"

  if [ "$DRY_RUN" = true ]; then
    log_warn "[DRY RUN] Generated ${env_output} and would upload it as ${REMOTE_DIR}/.env"
    if [ "$SCHEDULER_MODE" = cron ]; then
      log_warn "[DRY RUN] Timer install skipped. Use the generated cron helper after a real deploy."
    fi
    rm -rf "$local_stage"
    trap - RETURN
    return 0
  fi

  timer_preexisting="$(ssh_run "$ssh_target" "systemctl is-enabled server-monitor.timer >/dev/null 2>&1 && echo true || echo false" 2>/dev/null || echo false)"
  pam_preexisting="$(ssh_run "$ssh_target" "grep -qF '${REMOTE_DIR}/scripts/monitors/ssh-login.sh' /etc/pam.d/sshd 2>/dev/null && echo true || echo false" 2>/dev/null || echo false)"
  env_preexisting="$(ssh_run "$ssh_target" "[ -f '${REMOTE_DIR}/.env' ] && echo true || echo false" 2>/dev/null || echo false)"

  ssh_run "$ssh_target" "rm -rf '${remote_stage}' && mkdir -p '${remote_stage}'"
  rsync -az --delete "$local_stage/" "${ssh_target}:${remote_stage}/"

  # Show which files will change before applying them
  [ "$env_preexisting" = true ] && report_deploy_changes "$ssh_target" "$remote_stage"

  ssh_sudo "$ssh_target" "$SUDO_PASS" "
    install -d -m 700 '${REMOTE_DIR}' '${REMOTE_DIR}/state' '${REMOTE_DIR}/scripts' '${REMOTE_DIR}/scripts/monitors' '${REMOTE_DIR}/scripts/notifiers' '${REMOTE_DIR}/scripts/lib' '${REMOTE_DIR}/systemd'
    cp -R '${remote_stage}/scripts/.' '${REMOTE_DIR}/scripts/'
    cp -R '${remote_stage}/systemd/.' '${REMOTE_DIR}/systemd/'
    install -m 600 '${remote_stage}/.env' '${REMOTE_DIR}/.env'
    find '${REMOTE_DIR}/scripts' -type f -name '*.sh' -exec chmod 700 {} +
    chmod 644 '${REMOTE_DIR}/systemd/'*.service '${REMOTE_DIR}/systemd/'*.timer
    chown -R root:root '${REMOTE_DIR}'
    rm -f '${REMOTE_DIR}/notify.sh' '${REMOTE_DIR}/run-monitors.sh' '${REMOTE_DIR}/resources.sh' '${REMOTE_DIR}/services.sh' '${REMOTE_DIR}/brute-force.sh' '${REMOTE_DIR}/log-size.sh' '${REMOTE_DIR}/ssh-login.sh' '${REMOTE_DIR}/telegram.sh' '${REMOTE_DIR}/discord.sh' '${REMOTE_DIR}/webhook.sh'
    rm -rf '${remote_stage}'
  "
  log_ok "Uploaded scripts, units, and remote .env"

  install_dependencies "$ssh_target"

  ssh_sudo "$ssh_target" "$SUDO_PASS" "bash '${REMOTE_DIR}/scripts/run-monitors.sh' --list >/dev/null"
  log_ok "Remote configuration validated"

  if [ "$SETUP_TIMER" = true ]; then
    ssh_sudo "$ssh_target" "$SUDO_PASS" "
      cp '${REMOTE_DIR}/systemd/server-monitor.service' /etc/systemd/system/server-monitor.service
      cp '${REMOTE_DIR}/systemd/server-monitor.timer' /etc/systemd/system/server-monitor.timer
      systemctl daemon-reload
      systemctl reset-failed server-monitor.service server-monitor.timer >/dev/null 2>&1 || true
      if systemctl is-enabled server-monitor.timer >/dev/null 2>&1; then
        systemctl restart server-monitor.timer
      else
        systemctl enable --now server-monitor.timer
      fi
    "
    if [ "$timer_preexisting" = true ]; then
      log_ok "Systemd timer reloaded"
    else
      log_ok "Systemd timer enabled"
    fi
  else
    ssh_sudo "$ssh_target" "$SUDO_PASS" "systemctl disable --now server-monitor.timer >/dev/null 2>&1 || true"
    log_ok "Systemd timer disabled; cron mode selected"
  fi

  if [ "$SETUP_PAM" = true ]; then
    ssh_sudo "$ssh_target" "$SUDO_PASS" "grep -qF '${REMOTE_DIR}/scripts/monitors/ssh-login.sh' /etc/pam.d/sshd || printf '%s\n' 'session optional pam_exec.so ${REMOTE_DIR}/scripts/monitors/ssh-login.sh' >> /etc/pam.d/sshd"
    if [ "$pam_preexisting" = true ]; then
      log_ok "PAM hook already present"
    else
      log_ok "PAM hook installed"
    fi
  else
    log_warn "PAM hook setup skipped. SSH login alerts stay unchanged."
  fi

  if [ "$env_preexisting" = true ]; then
    ssh_sudo "$ssh_target" "$SUDO_PASS" "bash '${REMOTE_DIR}/scripts/notify.sh' 'Server Monitor Updated' 'Deployment reloaded the monitor configuration on ${SERVER_NAME}. Timer and alert configuration were refreshed.' '✅' 'info' || true"
    log_ok "Update notification sent"
  fi

  rm -rf "$local_stage"
  trap - RETURN

  echo ""
  log_ok "Deployment complete for ${label}"
}

collect_targets() {
  TARGETS=()

  if [ -n "$EXPLICIT_HOST" ]; then
    TARGETS+=("${EXPLICIT_HOST}|${EXPLICIT_HOST}")
    return
  fi

  if [ "$TARGET_SERVER" = all ]; then
    while IFS= read -r var_name; do
      host="${!var_name}"
      [ -n "$host" ] || continue
      index="${var_name#SSH_HOST_}"
      TARGETS+=("${host}|Server ${index}")
    done < <(compgen -A variable | grep '^SSH_HOST_[0-9]\+$' | sort -V)
    return
  fi

  selected_var="SSH_HOST_${TARGET_SERVER}"
  if [ -z "${!selected_var:-}" ]; then
    log_error "${selected_var} is not set in .env"
    exit 1
  fi
  TARGETS+=("${!selected_var}|Server ${TARGET_SERVER}")
}

show_help() {
  cat <<USAGE
Usage: ./deploy.sh [options]

  --server N         Deploy to SSH_HOST_N from local .env
  --host HOST        Deploy to a specific host directly
  --user USER        SSH user override (default: from .env or ploi)
  --remote-dir PATH  Remote install path (default: /etc/server-monitor)
  --cron             Skip systemd timer and generate cron setup artifacts
  --no-timer         Skip timer install/reload
  --no-pam           Skip PAM hook install
  --non-interactive  Reuse defaults from .env.server/.env.server.example
  --test             Dry run: generate config only, no remote changes
  --help             Show this help
USAGE
}

source_if_present "$LOCAL_ENV_FILE"

SSH_USER="${SSH_USER:-ploi}"
REMOTE_DIR="${REMOTE_DIR:-/etc/server-monitor}"
TARGET_SERVER=all
EXPLICIT_HOST=""
SETUP_TIMER=true
SCHEDULER_MODE=timer
SCHEDULER_MODE_EXPLICIT=false
SETUP_PAM=true
DRY_RUN=false
INTERACTIVE_MODE=true

while [ $# -gt 0 ]; do
  case "$1" in
    --server)
      TARGET_SERVER="$2"
      shift 2
      ;;
    --host)
      EXPLICIT_HOST="$2"
      shift 2
      ;;
    --user)
      SSH_USER="$2"
      shift 2
      ;;
    --remote-dir)
      REMOTE_DIR="$2"
      shift 2
      ;;
    --cron)
      SETUP_TIMER=false
      SCHEDULER_MODE=cron
      SCHEDULER_MODE_EXPLICIT=true
      shift
      ;;
    --no-timer)
      SETUP_TIMER=false
      SCHEDULER_MODE=cron
      SCHEDULER_MODE_EXPLICIT=true
      shift
      ;;
    --no-pam)
      SETUP_PAM=false
      shift
      ;;
    --non-interactive)
      INTERACTIVE_MODE=false
      shift
      ;;
    --test)
      DRY_RUN=true
      shift
      ;;
    --help|-h)
      show_help
      exit 0
      ;;
    *)
      log_error "Unknown argument: $1"
      show_help
      exit 1
      ;;
  esac
done

if [ ! -t 0 ] || [ ! -t 1 ]; then
  INTERACTIVE_MODE=false
fi

if [ ! -f "$SERVER_ENV_EXAMPLE" ]; then
  log_error "Missing ${SERVER_ENV_EXAMPLE}."
  exit 1
fi

for command in ssh rsync mktemp sed awk grep; do
  command -v "$command" >/dev/null 2>&1 || {
    log_error "Required command not found: ${command}"
    exit 1
  }
done

collect_targets
TARGET_COUNT="${#TARGETS[@]}"
if [ "$TARGET_COUNT" -eq 0 ]; then
  log_error "No deployment targets found. Check local .env."
  exit 1
fi

if [ "$DRY_RUN" = true ]; then
  log_warn "Dry run mode enabled. No remote changes will be made."
fi

configure_scheduler_mode

for target in "${TARGETS[@]}"; do
  host="${target%%|*}"
  label="${target#*|}"
  deploy_to_host "$host" "$label"
done

echo ""
log_ok "All deployments completed"