run_multi_trial.sh

#!/bin/bash
# run_multi_trial.sh
# Run all baselines × N trials, archiving each trial separately.
# Used for paper-quality experiments (A1 main result + B4 ablation).
#
# Usage:
#   ./run_multi_trial.sh [n_trials] [options]
#
# Options:
#   --ablation        Also run each mode with --no-cow (B4 ablation).
#   --spike-rps <N>   Override spike RPS (B5 scale invariance, e.g. 100 or 300).
#   --no-ramp         Use step workload (C3, no pre-spike signal).
#   --tag <name>      Append <name> to the multitrial output dir for clarity
#                     (e.g. "spike100", "step"). Auto-derived if omitted.
#
# Examples:
#   ./run_multi_trial.sh 5                              # A1: n=5 main result
#   ./run_multi_trial.sh 5 --ablation                   # B4: n=5 × 5 modes × {CoW=ON,OFF}
#   ./run_multi_trial.sh 3 --spike-rps 100 --tag spike100   # B5: scale invariance
#   ./run_multi_trial.sh 3 --no-ramp --tag step             # C3: step workload
#
# Output:
#   logs/multitrial_<TIMESTAMP>[_<tag>]/
#     trial1_<mode>_<cow_label>/       (one dir per (trial, mode, cow_state))
#       server.log, test.log, resource.csv, load_tester_output.txt
#     ...
#
# Then run:
#   python3 analyze_trials.py logs/multitrial_<TIMESTAMP>[_<tag>]/
#
# CSCI 599: Network Systems for Cloud Computing
# University of Southern California

set -e
cd "$(dirname "$0")"

# ── Arg parser ─────────────────────────────────────────────────────────────────
N_TRIALS=5
DO_ABLATION=0
LOAD_ARGS=""
TAG=""

# First positional arg may be n_trials (an integer); subsequent args are flags.
if [[ "$1" =~ ^[0-9]+$ ]]; then
    N_TRIALS="$1"; shift
fi
while [ $# -gt 0 ]; do
    case "$1" in
        --ablation)     DO_ABLATION=1; shift ;;
        --spike-rps)    LOAD_ARGS="$LOAD_ARGS --spike-rps $2"; shift 2 ;;
        --no-ramp)      LOAD_ARGS="$LOAD_ARGS --no-ramp"; shift ;;
        --tag)          TAG="$2"; shift 2 ;;
        *)              echo "Unknown arg: $1" >&2; exit 1 ;;
    esac
done

# Auto-derive tag if not given so output dirs are self-describing.
if [ -z "$TAG" ]; then
    case "$LOAD_ARGS" in
        *--no-ramp*)        TAG="step" ;;
        *--spike-rps*)      TAG="spike$(echo "$LOAD_ARGS" | grep -oE 'spike-rps [0-9]+' | awk '{print $2}')" ;;
        *)                  TAG="" ;;
    esac
fi

# ── Colour helpers ─────────────────────────────────────────────────────────────
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
info()    { echo -e "${CYAN}[INFO]${NC} $*"; }
success() { echo -e "${GREEN}[OK]${NC}   $*"; }
warn()    { echo -e "${YELLOW}[WARN]${NC} $*"; }
error()   { echo -e "${RED}[ERR]${NC}  $*"; }

# ── Build ──────────────────────────────────────────────────────────────────────
info "Building server..."
make -s
success "Build OK"

# ── Output dir for this multi-trial campaign ──────────────────────────────────
OVERALL_TS=$(date +%Y%m%d_%H%M%S)
if [ -n "$TAG" ]; then
    OUT_ROOT="logs/multitrial_${OVERALL_TS}_${TAG}"
else
    OUT_ROOT="logs/multitrial_${OVERALL_TS}"
fi
mkdir -p "$OUT_ROOT"
info "Output root: $OUT_ROOT"
[ -n "$LOAD_ARGS" ] && info "Load tester args:$LOAD_ARGS"

# ── Mode list ──────────────────────────────────────────────────────────────────
ALL_MODES=("ewma" "ewma_adaptive" "reactive" "static_15" "arima")

# ── Single experiment ──────────────────────────────────────────────────────────
# run_one <out_dir> <label> <server_args...>
run_one() {
    local out_dir="$1"; shift
    local label="$1"; shift
    local server_args="$@"

    echo ""
    echo "── ${label} → ${out_dir} ──"
    mkdir -p "$out_dir"

    # Clean any stale state from previous run.
    pkill -x server 2>/dev/null || true; sleep 1
    rm -f /tmp/faas_worker_*.sock /tmp/faas_template_ctrl.sock /tmp/faas_arima.sock
    rm -f logs/server_*.log

    # Start server in background.
    info "Starting: ./server ${server_args}"
    ./server $server_args &
    local SERVER_PID=$!

    # Wait for ready signal.
    local ready=0
    for i in $(seq 1 20); do
        sleep 0.5
        if ! kill -0 "$SERVER_PID" 2>/dev/null; then
            error "Server crashed during startup."
            return 1
        fi
        # Match any of the freshly-created server logs.
        if ls logs/server_*.log >/dev/null 2>&1 && \
           grep -ql "READY\|starting\|Pre-warming" logs/server_*.log 2>/dev/null; then
            ready=1; break
        fi
    done
    [ "$ready" -eq 0 ] && warn "Server did not signal ready in 10s — proceeding anyway."
    sleep 1

    # Resource monitor.
    python3 resource_monitor.py "${out_dir}/resource.csv" \
        > "${out_dir}/resource_monitor_output.txt" 2>&1 &
    local MONITOR_PID=$!
    sleep 0.5

    # Run load test (LOAD_ARGS forwarded for B5 / C3 ablations).
    info "Running load_tester${LOAD_ARGS:+$LOAD_ARGS}..."
    local sentinel; sentinel=$(mktemp logs/.sentinel.XXXXXX)
    python3 load_tester.py $LOAD_ARGS 2>&1 | tee "${out_dir}/load_tester_output.txt"

    # Stop monitor + server.
    kill "$MONITOR_PID" 2>/dev/null; wait "$MONITOR_PID" 2>/dev/null || true
    kill "$SERVER_PID"  2>/dev/null; wait "$SERVER_PID"  2>/dev/null || true

    # Archive logs.
    # Server log filename: server_<mode>[_static_N][_nocow].log — exact
    # name depends on mode + CoW state. Just copy whatever single
    # server_*.log exists right now (we cleaned others before starting).
    local server_log
    server_log=$(ls logs/server_*.log 2>/dev/null | head -1 || true)
    if [ -n "$server_log" ] && [ -f "$server_log" ]; then
        mv "$server_log" "${out_dir}/server.log"
    else
        warn "No server log found for ${label}"
    fi

    # Test log archived from load_tester.
    local test_log
    test_log=$(find logs -maxdepth 1 -name "test_*.log" -newer "$sentinel" 2>/dev/null \
               | sort | tail -1)
    rm -f "$sentinel"
    if [ -n "$test_log" ] && [ -f "$test_log" ]; then
        mv "$test_log" "${out_dir}/test.log"
    else
        warn "No test log found for ${label}"
    fi

    success "Archived → ${out_dir}/"
}

# ── Trial loop ─────────────────────────────────────────────────────────────────
for trial in $(seq 1 "$N_TRIALS"); do
    echo ""
    echo "════════════════════════════════════════════════════════════════"
    info "Trial ${trial} / ${N_TRIALS}"
    echo "════════════════════════════════════════════════════════════════"

    for mode in "${ALL_MODES[@]}"; do
        # Resolve server args for the mode.
        case "$mode" in
            static_15) args="static 15" ;;
            *)         args="$mode" ;;
        esac

        # CoW=ON run (always).
        run_one "${OUT_ROOT}/trial${trial}_${mode}_cow" "${mode} CoW=ON" $args

        # CoW=OFF run only when --ablation is given (B4).
        if [ "$DO_ABLATION" -eq 1 ]; then
            run_one "${OUT_ROOT}/trial${trial}_${mode}_nocow" "${mode} CoW=OFF" $args --no-cow
        fi
    done
done

echo ""
echo "════════════════════════════════════════════════════════════════"
success "All trials complete."
info "Run analysis:"
echo "    python3 analyze_trials.py ${OUT_ROOT}/"
echo "════════════════════════════════════════════════════════════════"