From 741d7b1a5a7711d09b6a8087e72ab1ca37ad308f Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Mon, 20 Jul 2026 17:45:26 -0500
Subject: [PATCH 1/6] Add instructions for working with rocm+pytorch nightly
 build, scripts are not tested yet

---
 .../README_ROCM_NIGHTLY_TESTING.md            | 137 ++++++++++++++++++
 .../ROCM_PYTORCH_PIP_VENV_SETUP.md            | 128 ++++++++++++++++
 .../slurm_single_process_noprofile.sh         |  75 ++++++++++
 .../slurm_single_process_analyze.sh           | 127 ++++++++++++++++
 .../slurm_single_process_profile.sh           | 112 ++++++++++++++
 .../slurm_single_process.sh                   |  78 ++++++++++
 .../rocprofv3/slurm_single_process_kernels.sh |  79 ++++++++++
 .../rocprofv3/slurm_single_process_traces.sh  |  79 ++++++++++
 .../slurm_single_process.sh                   | 100 +++++++++++++
 MLExamples/PyTorch_Profiling/setup_rocm.sh    |  13 ++
 10 files changed, 928 insertions(+)
 create mode 100644 MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
 create mode 100644 MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
 create mode 100755 MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
 create mode 100755 MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
 create mode 100755 MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
 create mode 100755 MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
 create mode 100755 MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
 create mode 100755 MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
 create mode 100755 MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
 create mode 100644 MLExamples/PyTorch_Profiling/setup_rocm.sh

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
new file mode 100644
index 00000000..41889cbb
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -0,0 +1,137 @@
+# Testing a ROCm Nightly Build with PyTorch (CIFAR-100 train + profile)
+
+This guide walks you through testing a **nightly ROCm build** together with
+PyTorch on an AMD GPU. You will:
+
+1. Build a self-contained Python virtual environment with nightly ROCm, PyTorch,
+   and the ROCm profilers.
+2. Create a `setup_rocm.sh` script that activates that environment.
+3. Run the CIFAR-100 training workload through each profiling tool using the
+   provided SLURM scripts, and check that everything works end to end.
+
+The workload itself is the same `train_cifar_100.py` used throughout this
+directory — a small vision model trained on CIFAR-100. It is intentionally
+short so that a nightly build can be validated quickly.
+
+> This guide targets an AMD MI300A GPU (`gfx942`) on a SLURM cluster. For a
+> different GPU, change the architecture (`device-gfx942` / `--arch`) and the
+> SLURM `--partition` in the scripts accordingly.
+
+---
+
+## Step 1 — Build the virtual environment
+
+Follow [`ROCM_PYTORCH_PIP_VENV_SETUP.md`](./ROCM_PYTORCH_PIP_VENV_SETUP.md) to
+create the `rocm-pytorch-pip` venv. In short, it:
+
+- creates a venv at `~/venvs/rocm-pytorch-pip`,
+- installs nightly ROCm + PyTorch + profilers from the multi-arch nightly index,
+- installs `transformers` and the `rocprof-compute analyze` dependencies,
+- runs `rocm-sdk init` to extract the development headers and device code.
+
+To test a **specific nightly**, set the ROCm version pin in that guide, e.g.:
+
+```bash
+ROCM_VERSION=7.15.0a20260707
+```
+
+Change this value to the nightly date you want to validate.
+
+## Step 2 — Create `setup_rocm.sh`
+
+The SLURM scripts in each sub-directory activate the environment by sourcing
+`../setup_rocm.sh` — i.e. a `setup_rocm.sh` located in **this**
+(`PyTorch_Profiling/`) directory. Create it here (see step 6 of the venv guide
+for the full contents), pointing at the venv you just built:
+
+```bash
+#!/usr/bin/env bash
+VENV="$HOME/venvs/rocm-pytorch-pip"
+DEVEL="$VENV/lib/python3.12/site-packages/_rocm_sdk_devel"
+source "$VENV/bin/activate"
+export ROCM_PATH="$DEVEL"
+export HIP_PATH="$DEVEL"
+export HIP_DEVICE_LIB_PATH="$DEVEL/lib/llvm/amdgcn/bitcode"
+export PATH="$DEVEL/bin:$PATH"
+export LD_LIBRARY_PATH="$DEVEL/lib:$DEVEL/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH"
+echo "ROCm venv active: $VENV"
+```
+
+Adjust `python3.12` to match your Python version.
+
+Verify the environment on a GPU node before profiling:
+
+```bash
+source setup_rocm.sh
+srun -n1 --gpus=1 python3 -c "import torch; print('torch', torch.__version__); \
+x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())"
+```
+
+If you see `device ok:`, the nightly ROCm build runs GPU kernels through PyTorch
+and you are ready to profile.
+
+## Step 3 — Run the SLURM scripts
+
+Each sub-directory contains a single-process SLURM script that sources
+`../setup_rocm.sh`, pre-downloads the dataset if needed, and runs the workload
+under one tool. All of them use a single GPU and a short run
+(`--batch-size 64 --max-steps 5`) so a nightly can be checked quickly.
+
+**Submit each script from its own directory** (the scripts use
+`SLURM_SUBMIT_DIR` to locate themselves):
+
+| Tool | Directory | Script | What it produces |
+|------|-----------|--------|------------------|
+| None (baseline) | `no-profiling/` | `slurm_single_process_noprofile.sh` | Plain training run — confirms the workload runs without a profiler. |
+| ROCm Compute Profiler | `rocm-compute-profiler/` | `slurm_single_process_profile.sh` | Hardware-counter profile under `workloads/` (analyze with `rocprof-compute analyze`). |
+| RocProfiler (kernels) | `rocprofv3/` | `slurm_single_process_kernels.sh` | Kernel stats + trace CSVs under `single_process/`. |
+| RocProfiler (traces) | `rocprofv3/` | `slurm_single_process_traces.sh` | System timeline trace (`.pftrace`) under `single_process/`. |
+| ROCm Systems Profiler | `rocm-systems-profiler/` | `slurm_single_process.sh` | Sampling profile + trace under `rocprofsys-python3-output/`. |
+| Roofline Extractor | `roofline-extractor/` | `slurm_single_process.sh` | Per-kernel roofline analysis + interactive HTML plot under `output/`. |
+
+Example (baseline sanity check first, then a profiler):
+
+```bash
+cd no-profiling
+sbatch slurm_single_process_noprofile.sh
+
+cd ../rocm-compute-profiler
+sbatch slurm_single_process_profile.sh
+```
+
+Check job status and output:
+
+```bash
+squeue --me
+# stdout/stderr land in the submit directory as <name>_<jobid>.out / .err
+```
+
+### Analyzing the results
+
+> **Important — running `rocprof-compute analyze`:** the analysis step (the
+> `rocm-compute-profiler/slurm_single_process_analyze.sh` helper) must be run
+> **only after** its profile job has finished, and **not while any other job is
+> using the same venv**. `rocprof-compute analyze` requires `numpy==1.26.4`,
+> which conflicts with the `numpy>=2.0` needed by the training/profiling jobs.
+> The analysis script temporarily changes numpy in the shared venv and restores
+> it on exit, so a training or profiling job running in parallel would pick up
+> the wrong numpy version and crash on import. Let all profiling jobs finish,
+> then run the analysis by itself.
+
+- **ROCm Compute Profiler:** `rocprof-compute analyze -p rocm-compute-profiler/workloads/cifar_100_single_proc/<subdir>`
+- **RocProfiler:** open the CSVs (kernels) or load the `.pftrace` in
+  [ui.perfetto.dev](https://ui.perfetto.dev) (traces).
+- **ROCm Systems Profiler:** load the `perfetto-trace-*.proto` from
+  `rocprofsys-python3-output/<timestamp>/` in [ui.perfetto.dev](https://ui.perfetto.dev).
+- **Roofline Extractor:** open the generated `.html` in `roofline-extractor/output/`.
+
+---
+
+## Notes
+
+- If a job fails to start, check the SLURM `--partition` and time limits in the
+  script headers match your cluster.
+- The scripts derive a per-job rendezvous port from the SLURM job ID, so
+  multiple jobs can share a node without port collisions.
+- To validate a different nightly, rebuild the venv (Step 1) with a new
+  `ROCM_VERSION` and re-run the scripts.
diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
new file mode 100644
index 00000000..0da33f74
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
@@ -0,0 +1,128 @@
+# Creating the `rocm-pytorch-pip` venv (nightly ROCm + PyTorch, MI300A / gfx942)
+
+This guide walks you through building a Python virtual environment with ROCm,
+PyTorch, and the ROCm profiling tools, so you can train and profile a model on
+an AMD GPU. Follow the steps in order.
+
+Everything installs from pip into one self-contained venv, so PyTorch and the
+profilers use the same ROCm. Each step below is a single command block you can
+copy and run.
+
+> This guide targets an AMD MI300A GPU (`gfx942`). If you have a different GPU,
+> change `device-gfx942` to your architecture. Also make sure the Python version
+> in the paths below (`python3.12`) matches your `python --version`.
+
+---
+
+## 1. Create and activate the venv
+
+```bash
+mkdir -p ~/venvs
+python -m venv ~/venvs/rocm-pytorch-pip
+source ~/venvs/rocm-pytorch-pip/bin/activate
+```
+
+> Tip: the setup script below hardcodes the Python minor version in the
+> `site-packages` path (e.g. `python3.12`). Check what `python --version` (or
+> `module load python`) gives you and keep the two consistent.
+
+## 2. Install ROCm + PyTorch from the nightly multi-arch index
+
+```bash
+# Pin the nightly ROCm version once and reuse it everywhere below.
+ROCM_VERSION=7.15.0a20260707
+
+pip install --index-url https://rocm.nightlies.amd.com/whl-multi-arch/ \
+    "rocm[profiler,devel,libraries,device-gfx942]==${ROCM_VERSION}" \
+    "torch[device-gfx942]" \
+    "torchvision[device-gfx942]"
+```
+
+The `rocm[...]` extras pull in the pieces this workflow needs:
+- `profiler`  — the ROCm profilers: `rocprof-compute`, `rocprofv3`, and
+  `rocprof-sys` (bundled `_rocm_profiler`)
+- `devel`     — development package (headers/device code, extracted in step 5)
+- `libraries` — math libraries (hipBLAS, rocBLAS, ...)
+- `device-gfx942` — the GPU-arch kernels for MI300A
+
+## 3. Install `transformers` (required by the training script)
+
+```bash
+pip install transformers
+```
+
+The training script builds its models with `transformers`, so this package is
+required.
+
+## 4. Install `rocprof-compute analyze` dependencies
+
+```bash
+pip install -r ~/venvs/rocm-pytorch-pip/lib/python3.12/site-packages/_rocm_profiler/libexec/rocprofiler-compute/requirements.txt
+```
+
+These extra Python packages are needed by `rocprof-compute analyze` (the
+reporting/analysis step), not by counter collection itself.
+
+## 5. Extract development headers and device code
+
+```bash
+~/venvs/rocm-pytorch-pip/bin/rocm-sdk init
+```
+
+`rocm-sdk init` unpacks the `devel` payload (headers, LLVM device bitcode, etc.)
+into `_rocm_sdk_devel/` inside the venv. This provides the device bitcode that
+HIP needs to run GPU kernels, and the paths that `setup_rocm.sh` points to in the
+next step.
+
+---
+
+## 6. Write `setup_rocm.sh`
+
+Create `~/setup_rocm.sh` to activate the venv and point the ROCm environment at
+the extracted `_rocm_sdk_devel` tree. Adjust `python3.12` to your Python version.
+
+```bash
+#!/usr/bin/env bash
+# Source this to activate the ROCm venv and set ROCm env vars:
+#   source setup_rocm.sh
+VENV="$HOME/venvs/rocm-pytorch-pip"
+DEVEL="$VENV/lib/python3.12/site-packages/_rocm_sdk_devel"
+source "$VENV/bin/activate"
+export ROCM_PATH="$DEVEL"
+export HIP_PATH="$DEVEL"
+export HIP_DEVICE_LIB_PATH="$DEVEL/lib/llvm/amdgcn/bitcode"
+export PATH="$DEVEL/bin:$PATH"
+export LD_LIBRARY_PATH="$DEVEL/lib:$DEVEL/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH"
+echo "ROCm venv active: $VENV"
+```
+
+## 7. Re-source to pick up the ROCm env vars
+
+If the venv is already active from step 1, deactivate and source the script so
+the `ROCM_PATH` / `LD_LIBRARY_PATH` exports take effect:
+
+```bash
+deactivate
+source setup_rocm.sh
+```
+
+---
+
+## 8. Verify (on a GPU node)
+
+```bash
+source ~/setup_rocm.sh
+srun -n1 --gpus=1 python3 -c "import torch; print('torch', torch.__version__); \
+x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())"
+```
+
+Expected output resembles:
+
+```
+ROCm venv active: /.../rocm-pytorch-pip
+torch 2.12.0+rocm7.15.0a20260707
+device ok: 8.0
+```
+
+If you see `device ok:`, the GPU is working and your environment is ready to use.
+From now on, just run `source setup_rocm.sh` in any new shell to activate it.
diff --git a/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
new file mode 100755
index 00000000..b8b111cd
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#SBATCH --job-name=rpc-single-noprofile
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=00:30:00
+#SBATCH --output=rpc_single_process_noprofile_%j.out
+#SBATCH --error=rpc_single_process_noprofile_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: run the single-process CIFAR-100 workload DIRECTLY with
+# python3, WITHOUT rocprof-compute.
+#
+# This is the no-profiler counterpart to slurm_single_process_profile.sh.
+# It sources the same environment (../setup_rocm.sh) and runs the same python3
+# command, which is useful for validating the ROCm + PyTorch environment before
+# profiling.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/no-profiling/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars, so python3 resolves from
+# the same install used by the profiling driver.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present.
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+cd ${SCRIPT_DIR}
+
+# ---------------------------------------------------------------------------
+# Run the workload directly (no profiler).
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "Running (no profiler): python3 train_cifar_100.py"
+echo "==================================================================="
+srun -n 1 --gpus=1 \
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --batch-size 32 --max-steps 5 \
+        --data-path ${PROFILER_TOP_DIR}/data
+
+echo
+echo "==================================================================="
+echo "Workload run complete."
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
new file mode 100755
index 00000000..8e8eeaa0
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+#SBATCH --job-name=rpc-single-analyze
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=00:15:00
+#SBATCH --output=rpc_single_process_analyze_%j.out
+#SBATCH --error=rpc_single_process_analyze_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: analyze the workload produced by slurm_single_process_profile.sh
+# with rocprofiler-compute (rocprof-compute analyze).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+#
+# Analysis is CPU-only (it parses the counter database), so no GPU is requested.
+#
+# !!! IMPORTANT - WHEN TO RUN THIS SCRIPT !!!
+#   1. Run this ONLY AFTER slurm_single_process_profile.sh has completed, so
+#      the workload counter database exists.
+#   2. Do NOT run this while any other job that uses the same venv is running.
+#      This script temporarily pins numpy==1.26.4 (required by rocprof-compute
+#      analyze) in the SHARED venv and restores it on exit. If a training or
+#      profiling job runs concurrently, it will observe the wrong numpy version
+#      and crash on import. Run all such jobs to completion first, then run
+#      this analysis by itself.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/rocm-compute-profiler/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprof-compute --version
+
+# ---------------------------------------------------------------------------
+# rocprof-compute analyze pins numpy==1.26.4, but other tools that share this
+# venv (e.g. the roofline extractor's scipy) can bump numpy to 2.x, which makes
+# analyze abort with a version-requirement error. Temporarily install the
+# required numpy for the analysis, then restore whatever version was there
+# before so the shared venv is left unchanged (restored even on failure).
+# ---------------------------------------------------------------------------
+REQUIRED_NUMPY=1.26.4
+ORIG_NUMPY="$(python3 -c 'import numpy; print(numpy.__version__)' 2>/dev/null)"
+
+restore_numpy() {
+    if [[ -n "${ORIG_NUMPY}" && "${ORIG_NUMPY}" != "${REQUIRED_NUMPY}" ]]; then
+        echo "Restoring numpy==${ORIG_NUMPY}"
+        pip install "numpy==${ORIG_NUMPY}"
+    fi
+}
+
+if [[ "${ORIG_NUMPY}" != "${REQUIRED_NUMPY}" ]]; then
+    echo "Installing numpy==${REQUIRED_NUMPY} for rocprof-compute analyze"
+    pip install "numpy==${REQUIRED_NUMPY}"
+    trap restore_numpy EXIT
+fi
+
+# ---------------------------------------------------------------------------
+# Locate the workload written by slurm_single_process_profile.sh. This must
+# match WORKLOAD_NAME / WORK_ROOT in that script.
+# ---------------------------------------------------------------------------
+WORKLOAD_NAME=cifar_100_single_proc
+WORK_ROOT=${SCRIPT_DIR}/workloads
+WORK_DIR=${WORK_ROOT}/${WORKLOAD_NAME}
+
+if [[ ! -d "${WORK_DIR}" ]]; then
+    echo "ERROR: workload directory not found: ${WORK_DIR}" >&2
+    echo "Run slurm_single_process_profile.sh first." >&2
+    exit 1
+fi
+
+# The profile is written into a single subdirectory named after the architecture
+# (e.g. `MI300A_*`) or numerically (`0`, `1`, ...). Pick the first one.
+ARCH_DIR=$(find ${WORK_DIR} -mindepth 1 -maxdepth 1 -type d | sort | head -1)
+if [[ -z "${ARCH_DIR}" ]]; then
+    echo "ERROR: no workload subdirectory found under ${WORK_DIR}" >&2
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Analyze.
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "rocprof-compute analyze --list-stats -p ${ARCH_DIR}"
+echo "==================================================================="
+STATS_FILE=${SCRIPT_DIR}/stats_${SLURM_JOB_ID}.txt
+rocprof-compute analyze --list-stats -p ${ARCH_DIR} >& ${STATS_FILE}
+echo "Stats and dispatch IDs written to ${STATS_FILE}"
+
+echo
+echo "==================================================================="
+echo "rocprof-compute analyze -p ${ARCH_DIR}"
+echo "==================================================================="
+ANALYSIS_FILE=${SCRIPT_DIR}/analysis_${SLURM_JOB_ID}.txt
+rocprof-compute analyze -p ${ARCH_DIR} > ${ANALYSIS_FILE} 2>&1
+echo "Analysis written to ${ANALYSIS_FILE}"
+
+echo
+echo "-------------------------------------------------------------------"
+echo "NOTE: The analysis above aggregates all kernels. To focus on a"
+echo "specific kernel, inspect ${STATS_FILE} for the kernel names and"
+echo "dispatch IDs, then re-run rocprof-compute analyze narrowing to that"
+echo "kernel with --kernel <kernel-id>, or to a specific invocation with"
+echo "--dispatch <dispatch-id> (you may pass either one, or both):"
+echo "  rocprof-compute analyze -p ${ARCH_DIR} --kernel <kernel-id>"
+echo "  rocprof-compute analyze -p ${ARCH_DIR} --dispatch <dispatch-id>"
+echo "-------------------------------------------------------------------"
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
new file mode 100755
index 00000000..a8c98c8a
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+#SBATCH --job-name=rpc-single-profile
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=02:00:00
+# Reserve the whole node so no other job shares the GPUs during profiling.
+#SBATCH --exclusive
+#SBATCH --output=rpc_single_process_profile_%j.out
+#SBATCH --error=rpc_single_process_profile_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: profile the single-process CIFAR-100 workload with
+# rocprofiler-compute (rocprof-compute profile).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/rocm-compute-profiler/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprof-compute --version
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present (the wrapper also does this, but doing it
+# here keeps the profiled step from spending time on the download).
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+WORKLOAD_NAME=cifar_100_single_proc
+WORK_ROOT=${SCRIPT_DIR}/workloads
+WORK_DIR=${WORK_ROOT}/${WORKLOAD_NAME}
+
+mkdir -p ${WORK_ROOT}
+rm -rf ${WORK_DIR}
+cd ${SCRIPT_DIR}
+
+# ---------------------------------------------------------------------------
+# Profile.
+#
+# Notes:
+#  * --no-roof skips roofline capture (faster; not needed for this demo).
+#  * rocprof-compute replays the application multiple times to cover all
+#    hardware counters, so keep the workload short.
+#  * We profile python3 directly (not the no-profiling/*.sh bash wrapper).
+#    rocprof-compute 3.7.0 / rocprofiler-sdk v1.3.1 injects into the bash
+#    wrapper process too and aborts it with "Output path is empty", so the
+#    application target must be the python interpreter itself. The environment
+#    (modules) is already configured above and is inherited by the profiled
+#    process.
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "rocprof-compute profile -- python3 train_cifar_100.py"
+echo "==================================================================="
+# With --exclusive the whole node is ours; give the single task every core on
+# the node (otherwise srun would pin it to just one CPU).
+srun -n 1 --gpus=1 --cpus-per-task=${SLURM_CPUS_ON_NODE} \
+    rocprof-compute profile \
+        --name ${WORKLOAD_NAME} \
+        --no-roof \
+        -- \
+        python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+            --batch-size 32 --max-steps 5 \
+            --data-path ${PROFILER_TOP_DIR}/data
+
+# Resolve the workload subdirectory written by rocprof-compute. Depending on
+# the version this is named after the architecture (e.g. `MI300A_*`) or
+# numerically (`0`, `1`, ...). Pick the first (typically only) subdirectory.
+ARCH_DIR=$(find ${WORK_DIR} -mindepth 1 -maxdepth 1 -type d | sort | head -1)
+if [[ -z "${ARCH_DIR}" ]]; then
+    echo "ERROR: no workload subdirectory found under ${WORK_DIR}" >&2
+    exit 1
+fi
+
+echo
+echo "==================================================================="
+echo "Profile complete."
+echo "Workload directory: ${ARCH_DIR}"
+echo
+echo "Analyze with, e.g.:"
+echo "  rocprof-compute analyze -p ${ARCH_DIR}"
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
new file mode 100755
index 00000000..ca9a4506
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+#SBATCH --job-name=rps-single-sample
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=00:30:00
+#SBATCH --output=rps_single_process_%j.out
+#SBATCH --error=rps_single_process_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: profile the single-process CIFAR-100 workload with the ROCm
+# Systems Profiler (rocprof-sys-sample).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/rocm-systems-profiler/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprof-sys-run --version
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present before the profiled step.
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+cd ${SCRIPT_DIR}
+
+# Remove any stale output directories from previous runs.
+rm -rf ${SCRIPT_DIR}/rocprofsys-python3-output
+
+# ---------------------------------------------------------------------------
+# Profile with rocprof-sys-run (profiling + tracing).
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "rocprof-sys-run --profile --trace -- python3 train_cifar_100.py"
+echo "==================================================================="
+srun -n 1 --gpus=1 \
+    rocprof-sys-run --profile --trace -- \
+        python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+            --batch-size 32 --max-steps 5 \
+            --data-path ${PROFILER_TOP_DIR}/data
+
+echo
+echo "==================================================================="
+echo "Profile complete."
+echo "Output directory: ${SCRIPT_DIR}/rocprofsys-python3-output"
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
new file mode 100755
index 00000000..de08539d
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#SBATCH --job-name=rpv3-single-kernels
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=00:30:00
+#SBATCH --output=rpv3_single_process_kernels_%j.out
+#SBATCH --error=rpv3_single_process_kernels_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: profile the single-process CIFAR-100 workload with rocprofv3
+# kernel tracing (--stats --kernel-trace).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/rocprofv3/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprofv3 --version
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present before the profiled step.
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+OUT_DIR=${SCRIPT_DIR}/single_process
+rm -rf ${OUT_DIR}
+cd ${SCRIPT_DIR}
+
+# ---------------------------------------------------------------------------
+# Profile kernels with rocprofv3.
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "rocprofv3 --stats --kernel-trace -- python3 train_cifar_100.py"
+echo "==================================================================="
+srun -n 1 --gpus=1 \
+    rocprofv3 --stats --kernel-trace --output-format csv \
+        --output-directory ${OUT_DIR} --output-file kernels \
+        -- \
+        python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+            --batch-size 32 --max-steps 5 \
+            --data-path ${PROFILER_TOP_DIR}/data
+
+echo
+echo "==================================================================="
+echo "Profile complete."
+echo "Output directory: ${OUT_DIR}"
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
new file mode 100755
index 00000000..67a701c7
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#SBATCH --job-name=rpv3-single-traces
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=00:30:00
+#SBATCH --output=rpv3_single_process_traces_%j.out
+#SBATCH --error=rpv3_single_process_traces_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: collect GPU timeline traces for the single-process CIFAR-100
+# workload with rocprofv3 (--sys-trace).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/rocprofv3/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprofv3 --version
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present before the profiled step.
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+OUT_DIR=${SCRIPT_DIR}/single_process
+rm -rf ${OUT_DIR}
+cd ${SCRIPT_DIR}
+
+# ---------------------------------------------------------------------------
+# Collect GPU timeline traces with rocprofv3.
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "rocprofv3 --sys-trace -- python3 train_cifar_100.py"
+echo "==================================================================="
+srun -n 1 --gpus=1 \
+    rocprofv3 --sys-trace --output-format pftrace \
+        --output-directory ${OUT_DIR} --output-file traces \
+        -- \
+        python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+            --batch-size 32 --max-steps 5 \
+            --data-path ${PROFILER_TOP_DIR}/data
+
+echo
+echo "==================================================================="
+echo "Trace collection complete."
+echo "Output directory: ${OUT_DIR}"
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
new file mode 100755
index 00000000..5bfd9e82
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#SBATCH --job-name=roofline-single
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --gpus=1
+#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --time=02:00:00
+#SBATCH --output=roofline_single_process_%j.out
+#SBATCH --error=roofline_single_process_%j.err
+
+# ---------------------------------------------------------------------------
+# SLURM script: collect roofline plots for the single-process CIFAR-100
+# workload with AMD-HPC's rooflineExtractor (https://github.com/AMD-HPC/rooflineExtractor).
+#
+# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
+# matching ROCm env vars.
+#
+# rooflineExtractor's profile_app.py automates the whole flow: it runs rocprofv3
+# several times to collect hardware counters and a kernel trace, post-processes
+# them, then runs rooflineExtractor.py to produce the per-kernel roofline
+# analysis and an interactive HTML plot.
+# ---------------------------------------------------------------------------
+
+set -e
+
+# Resolve the directory of this script. Under sbatch, $0 points to a copy in
+# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
+# job was submitted). Submit this script from
+# `MLExamples/PyTorch_Profiling/roofline-extractor/`.
+if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
+    SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
+else
+    SCRIPT_DIR="$(dirname "$(readlink -fm "$0")")"
+fi
+PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
+echo "SCRIPT_DIR=${SCRIPT_DIR}"
+echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
+
+# ---------------------------------------------------------------------------
+# Software environment.
+#
+# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
+# PyTorch venv and exports the matching ROCm env vars.
+# ---------------------------------------------------------------------------
+source ${PROFILER_TOP_DIR}/setup_rocm.sh
+rocprofv3 --version
+
+# ---------------------------------------------------------------------------
+# Fetch rooflineExtractor and install its Python dependencies (into the venv).
+# ---------------------------------------------------------------------------
+RE_DIR=${SCRIPT_DIR}/rooflineExtractor
+if [ ! -d ${RE_DIR} ]; then
+    git clone https://github.com/AMD-HPC/rooflineExtractor.git ${RE_DIR}
+fi
+pip install -r ${RE_DIR}/requirements.txt
+
+# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+export NPROCS=1
+export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
+# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
+# node (a fixed port like 1234 can fail with EADDRINUSE).
+export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
+
+# Make sure the dataset is present before the profiled runs (profile_app.py runs
+# the application several times, so pre-downloading avoids repeated downloads).
+if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
+    python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+        --data-path ${PROFILER_TOP_DIR}/data --download-only
+fi
+
+OUT_DIR=${SCRIPT_DIR}/output
+rm -rf ${OUT_DIR}
+cd ${SCRIPT_DIR}
+
+# ---------------------------------------------------------------------------
+# Collect roofline data and generate plots.
+#
+# --arch MI300A selects the matching counter set (gfx942). profile_app.py runs
+# rocprofv3 itself, so it is the application target (not launched under another
+# profiler).
+# ---------------------------------------------------------------------------
+echo
+echo "==================================================================="
+echo "profile_app.py --arch MI300A -- python3 train_cifar_100.py"
+echo "==================================================================="
+srun -n 1 --gpus=1 \
+    python3 ${RE_DIR}/profile_app.py \
+        --arch MI300A \
+        -o ${OUT_DIR} \
+        -- \
+        python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
+            --batch-size 32 --max-steps 5 \
+            --data-path ${PROFILER_TOP_DIR}/data
+
+echo
+echo "==================================================================="
+echo "Roofline analysis complete."
+echo "Output directory (counters, traces, plots, analysis): ${OUT_DIR}"
+echo "Open the generated .html file for the interactive roofline plot."
+echo "==================================================================="
diff --git a/MLExamples/PyTorch_Profiling/setup_rocm.sh b/MLExamples/PyTorch_Profiling/setup_rocm.sh
new file mode 100644
index 00000000..f074f414
--- /dev/null
+++ b/MLExamples/PyTorch_Profiling/setup_rocm.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Source this to activate the ROCm venv and set ROCm env vars:
+#   source setup_rocm.sh
+VENV="${HOME}/venvs/rocm-pytorch-pip"
+DEVEL="${VENV}/lib/python3.12/site-packages/_rocm_sdk_devel"
+source "${VENV}/bin/activate"
+export ROCM_PATH="$DEVEL"
+export HIP_PATH="$DEVEL"
+export HIP_DEVICE_LIB_PATH="${DEVEL}/lib/llvm/amdgcn/bitcode"
+export PATH="${DEVEL}/bin:$PATH"
+export LD_LIBRARY_PATH="${DEVEL}/lib:${DEVEL}/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH"
+echo "ROCm venv active: $VENV"
+

From 06c437e79e16a4bef634055adbdc995773f7d554 Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Mon, 20 Jul 2026 17:50:09 -0500
Subject: [PATCH 2/6] Fix README to point to new analysis slurm script in
 rocprof-compute directory

---
 .../README_ROCM_NIGHTLY_TESTING.md                | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
index 41889cbb..ecc80ced 100644
--- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -95,8 +95,8 @@ Example (baseline sanity check first, then a profiler):
 cd no-profiling
 sbatch slurm_single_process_noprofile.sh
 
-cd ../rocm-compute-profiler
-sbatch slurm_single_process_profile.sh
+cd ../rocprofv3
+sbatch slurm_single_process_kernels.sh
 ```
 
 Check job status and output:
@@ -118,7 +118,16 @@ squeue --me
 > the wrong numpy version and crash on import. Let all profiling jobs finish,
 > then run the analysis by itself.
 
-- **ROCm Compute Profiler:** `rocprof-compute analyze -p rocm-compute-profiler/workloads/cifar_100_single_proc/<subdir>`
+- **ROCm Compute Profiler:** submit the companion analysis job from
+  `rocm-compute-profiler/` (it locates the workload and runs the analysis for
+  you, handling the numpy pin described above):
+
+```bash
+cd rocm-compute-profiler
+sbatch slurm_single_process_analyze.sh
+```
+
+  Or run it by hand: `rocprof-compute analyze -p rocm-compute-profiler/workloads/cifar_100_single_proc/<subdir>`
 - **RocProfiler:** open the CSVs (kernels) or load the `.pftrace` in
   [ui.perfetto.dev](https://ui.perfetto.dev) (traces).
 - **ROCm Systems Profiler:** load the `perfetto-trace-*.proto` from

From aaec72ab11937b56c4fe848bef06a306b5313314 Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Tue, 21 Jul 2026 15:56:10 -0500
Subject: [PATCH 3/6] Add separate environment set up instructions for running
 rocprof-compute analysis

---
 .../README_ROCM_NIGHTLY_TESTING.md            |  24 ++--
 .../ROCM_PYTORCH_PIP_VENV_SETUP.md            |  25 ++--
 .../slurm_single_process_analyze.sh           | 117 ++++++++++++------
 3 files changed, 101 insertions(+), 65 deletions(-)

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
index ecc80ced..522c98fb 100644
--- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -26,7 +26,7 @@ create the `rocm-pytorch-pip` venv. In short, it:
 
 - creates a venv at `~/venvs/rocm-pytorch-pip`,
 - installs nightly ROCm + PyTorch + profilers from the multi-arch nightly index,
-- installs `transformers` and the `rocprof-compute analyze` dependencies,
+- installs `transformers` (required by the training script),
 - runs `rocm-sdk init` to extract the development headers and device code.
 
 To test a **specific nightly**, set the ROCm version pin in that guide, e.g.:
@@ -41,7 +41,7 @@ Change this value to the nightly date you want to validate.
 
 The SLURM scripts in each sub-directory activate the environment by sourcing
 `../setup_rocm.sh` — i.e. a `setup_rocm.sh` located in **this**
-(`PyTorch_Profiling/`) directory. Create it here (see step 6 of the venv guide
+(`PyTorch_Profiling/`) directory. Create it here (see step 5 of the venv guide
 for the full contents), pointing at the venv you just built:
 
 ```bash
@@ -108,19 +108,19 @@ squeue --me
 
 ### Analyzing the results
 
-> **Important — running `rocprof-compute analyze`:** the analysis step (the
-> `rocm-compute-profiler/slurm_single_process_analyze.sh` helper) must be run
-> **only after** its profile job has finished, and **not while any other job is
-> using the same venv**. `rocprof-compute analyze` requires `numpy==1.26.4`,
-> which conflicts with the `numpy>=2.0` needed by the training/profiling jobs.
-> The analysis script temporarily changes numpy in the shared venv and restores
-> it on exit, so a training or profiling job running in parallel would pick up
-> the wrong numpy version and crash on import. Let all profiling jobs finish,
-> then run the analysis by itself.
+> **Note — running `rocprof-compute analyze`:** run the analysis step only
+> **after** its profile job has finished (the counter database must exist).
+> `rocprof-compute analyze` requires `numpy==1.26.4`, which conflicts with the
+> `numpy>=2.0` needed by the training/profiling jobs. To avoid disturbing the
+> shared venv, the analysis script uses a separate, isolated venv
+> (`~/venvs/rocprof-compute-analyze`) that holds only rocprof-compute's pinned
+> requirements, while reusing the ROCm install from the shared venv. The shared
+> venv is never modified, so the analysis can run even while training or
+> profiling jobs are using it.
 
 - **ROCm Compute Profiler:** submit the companion analysis job from
   `rocm-compute-profiler/` (it locates the workload and runs the analysis for
-  you, handling the numpy pin described above):
+  you, using the isolated analysis venv described above):
 
 ```bash
 cd rocm-compute-profiler
diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
index 0da33f74..93fe9fdc 100644
--- a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
+++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
@@ -41,7 +41,7 @@ pip install --index-url https://rocm.nightlies.amd.com/whl-multi-arch/ \
 The `rocm[...]` extras pull in the pieces this workflow needs:
 - `profiler`  — the ROCm profilers: `rocprof-compute`, `rocprofv3`, and
   `rocprof-sys` (bundled `_rocm_profiler`)
-- `devel`     — development package (headers/device code, extracted in step 5)
+- `devel`     — development package (headers/device code, extracted in step 4)
 - `libraries` — math libraries (hipBLAS, rocBLAS, ...)
 - `device-gfx942` — the GPU-arch kernels for MI300A
 
@@ -54,16 +54,15 @@ pip install transformers
 The training script builds its models with `transformers`, so this package is
 required.
 
-## 4. Install `rocprof-compute analyze` dependencies
+> **Note:** Do **not** install `rocprof-compute analyze`'s `requirements.txt`
+> into this shared venv. Those packages pin `numpy==1.26.4`, which conflicts
+> with the `numpy>=2.0` that PyTorch/transformers need and will break the
+> training/profiling runs. The analysis step
+> (`rocm-compute-profiler/slurm_single_process_analyze.sh`) provisions its own
+> isolated venv for those dependencies and reuses the ROCm install from this
+> venv, so this shared venv stays on `numpy>=2.0`.
 
-```bash
-pip install -r ~/venvs/rocm-pytorch-pip/lib/python3.12/site-packages/_rocm_profiler/libexec/rocprofiler-compute/requirements.txt
-```
-
-These extra Python packages are needed by `rocprof-compute analyze` (the
-reporting/analysis step), not by counter collection itself.
-
-## 5. Extract development headers and device code
+## 4. Extract development headers and device code
 
 ```bash
 ~/venvs/rocm-pytorch-pip/bin/rocm-sdk init
@@ -76,7 +75,7 @@ next step.
 
 ---
 
-## 6. Write `setup_rocm.sh`
+## 5. Write `setup_rocm.sh`
 
 Create `~/setup_rocm.sh` to activate the venv and point the ROCm environment at
 the extracted `_rocm_sdk_devel` tree. Adjust `python3.12` to your Python version.
@@ -96,7 +95,7 @@ export LD_LIBRARY_PATH="$DEVEL/lib:$DEVEL/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH"
 echo "ROCm venv active: $VENV"
 ```
 
-## 7. Re-source to pick up the ROCm env vars
+## 6. Re-source to pick up the ROCm env vars
 
 If the venv is already active from step 1, deactivate and source the script so
 the `ROCM_PATH` / `LD_LIBRARY_PATH` exports take effect:
@@ -108,7 +107,7 @@ source setup_rocm.sh
 
 ---
 
-## 8. Verify (on a GPU node)
+## 7. Verify (on a GPU node)
 
 ```bash
 source ~/setup_rocm.sh
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
index 8e8eeaa0..4e895c24 100755
--- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
@@ -11,20 +11,19 @@
 # SLURM script: analyze the workload produced by slurm_single_process_profile.sh
 # with rocprofiler-compute (rocprof-compute analyze).
 #
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
+# Instead of downgrading numpy inside the shared ROCm PyTorch venv (and
+# restoring it afterwards), this uses a SEPARATE, isolated virtual environment
+# that contains only rocprof-compute's own pinned requirements (numpy==1.26.4,
+# pandas, dash, textual, ...). The shared venv is never modified, so this can
+# run safely even while training/profiling jobs use the shared venv.
 #
-# Analysis is CPU-only (it parses the counter database), so no GPU is requested.
+# rocprof-compute itself is NOT reinstalled into the analysis venv: the bundled
+# tool that ships inside the shared venv (the `_rocm_profiler` package) is
+# reused by pointing PATH / PYTHONPATH / LD_LIBRARY_PATH at it. Only the pure
+# Python analysis dependencies live in the dedicated venv.
 #
-# !!! IMPORTANT - WHEN TO RUN THIS SCRIPT !!!
-#   1. Run this ONLY AFTER slurm_single_process_profile.sh has completed, so
-#      the workload counter database exists.
-#   2. Do NOT run this while any other job that uses the same venv is running.
-#      This script temporarily pins numpy==1.26.4 (required by rocprof-compute
-#      analyze) in the SHARED venv and restores it on exit. If a training or
-#      profiling job runs concurrently, it will observe the wrong numpy version
-#      and crash on import. Run all such jobs to completion first, then run
-#      this analysis by itself.
+# Analysis is CPU-only (it parses the counter database), so no GPU is requested.
+# Run this after slurm_single_process_profile.sh has completed.
 # ---------------------------------------------------------------------------
 
 set -e
@@ -43,37 +42,75 @@ echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
 # ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
+# Locate the shared ROCm PyTorch venv (built via ROCM_PYTORCH_PIP_VENV_SETUP.md
+# and activated by ../setup_rocm.sh) WITHOUT activating it. We only need the
+# ROCm install that lives inside it: the bundled rocprof-compute launcher and
+# the ROCm shared libraries.
 # ---------------------------------------------------------------------------
-source ${PROFILER_TOP_DIR}/setup_rocm.sh
-rocprof-compute --version
+MAIN_VENV="${HOME}/venvs/rocm-pytorch-pip"
+if [[ ! -x "${MAIN_VENV}/bin/python3" ]]; then
+    echo "ERROR: shared ROCm venv not found at ${MAIN_VENV}" >&2
+    exit 1
+fi
+# site-packages of the shared venv (avoids hard-coding the python3.x version).
+MAIN_SP="$("${MAIN_VENV}/bin/python3" -c 'import site; print(site.getsitepackages()[0])')"
+ROCM_PROFILER="${MAIN_SP}/_rocm_profiler"
+ROCM_CORE="${MAIN_SP}/_rocm_sdk_core"
+ROCM_DEVEL="${MAIN_SP}/_rocm_sdk_devel"
+COMPUTE_LIBEXEC="${ROCM_PROFILER}/libexec/rocprofiler-compute"
+REQ_FILE="${COMPUTE_LIBEXEC}/requirements.txt"
+RPC="${ROCM_PROFILER}/bin/rocprof-compute"
+
+for p in "${RPC}" "${REQ_FILE}" "${ROCM_CORE}" "${ROCM_DEVEL}"; do
+    if [[ ! -e "${p}" ]]; then
+        echo "ERROR: expected ROCm component not found: ${p}" >&2
+        exit 1
+    fi
+done
 
 # ---------------------------------------------------------------------------
-# rocprof-compute analyze pins numpy==1.26.4, but other tools that share this
-# venv (e.g. the roofline extractor's scipy) can bump numpy to 2.x, which makes
-# analyze abort with a version-requirement error. Temporarily install the
-# required numpy for the analysis, then restore whatever version was there
-# before so the shared venv is left unchanged (restored even on failure).
+# Create / refresh the dedicated analysis venv with just rocprof-compute's
+# pinned requirements (numpy==1.26.4 etc.). This is isolated from the shared
+# venv (no --system-site-packages), so it can never be shadowed by the shared
+# venv's numpy 2.x. Re-installation is skipped unless requirements.txt changed.
 # ---------------------------------------------------------------------------
-REQUIRED_NUMPY=1.26.4
-ORIG_NUMPY="$(python3 -c 'import numpy; print(numpy.__version__)' 2>/dev/null)"
-
-restore_numpy() {
-    if [[ -n "${ORIG_NUMPY}" && "${ORIG_NUMPY}" != "${REQUIRED_NUMPY}" ]]; then
-        echo "Restoring numpy==${ORIG_NUMPY}"
-        pip install "numpy==${ORIG_NUMPY}"
-    fi
-}
+ANALYZE_VENV="${HOME}/venvs/rocprof-compute-analyze"
+if [[ ! -x "${ANALYZE_VENV}/bin/python3" ]]; then
+    echo "Creating analysis venv at ${ANALYZE_VENV}"
+    /usr/bin/python3 -m venv "${ANALYZE_VENV}"
+fi
+source "${ANALYZE_VENV}/bin/activate"
 
-if [[ "${ORIG_NUMPY}" != "${REQUIRED_NUMPY}" ]]; then
-    echo "Installing numpy==${REQUIRED_NUMPY} for rocprof-compute analyze"
-    pip install "numpy==${REQUIRED_NUMPY}"
-    trap restore_numpy EXIT
+STAMP="${ANALYZE_VENV}/.rocprof_compute_reqs_installed"
+if [[ ! -f "${STAMP}" || "${REQ_FILE}" -nt "${STAMP}" ]]; then
+    echo "Installing rocprof-compute requirements into ${ANALYZE_VENV}"
+    python3 -m pip install --upgrade pip
+    python3 -m pip install -r "${REQ_FILE}"
+    cp "${REQ_FILE}" "${STAMP}"
+else
+    echo "Analysis venv already satisfies ${REQ_FILE}"
 fi
 
+# ---------------------------------------------------------------------------
+# Point the isolated venv at the ROCm install in the shared venv so the bundled
+# rocprof-compute launcher and its shared libraries are found.
+#   * PYTHONPATH  -> the rocprofiler-compute python sources (libexec).
+#   * LD_LIBRARY_PATH -> ROCm core/devel libs (+ bundled sysdeps).
+#   * PATH        -> append the ROCm devel bin LAST so it can never shadow the
+#                    analysis venv's `python3` (isolation must be preserved).
+# The bundled launcher (${RPC}) is invoked by absolute path and re-execs with
+# `#!/usr/bin/env python3`, which resolves to the analysis venv's python since
+# its bin dir is first on PATH after activation -> numpy==1.26.4 is used.
+# ---------------------------------------------------------------------------
+export PYTHONPATH="${COMPUTE_LIBEXEC}:${PYTHONPATH}"
+export LD_LIBRARY_PATH="${ROCM_CORE}/lib:${ROCM_CORE}/lib/rocm_sysdeps/lib:${ROCM_DEVEL}/lib:${ROCM_DEVEL}/lib/rocm_sysdeps/lib:${LD_LIBRARY_PATH}"
+export ROCM_PATH="${ROCM_DEVEL}"
+export HIP_PATH="${ROCM_DEVEL}"
+export PATH="${PATH}:${ROCM_DEVEL}/bin"
+
+echo "Analysis python : $(which python3)  (numpy $(python3 -c 'import numpy; print(numpy.__version__)'))"
+"${RPC}" --version
+
 # ---------------------------------------------------------------------------
 # Locate the workload written by slurm_single_process_profile.sh. This must
 # match WORKLOAD_NAME / WORK_ROOT in that script.
@@ -104,7 +141,7 @@ echo "==================================================================="
 echo "rocprof-compute analyze --list-stats -p ${ARCH_DIR}"
 echo "==================================================================="
 STATS_FILE=${SCRIPT_DIR}/stats_${SLURM_JOB_ID}.txt
-rocprof-compute analyze --list-stats -p ${ARCH_DIR} >& ${STATS_FILE}
+"${RPC}" analyze --list-stats -p ${ARCH_DIR} >& ${STATS_FILE}
 echo "Stats and dispatch IDs written to ${STATS_FILE}"
 
 echo
@@ -112,7 +149,7 @@ echo "==================================================================="
 echo "rocprof-compute analyze -p ${ARCH_DIR}"
 echo "==================================================================="
 ANALYSIS_FILE=${SCRIPT_DIR}/analysis_${SLURM_JOB_ID}.txt
-rocprof-compute analyze -p ${ARCH_DIR} > ${ANALYSIS_FILE} 2>&1
+"${RPC}" analyze -p ${ARCH_DIR} > ${ANALYSIS_FILE} 2>&1
 echo "Analysis written to ${ANALYSIS_FILE}"
 
 echo
@@ -122,6 +159,6 @@ echo "specific kernel, inspect ${STATS_FILE} for the kernel names and"
 echo "dispatch IDs, then re-run rocprof-compute analyze narrowing to that"
 echo "kernel with --kernel <kernel-id>, or to a specific invocation with"
 echo "--dispatch <dispatch-id> (you may pass either one, or both):"
-echo "  rocprof-compute analyze -p ${ARCH_DIR} --kernel <kernel-id>"
-echo "  rocprof-compute analyze -p ${ARCH_DIR} --dispatch <dispatch-id>"
+echo "  ${RPC} analyze -p ${ARCH_DIR} --kernel <kernel-id>"
+echo "  ${RPC} analyze -p ${ARCH_DIR} --dispatch <dispatch-id>"
 echo "-------------------------------------------------------------------"

From a7cbcb177493455172c6d66442c24f799d91b29d Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Tue, 21 Jul 2026 16:58:52 -0500
Subject: [PATCH 4/6] Move to a newer nightly build that has a key
 rocprof-compute fix

---
 MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md | 2 +-
 MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
index 522c98fb..0ce201bd 100644
--- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -32,7 +32,7 @@ create the `rocm-pytorch-pip` venv. In short, it:
 To test a **specific nightly**, set the ROCm version pin in that guide, e.g.:
 
 ```bash
-ROCM_VERSION=7.15.0a20260707
+ROCM_VERSION=7.15.0a20260716
 ```
 
 Change this value to the nightly date you want to validate.
diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
index 93fe9fdc..aba03981 100644
--- a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
+++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
@@ -30,7 +30,7 @@ source ~/venvs/rocm-pytorch-pip/bin/activate
 
 ```bash
 # Pin the nightly ROCm version once and reuse it everywhere below.
-ROCM_VERSION=7.15.0a20260707
+ROCM_VERSION=7.15.0a20260716
 
 pip install --index-url https://rocm.nightlies.amd.com/whl-multi-arch/ \
     "rocm[profiler,devel,libraries,device-gfx942]==${ROCM_VERSION}" \
@@ -119,7 +119,7 @@ Expected output resembles:
 
 ```
 ROCm venv active: /.../rocm-pytorch-pip
-torch 2.12.0+rocm7.15.0a20260707
+torch 2.12.0+rocm7.15.0a20260716
 device ok: 8.0
 ```
 

From d847361fbc71049acfaacdb288fd1de80e27b83c Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Tue, 21 Jul 2026 17:24:48 -0500
Subject: [PATCH 5/6] Address CoPilot review comments

---
 .../README_ROCM_NIGHTLY_TESTING.md            | 33 ++++++-------------
 .../ROCM_PYTORCH_PIP_VENV_SETUP.md            | 24 +++++++-------
 .../slurm_single_process_analyze.sh           |  6 +++-
 .../slurm_single_process.sh                   |  4 +--
 .../slurm_single_process.sh                   |  2 +-
 MLExamples/PyTorch_Profiling/setup_rocm.sh    |  4 ++-
 6 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
index 0ce201bd..4ab476f7 100644
--- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -37,29 +37,17 @@ ROCM_VERSION=7.15.0a20260716
 
 Change this value to the nightly date you want to validate.
 
-## Step 2 — Create `setup_rocm.sh`
+## Step 2 — Verify `setup_rocm.sh`
 
 The SLURM scripts in each sub-directory activate the environment by sourcing
-`../setup_rocm.sh` — i.e. a `setup_rocm.sh` located in **this**
-(`PyTorch_Profiling/`) directory. Create it here (see step 5 of the venv guide
-for the full contents), pointing at the venv you just built:
+`../setup_rocm.sh` — i.e. the `setup_rocm.sh` shipped in **this**
+(`PyTorch_Profiling/`) directory. It is already provided; just verify (and edit
+if needed) that its `VENV` points at the venv you built in Step 1. Its full
+contents, and a GPU-node sanity check, are covered in steps 5-7 of
+[`ROCM_PYTORCH_PIP_VENV_SETUP.md`](./ROCM_PYTORCH_PIP_VENV_SETUP.md).
 
-```bash
-#!/usr/bin/env bash
-VENV="$HOME/venvs/rocm-pytorch-pip"
-DEVEL="$VENV/lib/python3.12/site-packages/_rocm_sdk_devel"
-source "$VENV/bin/activate"
-export ROCM_PATH="$DEVEL"
-export HIP_PATH="$DEVEL"
-export HIP_DEVICE_LIB_PATH="$DEVEL/lib/llvm/amdgcn/bitcode"
-export PATH="$DEVEL/bin:$PATH"
-export LD_LIBRARY_PATH="$DEVEL/lib:$DEVEL/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH"
-echo "ROCm venv active: $VENV"
-```
-
-Adjust `python3.12` to match your Python version.
-
-Verify the environment on a GPU node before profiling:
+Once verified, a quick check that the nightly build runs GPU kernels through
+PyTorch:
 
 ```bash
 source setup_rocm.sh
@@ -67,15 +55,14 @@ srun -n1 --gpus=1 python3 -c "import torch; print('torch', torch.__version__); \
 x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())"
 ```
 
-If you see `device ok:`, the nightly ROCm build runs GPU kernels through PyTorch
-and you are ready to profile.
+If you see `device ok:`, you are ready to profile.
 
 ## Step 3 — Run the SLURM scripts
 
 Each sub-directory contains a single-process SLURM script that sources
 `../setup_rocm.sh`, pre-downloads the dataset if needed, and runs the workload
 under one tool. All of them use a single GPU and a short run
-(`--batch-size 64 --max-steps 5`) so a nightly can be checked quickly.
+(`--batch-size 32 --max-steps 5`) so a nightly can be checked quickly.
 
 **Submit each script from its own directory** (the scripts use
 `SLURM_SUBMIT_DIR` to locate themselves):
diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
index aba03981..2a7fcf0c 100644
--- a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
+++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
@@ -9,8 +9,7 @@ profilers use the same ROCm. Each step below is a single command block you can
 copy and run.
 
 > This guide targets an AMD MI300A GPU (`gfx942`). If you have a different GPU,
-> change `device-gfx942` to your architecture. Also make sure the Python version
-> in the paths below (`python3.12`) matches your `python --version`.
+> change `device-gfx942` to your architecture.
 
 ---
 
@@ -22,10 +21,6 @@ python -m venv ~/venvs/rocm-pytorch-pip
 source ~/venvs/rocm-pytorch-pip/bin/activate
 ```
 
-> Tip: the setup script below hardcodes the Python minor version in the
-> `site-packages` path (e.g. `python3.12`). Check what `python --version` (or
-> `module load python`) gives you and keep the two consistent.
-
 ## 2. Install ROCm + PyTorch from the nightly multi-arch index
 
 ```bash
@@ -75,18 +70,22 @@ next step.
 
 ---
 
-## 5. Write `setup_rocm.sh`
+## 5. Verify `setup_rocm.sh`
 
-Create `~/setup_rocm.sh` to activate the venv and point the ROCm environment at
-the extracted `_rocm_sdk_devel` tree. Adjust `python3.12` to your Python version.
+The repo already ships `setup_rocm.sh` in `MLExamples/PyTorch_Profiling/` (the
+SLURM scripts source it as `../setup_rocm.sh`). It activates the venv and points
+the ROCm environment at the extracted `_rocm_sdk_devel` tree. Verify (and edit
+if needed) that its `VENV` matches the venv you built in step 1. The
+site-packages path is derived from the active venv's `python3`, so it works
+regardless of the Python minor version. Its contents are:
 
 ```bash
 #!/usr/bin/env bash
 # Source this to activate the ROCm venv and set ROCm env vars:
 #   source setup_rocm.sh
 VENV="$HOME/venvs/rocm-pytorch-pip"
-DEVEL="$VENV/lib/python3.12/site-packages/_rocm_sdk_devel"
 source "$VENV/bin/activate"
+DEVEL="$(python3 -c 'import site; print(site.getsitepackages()[0])')/_rocm_sdk_devel"
 export ROCM_PATH="$DEVEL"
 export HIP_PATH="$DEVEL"
 export HIP_DEVICE_LIB_PATH="$DEVEL/lib/llvm/amdgcn/bitcode"
@@ -98,7 +97,8 @@ echo "ROCm venv active: $VENV"
 ## 6. Re-source to pick up the ROCm env vars
 
 If the venv is already active from step 1, deactivate and source the script so
-the `ROCM_PATH` / `LD_LIBRARY_PATH` exports take effect:
+the `ROCM_PATH` / `LD_LIBRARY_PATH` exports take effect (run from
+`MLExamples/PyTorch_Profiling/`):
 
 ```bash
 deactivate
@@ -110,7 +110,7 @@ source setup_rocm.sh
 ## 7. Verify (on a GPU node)
 
 ```bash
-source ~/setup_rocm.sh
+source setup_rocm.sh
 srun -n1 --gpus=1 python3 -c "import torch; print('torch', torch.__version__); \
 x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())"
 ```
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
index 4e895c24..f88201a7 100755
--- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
@@ -77,7 +77,11 @@ done
 ANALYZE_VENV="${HOME}/venvs/rocprof-compute-analyze"
 if [[ ! -x "${ANALYZE_VENV}/bin/python3" ]]; then
     echo "Creating analysis venv at ${ANALYZE_VENV}"
-    /usr/bin/python3 -m venv "${ANALYZE_VENV}"
+    # Use the shared venv's python3 (validated above) to create the analysis
+    # venv. A venv is isolated from its creator's site-packages, so this stays
+    # independent of the shared venv while avoiding a hard-coded /usr/bin/python3
+    # that may be absent or lack the venv module on some clusters.
+    "${MAIN_VENV}/bin/python3" -m venv "${ANALYZE_VENV}"
 fi
 source "${ANALYZE_VENV}/bin/activate"
 
diff --git a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
index ca9a4506..e8a283fc 100755
--- a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-#SBATCH --job-name=rps-single-sample
+#SBATCH --job-name=rps-single-run
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
 #SBATCH --gpus=1
@@ -10,7 +10,7 @@
 
 # ---------------------------------------------------------------------------
 # SLURM script: profile the single-process CIFAR-100 workload with the ROCm
-# Systems Profiler (rocprof-sys-sample).
+# Systems Profiler (rocprof-sys-run).
 #
 # It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
 # matching ROCm env vars.
diff --git a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
index 5bfd9e82..0c6d9195 100755
--- a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
+++ b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
@@ -52,7 +52,7 @@ RE_DIR=${SCRIPT_DIR}/rooflineExtractor
 if [ ! -d ${RE_DIR} ]; then
     git clone https://github.com/AMD-HPC/rooflineExtractor.git ${RE_DIR}
 fi
-pip install -r ${RE_DIR}/requirements.txt
+python3 -m pip install -r ${RE_DIR}/requirements.txt
 
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
diff --git a/MLExamples/PyTorch_Profiling/setup_rocm.sh b/MLExamples/PyTorch_Profiling/setup_rocm.sh
index f074f414..0f1d3583 100644
--- a/MLExamples/PyTorch_Profiling/setup_rocm.sh
+++ b/MLExamples/PyTorch_Profiling/setup_rocm.sh
@@ -2,8 +2,10 @@
 # Source this to activate the ROCm venv and set ROCm env vars:
 #   source setup_rocm.sh
 VENV="${HOME}/venvs/rocm-pytorch-pip"
-DEVEL="${VENV}/lib/python3.12/site-packages/_rocm_sdk_devel"
 source "${VENV}/bin/activate"
+# Derive site-packages from the active venv so this works regardless of the
+# venv's Python minor version (e.g. python3.12 vs python3.13).
+DEVEL="$(python3 -c 'import site; print(site.getsitepackages()[0])')/_rocm_sdk_devel"
 export ROCM_PATH="$DEVEL"
 export HIP_PATH="$DEVEL"
 export HIP_DEVICE_LIB_PATH="${DEVEL}/lib/llvm/amdgcn/bitcode"

From a304380de47c1c29f0c9da610aca5b96a7360fdb Mon Sep 17 00:00:00 2001
From: Gina Sitaraman <Gina.Sitaraman@amd.com>
Date: Wed, 22 Jul 2026 17:46:31 -0500
Subject: [PATCH 6/6] Shorten comments, remove queue name, move to newer
 nightly

---
 .../README_ROCM_NIGHTLY_TESTING.md            | 22 ++---
 .../ROCM_PYTORCH_PIP_VENV_SETUP.md            | 42 ++++----
 .../slurm_single_process_noprofile.sh         | 32 ++-----
 .../slurm_single_process_analyze.sh           | 95 +++++++------------
 .../slurm_single_process_profile.sh           | 58 +++--------
 .../slurm_single_process.sh                   | 29 ++----
 .../rocprofv3/slurm_single_process_kernels.sh | 29 ++----
 .../rocprofv3/slurm_single_process_traces.sh  | 29 ++----
 .../slurm_single_process.sh                   | 46 +++------
 MLExamples/PyTorch_Profiling/setup_rocm.sh    |  3 +-
 10 files changed, 123 insertions(+), 262 deletions(-)

diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
index 4ab476f7..e9a66f75 100644
--- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
+++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md
@@ -32,7 +32,7 @@ create the `rocm-pytorch-pip` venv. In short, it:
 To test a **specific nightly**, set the ROCm version pin in that guide, e.g.:
 
 ```bash
-ROCM_VERSION=7.15.0a20260716
+ROCM_VERSION=7.15.0a20260721
 ```
 
 Change this value to the nightly date you want to validate.
@@ -70,7 +70,8 @@ under one tool. All of them use a single GPU and a short run
 | Tool | Directory | Script | What it produces |
 |------|-----------|--------|------------------|
 | None (baseline) | `no-profiling/` | `slurm_single_process_noprofile.sh` | Plain training run — confirms the workload runs without a profiler. |
-| ROCm Compute Profiler | `rocm-compute-profiler/` | `slurm_single_process_profile.sh` | Hardware-counter profile under `workloads/` (analyze with `rocprof-compute analyze`). |
+| ROCm Compute Profiler (profile) | `rocm-compute-profiler/` | `slurm_single_process_profile.sh` | Hardware-counter profile under `workloads/`. |
+| ROCm Compute Profiler (analyze) | `rocm-compute-profiler/` | `slurm_single_process_analyze.sh` | Analysis report from the profile above. |
 | RocProfiler (kernels) | `rocprofv3/` | `slurm_single_process_kernels.sh` | Kernel stats + trace CSVs under `single_process/`. |
 | RocProfiler (traces) | `rocprofv3/` | `slurm_single_process_traces.sh` | System timeline trace (`.pftrace`) under `single_process/`. |
 | ROCm Systems Profiler | `rocm-systems-profiler/` | `slurm_single_process.sh` | Sampling profile + trace under `rocprofsys-python3-output/`. |
@@ -95,19 +96,14 @@ squeue --me
 
 ### Analyzing the results
 
-> **Note — running `rocprof-compute analyze`:** run the analysis step only
-> **after** its profile job has finished (the counter database must exist).
-> `rocprof-compute analyze` requires `numpy==1.26.4`, which conflicts with the
-> `numpy>=2.0` needed by the training/profiling jobs. To avoid disturbing the
-> shared venv, the analysis script uses a separate, isolated venv
-> (`~/venvs/rocprof-compute-analyze`) that holds only rocprof-compute's pinned
-> requirements, while reusing the ROCm install from the shared venv. The shared
-> venv is never modified, so the analysis can run even while training or
-> profiling jobs are using it.
+> **Note — running `rocprof-compute analyze`:** run it only **after** its
+> profile job has finished (the counter database must exist). Because it needs
+> `numpy==1.26.4` (vs the shared venv's `numpy>=2.0`), the analysis script uses
+> its own isolated venv (`~/venvs/rocprof-compute-analyze`) and never touches
+> the shared venv.
 
 - **ROCm Compute Profiler:** submit the companion analysis job from
-  `rocm-compute-profiler/` (it locates the workload and runs the analysis for
-  you, using the isolated analysis venv described above):
+  `rocm-compute-profiler/` (it locates the workload and runs the analysis for you):
 
 ```bash
 cd rocm-compute-profiler
diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
index 2a7fcf0c..1d023d59 100644
--- a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
+++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md
@@ -15,17 +15,22 @@ copy and run.
 
 ## 1. Create and activate the venv
 
+First choose where the venv should live. Set `VENV_BASE` to the directory that
+will hold the `venvs` folder (defaults to your home directory). Keep this shell
+open for the remaining steps, which reuse the variable.
+
 ```bash
-mkdir -p ~/venvs
-python -m venv ~/venvs/rocm-pytorch-pip
-source ~/venvs/rocm-pytorch-pip/bin/activate
+VENV_BASE=~
+mkdir -p "${VENV_BASE}/venvs"
+python -m venv "${VENV_BASE}/venvs/rocm-pytorch-pip"
+source "${VENV_BASE}/venvs/rocm-pytorch-pip/bin/activate"
 ```
 
 ## 2. Install ROCm + PyTorch from the nightly multi-arch index
 
 ```bash
 # Pin the nightly ROCm version once and reuse it everywhere below.
-ROCM_VERSION=7.15.0a20260716
+ROCM_VERSION=7.15.0a20260721
 
 pip install --index-url https://rocm.nightlies.amd.com/whl-multi-arch/ \
     "rocm[profiler,devel,libraries,device-gfx942]==${ROCM_VERSION}" \
@@ -49,24 +54,15 @@ pip install transformers
 The training script builds its models with `transformers`, so this package is
 required.
 
-> **Note:** Do **not** install `rocprof-compute analyze`'s `requirements.txt`
-> into this shared venv. Those packages pin `numpy==1.26.4`, which conflicts
-> with the `numpy>=2.0` that PyTorch/transformers need and will break the
-> training/profiling runs. The analysis step
-> (`rocm-compute-profiler/slurm_single_process_analyze.sh`) provisions its own
-> isolated venv for those dependencies and reuses the ROCm install from this
-> venv, so this shared venv stays on `numpy>=2.0`.
-
 ## 4. Extract development headers and device code
 
 ```bash
-~/venvs/rocm-pytorch-pip/bin/rocm-sdk init
+"${VENV_BASE}/venvs/rocm-pytorch-pip/bin/rocm-sdk" init
 ```
 
-`rocm-sdk init` unpacks the `devel` payload (headers, LLVM device bitcode, etc.)
-into `_rocm_sdk_devel/` inside the venv. This provides the device bitcode that
-HIP needs to run GPU kernels, and the paths that `setup_rocm.sh` points to in the
-next step.
+`rocm-sdk init` unpacks the `devel` payload (headers, LLVM device bitcode) into
+`_rocm_sdk_devel/` inside the venv — the device bitcode HIP needs to run kernels,
+and the paths `setup_rocm.sh` points at next.
 
 ---
 
@@ -74,16 +70,16 @@ next step.
 
 The repo already ships `setup_rocm.sh` in `MLExamples/PyTorch_Profiling/` (the
 SLURM scripts source it as `../setup_rocm.sh`). It activates the venv and points
-the ROCm environment at the extracted `_rocm_sdk_devel` tree. Verify (and edit
-if needed) that its `VENV` matches the venv you built in step 1. The
-site-packages path is derived from the active venv's `python3`, so it works
-regardless of the Python minor version. Its contents are:
+the ROCm environment at the extracted `_rocm_sdk_devel` tree. It defaults
+`VENV_BASE` to your home directory; if you used a different `VENV_BASE` in
+step 1, export it before sourcing (or edit the default here). Its contents are:
 
 ```bash
 #!/usr/bin/env bash
 # Source this to activate the ROCm venv and set ROCm env vars:
 #   source setup_rocm.sh
-VENV="$HOME/venvs/rocm-pytorch-pip"
+VENV_BASE="${VENV_BASE:-$HOME}"
+VENV="$VENV_BASE/venvs/rocm-pytorch-pip"
 source "$VENV/bin/activate"
 DEVEL="$(python3 -c 'import site; print(site.getsitepackages()[0])')/_rocm_sdk_devel"
 export ROCM_PATH="$DEVEL"
@@ -119,7 +115,7 @@ Expected output resembles:
 
 ```
 ROCm venv active: /.../rocm-pytorch-pip
-torch 2.12.0+rocm7.15.0a20260716
+torch 2.12.0+rocm7.15.0a20260721
 device ok: 8.0
 ```
 
diff --git a/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
index b8b111cd..11fef41b 100755
--- a/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
+++ b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh
@@ -2,28 +2,21 @@
 #SBATCH --job-name=rpc-single-noprofile
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=00:30:00
 #SBATCH --output=rpc_single_process_noprofile_%j.out
 #SBATCH --error=rpc_single_process_noprofile_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: run the single-process CIFAR-100 workload DIRECTLY with
-# python3, WITHOUT rocprof-compute.
-#
-# This is the no-profiler counterpart to slurm_single_process_profile.sh.
-# It sources the same environment (../setup_rocm.sh) and runs the same python3
-# command, which is useful for validating the ROCm + PyTorch environment before
-# profiling.
+# Run the single-process CIFAR-100 workload directly with python3 (no profiler).
+# Sources ../setup_rocm.sh to activate the ROCm PyTorch venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/no-profiling/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/no-profiling/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -33,20 +26,13 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars, so python3 resolves from
-# the same install used by the profiling driver.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
+# Derive a per-job port so concurrent jobs don't collide.
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
 # Make sure the dataset is present.
@@ -57,14 +43,12 @@ fi
 
 cd ${SCRIPT_DIR}
 
-# ---------------------------------------------------------------------------
 # Run the workload directly (no profiler).
-# ---------------------------------------------------------------------------
 echo
 echo "==================================================================="
 echo "Running (no profiler): python3 train_cifar_100.py"
 echo "==================================================================="
-srun -n 1 --gpus=1 \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
         --batch-size 32 --max-steps 5 \
         --data-path ${PROFILER_TOP_DIR}/data
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
index f88201a7..e587d8ae 100755
--- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh
@@ -2,36 +2,24 @@
 #SBATCH --job-name=rpc-single-analyze
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
-#SBATCH --partition=PPAC_MI300A_SPX
+#SBATCH --cpus-per-task=8
 #SBATCH --time=00:15:00
 #SBATCH --output=rpc_single_process_analyze_%j.out
 #SBATCH --error=rpc_single_process_analyze_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: analyze the workload produced by slurm_single_process_profile.sh
-# with rocprofiler-compute (rocprof-compute analyze).
+# Analyze the profiled workload with rocprof-compute analyze.
 #
-# Instead of downgrading numpy inside the shared ROCm PyTorch venv (and
-# restoring it afterwards), this uses a SEPARATE, isolated virtual environment
-# that contains only rocprof-compute's own pinned requirements (numpy==1.26.4,
-# pandas, dash, textual, ...). The shared venv is never modified, so this can
-# run safely even while training/profiling jobs use the shared venv.
-#
-# rocprof-compute itself is NOT reinstalled into the analysis venv: the bundled
-# tool that ships inside the shared venv (the `_rocm_profiler` package) is
-# reused by pointing PATH / PYTHONPATH / LD_LIBRARY_PATH at it. Only the pure
-# Python analysis dependencies live in the dedicated venv.
-#
-# Analysis is CPU-only (it parses the counter database), so no GPU is requested.
-# Run this after slurm_single_process_profile.sh has completed.
+# rocprof-compute analyze needs numpy 1.26.x, which conflicts with the shared
+# venv's numpy 2.x. So we use a separate venv holding only rocprof-compute's
+# pinned requirements, while reusing the tool bundled in the shared venv (the
+# shared venv is never modified). CPU-only; run after the profile job finishes.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/rocm-compute-profiler/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/rocm-compute-profiler/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -41,12 +29,8 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Locate the shared ROCm PyTorch venv (built via ROCM_PYTORCH_PIP_VENV_SETUP.md
-# and activated by ../setup_rocm.sh) WITHOUT activating it. We only need the
-# ROCm install that lives inside it: the bundled rocprof-compute launcher and
-# the ROCm shared libraries.
-# ---------------------------------------------------------------------------
+# Locate the shared ROCm venv WITHOUT activating it; we only need the ROCm
+# install inside it (bundled rocprof-compute launcher + ROCm libraries).
 MAIN_VENV="${HOME}/venvs/rocm-pytorch-pip"
 if [[ ! -x "${MAIN_VENV}/bin/python3" ]]; then
     echo "ERROR: shared ROCm venv not found at ${MAIN_VENV}" >&2
@@ -68,19 +52,13 @@ for p in "${RPC}" "${REQ_FILE}" "${ROCM_CORE}" "${ROCM_DEVEL}"; do
     fi
 done
 
-# ---------------------------------------------------------------------------
-# Create / refresh the dedicated analysis venv with just rocprof-compute's
-# pinned requirements (numpy==1.26.4 etc.). This is isolated from the shared
-# venv (no --system-site-packages), so it can never be shadowed by the shared
-# venv's numpy 2.x. Re-installation is skipped unless requirements.txt changed.
-# ---------------------------------------------------------------------------
+# Create/refresh the dedicated analysis venv with rocprof-compute's pinned
+# requirements (numpy 1.26.x). Reinstalled only when requirements.txt changes.
 ANALYZE_VENV="${HOME}/venvs/rocprof-compute-analyze"
 if [[ ! -x "${ANALYZE_VENV}/bin/python3" ]]; then
     echo "Creating analysis venv at ${ANALYZE_VENV}"
-    # Use the shared venv's python3 (validated above) to create the analysis
-    # venv. A venv is isolated from its creator's site-packages, so this stays
-    # independent of the shared venv while avoiding a hard-coded /usr/bin/python3
-    # that may be absent or lack the venv module on some clusters.
+    # A venv stays isolated from its creator's site-packages, so using the
+    # shared venv's python3 keeps this independent while avoiding system python3.
     "${MAIN_VENV}/bin/python3" -m venv "${ANALYZE_VENV}"
 fi
 source "${ANALYZE_VENV}/bin/activate"
@@ -95,17 +73,9 @@ else
     echo "Analysis venv already satisfies ${REQ_FILE}"
 fi
 
-# ---------------------------------------------------------------------------
-# Point the isolated venv at the ROCm install in the shared venv so the bundled
-# rocprof-compute launcher and its shared libraries are found.
-#   * PYTHONPATH  -> the rocprofiler-compute python sources (libexec).
-#   * LD_LIBRARY_PATH -> ROCm core/devel libs (+ bundled sysdeps).
-#   * PATH        -> append the ROCm devel bin LAST so it can never shadow the
-#                    analysis venv's `python3` (isolation must be preserved).
-# The bundled launcher (${RPC}) is invoked by absolute path and re-execs with
-# `#!/usr/bin/env python3`, which resolves to the analysis venv's python since
-# its bin dir is first on PATH after activation -> numpy==1.26.4 is used.
-# ---------------------------------------------------------------------------
+# Point the isolated venv at the ROCm install in the shared venv (bundled
+# rocprof-compute + libs). ROCm bin goes LAST on PATH so it can't shadow the
+# analysis venv's python3, keeping numpy 1.26.x in effect.
 export PYTHONPATH="${COMPUTE_LIBEXEC}:${PYTHONPATH}"
 export LD_LIBRARY_PATH="${ROCM_CORE}/lib:${ROCM_CORE}/lib/rocm_sysdeps/lib:${ROCM_DEVEL}/lib:${ROCM_DEVEL}/lib/rocm_sysdeps/lib:${LD_LIBRARY_PATH}"
 export ROCM_PATH="${ROCM_DEVEL}"
@@ -115,10 +85,8 @@ export PATH="${PATH}:${ROCM_DEVEL}/bin"
 echo "Analysis python : $(which python3)  (numpy $(python3 -c 'import numpy; print(numpy.__version__)'))"
 "${RPC}" --version
 
-# ---------------------------------------------------------------------------
-# Locate the workload written by slurm_single_process_profile.sh. This must
-# match WORKLOAD_NAME / WORK_ROOT in that script.
-# ---------------------------------------------------------------------------
+# Locate the workload written by slurm_single_process_profile.sh (must match
+# WORKLOAD_NAME / WORK_ROOT there).
 WORKLOAD_NAME=cifar_100_single_proc
 WORK_ROOT=${SCRIPT_DIR}/workloads
 WORK_DIR=${WORK_ROOT}/${WORKLOAD_NAME}
@@ -140,13 +108,16 @@ fi
 # ---------------------------------------------------------------------------
 # Analyze.
 # ---------------------------------------------------------------------------
-echo
-echo "==================================================================="
-echo "rocprof-compute analyze --list-stats -p ${ARCH_DIR}"
-echo "==================================================================="
-STATS_FILE=${SCRIPT_DIR}/stats_${SLURM_JOB_ID}.txt
-"${RPC}" analyze --list-stats -p ${ARCH_DIR} >& ${STATS_FILE}
-echo "Stats and dispatch IDs written to ${STATS_FILE}"
+# Optional: list per-kernel stats and dispatch IDs. Uncomment only if you want
+# to narrow the analysis to a specific kernel/dispatch (see the NOTE at the
+# bottom of this file).
+# echo
+# echo "==================================================================="
+# echo "rocprof-compute analyze --list-stats -p ${ARCH_DIR}"
+# echo "==================================================================="
+# STATS_FILE=${SCRIPT_DIR}/stats_${SLURM_JOB_ID}.txt
+# "${RPC}" analyze --list-stats -p ${ARCH_DIR} >& ${STATS_FILE}
+# echo "Stats and dispatch IDs written to ${STATS_FILE}"
 
 echo
 echo "==================================================================="
@@ -159,10 +130,10 @@ echo "Analysis written to ${ANALYSIS_FILE}"
 echo
 echo "-------------------------------------------------------------------"
 echo "NOTE: The analysis above aggregates all kernels. To focus on a"
-echo "specific kernel, inspect ${STATS_FILE} for the kernel names and"
-echo "dispatch IDs, then re-run rocprof-compute analyze narrowing to that"
-echo "kernel with --kernel <kernel-id>, or to a specific invocation with"
-echo "--dispatch <dispatch-id> (you may pass either one, or both):"
+echo "specific kernel, uncomment the --list-stats block above to list the"
+echo "kernel names and dispatch IDs, then re-run rocprof-compute analyze"
+echo "narrowing to that kernel with --kernel <kernel-id>, or to a specific"
+echo "invocation with --dispatch <dispatch-id> (you may pass either, or both):"
 echo "  ${RPC} analyze -p ${ARCH_DIR} --kernel <kernel-id>"
 echo "  ${RPC} analyze -p ${ARCH_DIR} --dispatch <dispatch-id>"
 echo "-------------------------------------------------------------------"
diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
index a8c98c8a..be3974ed 100755
--- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh
@@ -2,28 +2,21 @@
 #SBATCH --job-name=rpc-single-profile
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=02:00:00
-# Reserve the whole node so no other job shares the GPUs during profiling.
-#SBATCH --exclusive
 #SBATCH --output=rpc_single_process_profile_%j.out
 #SBATCH --error=rpc_single_process_profile_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: profile the single-process CIFAR-100 workload with
-# rocprofiler-compute (rocprof-compute profile).
-#
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
+# Profile the single-process CIFAR-100 workload with rocprof-compute profile.
+# Sources ../setup_rocm.sh to activate the ROCm PyTorch venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/rocm-compute-profiler/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/rocm-compute-profiler/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -33,24 +26,17 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 rocprof-compute --version
 
-# Distributed bootstrap variables expected by train_cifar_100.py (single rank).
+# Distributed bootstrap vars for train_cifar_100.py (single rank). Derive a
+# per-job port so concurrent jobs don't collide.
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
-# Make sure the dataset is present (the wrapper also does this, but doing it
-# here keeps the profiled step from spending time on the download).
+# Pre-download the dataset so the profiled step doesn't spend time on it.
 if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then
     python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
         --data-path ${PROFILER_TOP_DIR}/data --download-only
@@ -64,27 +50,15 @@ mkdir -p ${WORK_ROOT}
 rm -rf ${WORK_DIR}
 cd ${SCRIPT_DIR}
 
-# ---------------------------------------------------------------------------
-# Profile.
-#
-# Notes:
-#  * --no-roof skips roofline capture (faster; not needed for this demo).
-#  * rocprof-compute replays the application multiple times to cover all
-#    hardware counters, so keep the workload short.
-#  * We profile python3 directly (not the no-profiling/*.sh bash wrapper).
-#    rocprof-compute 3.7.0 / rocprofiler-sdk v1.3.1 injects into the bash
-#    wrapper process too and aborts it with "Output path is empty", so the
-#    application target must be the python interpreter itself. The environment
-#    (modules) is already configured above and is inherited by the profiled
-#    process.
-# ---------------------------------------------------------------------------
+# Profile. Notes:
+#  * --no-roof skips roofline capture (not needed here).
+#  * rocprof-compute replays the app multiple times to cover all counters, so
+#    keep the workload short.
 echo
 echo "==================================================================="
 echo "rocprof-compute profile -- python3 train_cifar_100.py"
 echo "==================================================================="
-# With --exclusive the whole node is ours; give the single task every core on
-# the node (otherwise srun would pin it to just one CPU).
-srun -n 1 --gpus=1 --cpus-per-task=${SLURM_CPUS_ON_NODE} \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     rocprof-compute profile \
         --name ${WORKLOAD_NAME} \
         --no-roof \
@@ -93,9 +67,7 @@ srun -n 1 --gpus=1 --cpus-per-task=${SLURM_CPUS_ON_NODE} \
             --batch-size 32 --max-steps 5 \
             --data-path ${PROFILER_TOP_DIR}/data
 
-# Resolve the workload subdirectory written by rocprof-compute. Depending on
-# the version this is named after the architecture (e.g. `MI300A_*`) or
-# numerically (`0`, `1`, ...). Pick the first (typically only) subdirectory.
+# The profile subdir is named by arch (e.g. MI300A_*) or numerically; pick the first.
 ARCH_DIR=$(find ${WORK_DIR} -mindepth 1 -maxdepth 1 -type d | sort | head -1)
 if [[ -z "${ARCH_DIR}" ]]; then
     echo "ERROR: no workload subdirectory found under ${WORK_DIR}" >&2
diff --git a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
index e8a283fc..c4e73749 100755
--- a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
+++ b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh
@@ -2,26 +2,21 @@
 #SBATCH --job-name=rps-single-run
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=00:30:00
 #SBATCH --output=rps_single_process_%j.out
 #SBATCH --error=rps_single_process_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: profile the single-process CIFAR-100 workload with the ROCm
-# Systems Profiler (rocprof-sys-run).
-#
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
+# Profile the single-process CIFAR-100 workload with the ROCm Systems Profiler
+# (rocprof-sys-run). Sources ../setup_rocm.sh to activate the ROCm venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/rocm-systems-profiler/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/rocm-systems-profiler/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -31,20 +26,14 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 rocprof-sys-run --version
 
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
+# Derive a per-job port so concurrent jobs don't collide.
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
 # Make sure the dataset is present before the profiled step.
@@ -58,14 +47,12 @@ cd ${SCRIPT_DIR}
 # Remove any stale output directories from previous runs.
 rm -rf ${SCRIPT_DIR}/rocprofsys-python3-output
 
-# ---------------------------------------------------------------------------
 # Profile with rocprof-sys-run (profiling + tracing).
-# ---------------------------------------------------------------------------
 echo
 echo "==================================================================="
 echo "rocprof-sys-run --profile --trace -- python3 train_cifar_100.py"
 echo "==================================================================="
-srun -n 1 --gpus=1 \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     rocprof-sys-run --profile --trace -- \
         python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \
             --batch-size 32 --max-steps 5 \
diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
index de08539d..e4cb75a4 100755
--- a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
+++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh
@@ -2,26 +2,21 @@
 #SBATCH --job-name=rpv3-single-kernels
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=00:30:00
 #SBATCH --output=rpv3_single_process_kernels_%j.out
 #SBATCH --error=rpv3_single_process_kernels_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: profile the single-process CIFAR-100 workload with rocprofv3
-# kernel tracing (--stats --kernel-trace).
-#
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
+# Profile the single-process CIFAR-100 workload with rocprofv3 kernel tracing
+# (--stats --kernel-trace). Sources ../setup_rocm.sh to activate the ROCm venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/rocprofv3/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/rocprofv3/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -31,20 +26,14 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 rocprofv3 --version
 
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
+# Derive a per-job port so concurrent jobs don't collide.
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
 # Make sure the dataset is present before the profiled step.
@@ -57,14 +46,12 @@ OUT_DIR=${SCRIPT_DIR}/single_process
 rm -rf ${OUT_DIR}
 cd ${SCRIPT_DIR}
 
-# ---------------------------------------------------------------------------
 # Profile kernels with rocprofv3.
-# ---------------------------------------------------------------------------
 echo
 echo "==================================================================="
 echo "rocprofv3 --stats --kernel-trace -- python3 train_cifar_100.py"
 echo "==================================================================="
-srun -n 1 --gpus=1 \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     rocprofv3 --stats --kernel-trace --output-format csv \
         --output-directory ${OUT_DIR} --output-file kernels \
         -- \
diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
index 67a701c7..910890ab 100755
--- a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
+++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh
@@ -2,26 +2,21 @@
 #SBATCH --job-name=rpv3-single-traces
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=00:30:00
 #SBATCH --output=rpv3_single_process_traces_%j.out
 #SBATCH --error=rpv3_single_process_traces_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: collect GPU timeline traces for the single-process CIFAR-100
-# workload with rocprofv3 (--sys-trace).
-#
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
+# Collect GPU timeline traces for the single-process CIFAR-100 workload with
+# rocprofv3 (--sys-trace). Sources ../setup_rocm.sh to activate the ROCm venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/rocprofv3/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/rocprofv3/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -31,20 +26,14 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 rocprofv3 --version
 
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
+# Derive a per-job port so concurrent jobs don't collide.
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
 # Make sure the dataset is present before the profiled step.
@@ -57,14 +46,12 @@ OUT_DIR=${SCRIPT_DIR}/single_process
 rm -rf ${OUT_DIR}
 cd ${SCRIPT_DIR}
 
-# ---------------------------------------------------------------------------
 # Collect GPU timeline traces with rocprofv3.
-# ---------------------------------------------------------------------------
 echo
 echo "==================================================================="
 echo "rocprofv3 --sys-trace -- python3 train_cifar_100.py"
 echo "==================================================================="
-srun -n 1 --gpus=1 \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     rocprofv3 --sys-trace --output-format pftrace \
         --output-directory ${OUT_DIR} --output-file traces \
         -- \
diff --git a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
index 0c6d9195..b98a1691 100755
--- a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
+++ b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh
@@ -2,31 +2,24 @@
 #SBATCH --job-name=roofline-single
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
 #SBATCH --gpus=1
-#SBATCH --partition=PPAC_MI300A_SPX
 #SBATCH --time=02:00:00
 #SBATCH --output=roofline_single_process_%j.out
 #SBATCH --error=roofline_single_process_%j.err
 
 # ---------------------------------------------------------------------------
-# SLURM script: collect roofline plots for the single-process CIFAR-100
-# workload with AMD-HPC's rooflineExtractor (https://github.com/AMD-HPC/rooflineExtractor).
-#
-# It sources ../setup_rocm.sh to activate the ROCm PyTorch venv and set the
-# matching ROCm env vars.
-#
-# rooflineExtractor's profile_app.py automates the whole flow: it runs rocprofv3
-# several times to collect hardware counters and a kernel trace, post-processes
-# them, then runs rooflineExtractor.py to produce the per-kernel roofline
-# analysis and an interactive HTML plot.
+# Collect roofline plots for the single-process CIFAR-100 workload with
+# AMD-HPC's rooflineExtractor (https://github.com/AMD-HPC/rooflineExtractor).
+# Its profile_app.py runs rocprofv3 several times to collect counters + a kernel
+# trace, then produces the per-kernel roofline analysis and an HTML plot.
+# Sources ../setup_rocm.sh to activate the ROCm venv.
 # ---------------------------------------------------------------------------
 
 set -e
 
-# Resolve the directory of this script. Under sbatch, $0 points to a copy in
-# the SLURM spool dir, so prefer SLURM_SUBMIT_DIR (the directory from which the
-# job was submitted). Submit this script from
-# `MLExamples/PyTorch_Profiling/roofline-extractor/`.
+# Resolve this script's dir; under sbatch prefer SLURM_SUBMIT_DIR. Submit from
+# MLExamples/PyTorch_Profiling/roofline-extractor/.
 if [[ -n "${SLURM_SUBMIT_DIR}" ]]; then
     SCRIPT_DIR="${SLURM_SUBMIT_DIR}"
 else
@@ -36,18 +29,11 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")"
 echo "SCRIPT_DIR=${SCRIPT_DIR}"
 echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}"
 
-# ---------------------------------------------------------------------------
-# Software environment.
-#
-# ../setup_rocm.sh (i.e. ${PROFILER_TOP_DIR}/setup_rocm.sh) activates the ROCm
-# PyTorch venv and exports the matching ROCm env vars.
-# ---------------------------------------------------------------------------
+# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars.
 source ${PROFILER_TOP_DIR}/setup_rocm.sh
 rocprofv3 --version
 
-# ---------------------------------------------------------------------------
 # Fetch rooflineExtractor and install its Python dependencies (into the venv).
-# ---------------------------------------------------------------------------
 RE_DIR=${SCRIPT_DIR}/rooflineExtractor
 if [ ! -d ${RE_DIR} ]; then
     git clone https://github.com/AMD-HPC/rooflineExtractor.git ${RE_DIR}
@@ -57,8 +43,7 @@ python3 -m pip install -r ${RE_DIR}/requirements.txt
 # Distributed bootstrap variables expected by train_cifar_100.py (single rank).
 export NPROCS=1
 export MASTER_ADDR=${MASTER_ADDR:-$(hostname)}
-# Derive a per-job port so concurrent/leftover jobs don't collide on a shared
-# node (a fixed port like 1234 can fail with EADDRINUSE).
+# Derive a per-job port so concurrent jobs don't collide.
 export MASTER_PORT=${MASTER_PORT:-$((20000 + SLURM_JOB_ID % 20000))}
 
 # Make sure the dataset is present before the profiled runs (profile_app.py runs
@@ -72,18 +57,13 @@ OUT_DIR=${SCRIPT_DIR}/output
 rm -rf ${OUT_DIR}
 cd ${SCRIPT_DIR}
 
-# ---------------------------------------------------------------------------
-# Collect roofline data and generate plots.
-#
-# --arch MI300A selects the matching counter set (gfx942). profile_app.py runs
-# rocprofv3 itself, so it is the application target (not launched under another
-# profiler).
-# ---------------------------------------------------------------------------
+# Collect roofline data and generate plots. --arch MI300A selects the gfx942
+# counter set; profile_app.py runs rocprofv3 itself, so it is the target.
 echo
 echo "==================================================================="
 echo "profile_app.py --arch MI300A -- python3 train_cifar_100.py"
 echo "==================================================================="
-srun -n 1 --gpus=1 \
+srun -n 1 --gpus=1 --cpus-per-task=8 \
     python3 ${RE_DIR}/profile_app.py \
         --arch MI300A \
         -o ${OUT_DIR} \
diff --git a/MLExamples/PyTorch_Profiling/setup_rocm.sh b/MLExamples/PyTorch_Profiling/setup_rocm.sh
index 0f1d3583..cb2998f5 100644
--- a/MLExamples/PyTorch_Profiling/setup_rocm.sh
+++ b/MLExamples/PyTorch_Profiling/setup_rocm.sh
@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # Source this to activate the ROCm venv and set ROCm env vars:
 #   source setup_rocm.sh
-VENV="${HOME}/venvs/rocm-pytorch-pip"
+VENV_BASE="${VENV_BASE:-$HOME}"
+VENV="${VENV_BASE}/venvs/rocm-pytorch-pip"
 source "${VENV}/bin/activate"
 # Derive site-packages from the active venv so this works regardless of the
 # venv's Python minor version (e.g. python3.12 vs python3.13).