From 148bb4c8e210d64adcacca89a33471e88ef2f440 Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Thu, 6 Aug 2026 15:46:04 -0700 Subject: [PATCH 1/3] update scripts --- pyproject.toml | 12 +++------ requirements.txt | 3 +-- scripts/install-tuolumne-torchpypi.sh | 5 ---- scripts/install-tuolumne.sh | 3 +++ scripts/scaffold-tuolumne-torchpypi.job | 36 ------------------------- scripts/scaffold-tuolumne.job | 29 ++++++++++++++++++++ 6 files changed, 37 insertions(+), 51 deletions(-) delete mode 100644 scripts/install-tuolumne-torchpypi.sh create mode 100644 scripts/install-tuolumne.sh delete mode 100644 scripts/scaffold-tuolumne-torchpypi.job create mode 100644 scripts/scaffold-tuolumne.job diff --git a/pyproject.toml b/pyproject.toml index ee2c2b0..f10344d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ authors = [ ] license = { file = "LICENSE" } dependencies = [ - "hpc-launcher @ git+https://github.com/llnl/HPC-launcher.git@v1.0.5", + "hpc-launcher==1.0.5", "matplotlib>=3.9.4", "numpy>=1.26.4", "numba>=0.60.0", @@ -77,13 +77,9 @@ cuda = [ "mpi4py==4.1.1", ] rocm = [ - "torch==2.12.0+rocm7.1", - "torchaudio==2.11.0+rocm7.1", - "torchvision==0.27.0+rocm7.1", - "mpi4py==4.1.1+mpich.9.1.0", -] -rocmwci = [ - "torch==2.10.0+rocm710", + "torch==2.12.0+rocm7.2", + "torchaudio==2.11.0+rocm7.2", + "torchvision==0.27.0+rocm7.2", "mpi4py==4.1.1+mpich.9.1.0", ] diff --git a/requirements.txt b/requirements.txt index 4aea607..79ccfdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ ---index-url https://pypi.org/simple -hpc-launcher>=1.0.4 +hpc-launcher==1.0.5 matplotlib>=3.9.4 numpy>=1.26.4 numba>=0.60.0 diff --git a/scripts/install-tuolumne-torchpypi.sh b/scripts/install-tuolumne-torchpypi.sh deleted file mode 100644 index 71de5df..0000000 --- a/scripts/install-tuolumne-torchpypi.sh +++ /dev/null @@ -1,5 +0,0 @@ -ml load python/3.11.5 && python3 -m venv .venvs/scaffoldvenv-tuo-pypi && source .venvs/scaffoldvenv-tuo-pypi/bin/activate && pip install --upgrade pip -ml cce/21.0.1 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi -pip install -e .[rocm] --find-links https://download.pytorch.org/whl/torch/ --find-links https://download.pytorch.org/whl/torchaudio/ --find-links https://download.pytorch.org/whl/torchvision/ --find-links https://download.pytorch.org/whl/triton-rocm/ 2>&1 | tee install.log -# libmpi.so.12 does not exist => ls /opt/cray/pe/lib64/ | grep libmpi -patchelf --replace-needed libmpi.so.12 libmpi_gnu.so.12 .venvs/scaffoldvenv-tuo-pypi/lib/python3.11/site-packages/mpi4py/MPI.cpython-311-x86_64-linux-gnu.so diff --git a/scripts/install-tuolumne.sh b/scripts/install-tuolumne.sh new file mode 100644 index 0000000..bb6c2f1 --- /dev/null +++ b/scripts/install-tuolumne.sh @@ -0,0 +1,3 @@ +ml load python/3.13.2 && python3 -m venv .venvs/scaffoldvenv-tuo && source .venvs/scaffoldvenv-tuo/bin/activate && pip install --upgrade pip +ml cce/21.0.2 cray-mpich/9.1.0 rocm/7.2.1 rccl/fast-env-slows-mpi +pip install -e .[rocm] --find-links https://download.pytorch.org/whl/torch/ --find-links https://download.pytorch.org/whl/torchaudio/ --find-links https://download.pytorch.org/whl/torchvision/ --find-links https://download.pytorch.org/whl/triton-rocm/ 2>&1 | tee install.log diff --git a/scripts/scaffold-tuolumne-torchpypi.job b/scripts/scaffold-tuolumne-torchpypi.job deleted file mode 100644 index fc66d62..0000000 --- a/scripts/scaffold-tuolumne-torchpypi.job +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# flux: --exclusive -# flux: -N 1 -# flux: -g=1 -# flux: -t 60m -# flux: -qpdebug -# flux: -B flask - -ml cce/21.0.1 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi - -. .venvs/scaffoldvenv-tuo-pypi/bin/activate - -export NCCL_NET_PLUGIN=/collab/usr/global/tools/rccl/toss_4_x86_64_ib_cray/rocm-7.1.1/install/lib/librccl-net.so - -# Disable direct convolution benchmarking (should speedup warmup by a significant amount, does the below three options together) -export MIOPEN_DEBUG_CONV_DIRECT=0 -# Disable direct naive convolution benchmarking (naive_conv_ab_nonpacked_fwd_ndhwc_half_double_half.kd) -# export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_FWD=0 -# Disable naive_conv_ab_nonpacked_bwd_ndhwc_half_double_half.kd -# export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_BWD=0 -# Disable naive_conv_ab_nonpacked_wrw_ndhwc_half_double_half.kd -# export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_WRW=0 - -CONFIG_PATH="$(pwd)/ScaFFold/configs/benchmark_default.yml" - -torchrun-hpc -N 1 -n 1 $(which scaffold) generate_fractals \ - -c "$CONFIG_PATH" \ - -# Uncomment if you want torch profiling -#export PROFILE_TORCH=ON - -torchrun-hpc -N 1 -n 4 --gpus-per-proc 1 $(which scaffold) benchmark \ - -c "$CONFIG_PATH" \ - -# torchrun-hpc -N 2 -n 4 --gpus-per-proc 1 $(which scaffold) benchmark -c "$CONFIG_PATH" diff --git a/scripts/scaffold-tuolumne.job b/scripts/scaffold-tuolumne.job new file mode 100644 index 0000000..59422ee --- /dev/null +++ b/scripts/scaffold-tuolumne.job @@ -0,0 +1,29 @@ +#!/bin/bash + +# flux: --exclusive +# flux: -N 1 +# flux: -g=1 +# flux: -t 60m +# flux: -q pdebug +# flux: -B flask + +ml cce/21.0.2 cray-mpich/9.1.0 rocm/7.2.1 rccl/fast-env-slows-mpi + +. .venvs/scaffoldvenv-tuo/bin/activate + +export NCCL_NET_PLUGIN=/collab/usr/global/tools/rccl/toss_4_x86_64_ib_cray/rocm-7.2.0/install/lib/librccl-net.so + +# Disable direct convolution benchmarking (should speedup warmup by a significant amount if using MIOpen) +export MIOPEN_DEBUG_CONV_DIRECT=0 +# Use Triton kernels instead of MIOpen kernels. Set to 0 to use MIOpen. +export SCAFFOLD_GROUPNORM_TRITON=1 +export SCAFFOLD_CONV_TRITON=1 + +CONFIG_PATH="$(pwd)/ScaFFold/configs/benchmark_default.yml" + +torchrun-hpc -N 1 -n 1 $(which scaffold) generate_fractals -c "$CONFIG_PATH" + +# Uncomment if you want torch profiling +#export PROFILE_TORCH=ON + +torchrun-hpc -N 1 -n 4 --gpus-per-proc 1 $(which scaffold) benchmark -c "$CONFIG_PATH" From 6e22ec22311f1a7c77533e44100a55c26262ee1f Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Thu, 6 Aug 2026 15:56:53 -0700 Subject: [PATCH 2/3] Update readme --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7cec997..d8347bf 100644 --- a/README.md +++ b/README.md @@ -24,27 +24,25 @@ The model is trained from a random initialization until convergence, which is de ## **Setup** -1. If running on an LLNL system, try using the scripts in `scripts/install-*.sh` for machine-specific install scripts. +1. If running on an LLNL system, use the machine-specific install scripts in `scripts/install-*.sh`. 1. Clone the repository: `git clone https://github.com/LBANN/ScaFFold.git && cd ScaFFold` -1. Create and activate a python venv for running the benchmark: - `ml load python/3.11.5 && python3 -m venv .venvs/scaffoldvenv && source .venvs/scaffoldvenv/bin/activate && pip install --upgrade pip` +1. Create and activate a python venv for running the benchmark: + - Matrix: `ml load python/3.13.2 && python3 -m venv .venvs/scaffoldvenv-matrix && source .venvs/scaffoldvenv-matrix/bin/activate && pip install --upgrade pip` + - Tuolumne: `ml load python/3.13.2 && python3 -m venv .venvs/scaffoldvenv-tuo && source .venvs/scaffoldvenv-tuo/bin/activate && pip install --upgrade pip` 1. Necessary LLNL settings: - CUDA (matrix): - 1. `ml cuda/12.9.1 gcc/13.3.1 mvapich2/2.3.7` + 1. `ml cuda/13.1.1 gcc/13.3.1 mvapich2/2.3.7` 1. `export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH` - - ROCm (elcap): - 1. `ml cce/21.0.0 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi` - - If using WCI wheel: - 1. `export LD_PRELOAD=/opt/rocm-7.1.1/llvm/lib/libomp.so` # for libomp.so + - ROCm (tuolumne): + 1. `ml cce/21.0.2 cray-mpich/9.1.0 rocm/7.2.1 rccl/fast-env-slows-mpi` 1. Install the benchmark in the python venv: - - CUDA: `pip install --no-binary=mpi4py .[cuda] --prefix=.venvs/scaffoldvenv --extra-index-url https://download.pytorch.org/whl/cu129 2>&1 | tee install.log` - - ROCm (generic): `pip install --no-binary=mpi4py .[rocm] --prefix=.venvs/scaffoldvenv --extra-index-url https://download.pytorch.org/whl/rocm7.1 2>&1 | tee install.log` - - ROCm (LLNL): `pip install .[rocmwci] --prefix=.venvs/scaffoldvenv 2>&1 | tee install.log` + - CUDA: `pip install --no-binary=mpi4py -e .[cuda] --prefix=.venvs/scaffoldvenv-matrix --extra-index-url https://download.pytorch.org/whl/cu132 2>&1 | tee install.log` + - ROCm: `pip install -e .[rocm] --find-links https://download.pytorch.org/whl/torch/ --find-links https://download.pytorch.org/whl/torchaudio/ --find-links https://download.pytorch.org/whl/torchvision/ --find-links https://download.pytorch.org/whl/triton-rocm/ 2>&1 | tee install.log` ## Running the benchmark @@ -222,8 +220,8 @@ make && make install git clone https://github.com/LLNL/Caliper.git cd Caliper mkdir pybuild && cd pybuild -ml rocm/7.1.1 -ml cuda/12.9.1 +ml rocm/7.2.1 +ml cuda/13.1.1 cmake -DWITH_PYTHON_BINDINGS=ON \ -DWITH_ROCPROFILER=ON \ -DWITH_CUPTI=ON \ From 8527630d13f76344a63813242558ef935682e467 Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Thu, 6 Aug 2026 16:11:19 -0700 Subject: [PATCH 3/3] Update requirements.txt --- .github/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/requirements.txt b/.github/requirements.txt index 332ec08..97e48e0 100644 --- a/.github/requirements.txt +++ b/.github/requirements.txt @@ -1 +1 @@ -ruff>=0.13.0 +ruff==0.16.1