Initial Settings

Singularity

# Load Singularity module
module purge
module load Singularity/4.3.4
singularity --version

# Build Singularity sandbox image from Docker
singularity build --sandbox star_env docker://nvidia/cuda:12.9.1-cudnn-devel-ubuntu22.04

Miniconda

# Download Miniconda intsaller on host
cd ~
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh

# Create  workspace directory
mkdir -p ~/workspace

# Enter the Singularity container (writable mode)
singularity shell --writable --nv star_env

# Install Miniconda in the user's home directory
bash ~/miniconda.sh -b -p ~/miniconda3

# Enable Conda for the current session
source ~/miniconda3/etc/profile.d/conda.sh

# Automatically initialize Conda for future sessions
echo 'source ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bashrc

# Create a Conda environment
conda env create -f environment.yml

Ollama w/o root permission

# =====================================================================
# STEP 1: Installation & Directory Setup (Run on Login Node)
# =====================================================================

# 1. Create a dedicated directory for Ollama installation in scratch storage and navigate into it
mkdir -p /scratch/star33/ollama_install
cd /scratch/star33/ollama_install

# 2. Download the official pre-built Ollama Linux binary package (tar.zst format)
curl -L https://ollama.com/download/ollama-linux-amd64.tar.zst -o ollama-linux-amd64.tar.zst

# 3. Extract the downloaded archive containing Ollama binaries and libraries
tar -xvf ollama-linux-amd64.tar.zst

# 4. Create a directory to store downloaded LLM model weights locally
mkdir -p models


# =====================================================================
# STEP 2: Model Pre-downloading (Run on Login Node with Internet Access)
# =====================================================================

# 1. Temporarily register Ollama to PATH and set the custom model storage path for the current session
export PATH=$PATH:/scratch/star33/ollama_install/bin
export OLLAMA_MODELS="/scratch/star33/ollama_install/models"

# 2. Start the Ollama server daemon in the background and discard its output logs
ollama serve > /dev/null 2>&1 &

# Wait for 5 seconds to ensure the Ollama server is fully initialized and running
sleep 5

# 3. Download (pull) the specific LLM model (Qwen 3 32B) to the local model directory
ollama pull qwen3:32b

# 4. Terminate the background Ollama server process once the download is complete
pkill ollama

sbatch ollama.sh

Run with GPU

# Load Singularity module
module purge
module load Singularity/4.3.4

# Launch the container with NVIDIA GPU support
singularity run --nv star_env

# Move to the workspace directory
cd ~/workspace

# Activate the Conda environment
conda activate <env-name>

Slurm Test

# Copy container to scratch
cp -r /home/star33/star_env /scratch/star33/star_env

# Submit the batch job from the login node
sbatch batch.sh

# View the standard output (stdout) log after the job finishes
cat slurm-gpu-test.<job-id>.iREMB-C-06.out

+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 580.95.05              Driver Version: 580.95.05      CUDA Version: 13.0     |
+-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H200                    On  |   00000000:06:00.0 Off |                    0 |
| N/A   38C    P0             76W /  700W |       0MiB / 143771MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

# List all jobs currently in the queue
squeue
# Check the status of a specific job (PD: Pending, R: Running, CG: Completing)
squeue -j <job-id>
# List only your jobs
squeue -u $USER
# Cancel a running or pending job
scancel <job-id>