Repository navigation
Expand file tree
/
Copy pathDockerfile.gpu-train
More file actions
31 lines (24 loc) · 1.69 KB
/
Copy pathDockerfile.gpu-train
File metadata and controls
31 lines (24 loc) · 1.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# GPU training image for encoder pretraining (ModernBERT). The default Dockerfile stays on
# torch 2.3.1 for the other flows; this one has a newer torch so a flash-attn wheel exists.
# torch must be >= 2.6: transformers 4.57 refuses to torch.load optimizer/scheduler/RNG state on older torch
# (CVE-2025-32434), which breaks resuming any Trainer run from a checkpoint after a spot eviction.
FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
WORKDIR /app
# gcc and libc6-dev: flash-attn's Triton rotary kernel compiles a small C stub at first use.
RUN apt-get update \
&& apt-get install -y --no-install-recommends fonts-dejavu-core git gcc libc6-dev \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Fail the build, not the pods, if the dependency set cannot import.
RUN python -c "from transformers import Trainer; import datasets, sentence_transformers" \
&& python -c "from transformers.utils.import_utils import check_torch_load_is_safe; check_torch_load_is_safe()"
# Prebuilt flash-attn wheel matching torch 2.6 / CUDA 12 / Python 3.11. If it cannot be installed the
# image still builds and the flows fall back to SDPA attention (slower); check the training report's
# attn_implementation field. Override the wheel with --build-arg FLASH_ATTN_WHEEL=<url>.
ARG FLASH_ATTN_WHEEL=https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl
RUN pip install --no-cache-dir "${FLASH_ATTN_WHEEL}" \
|| echo "WARNING: flash-attn wheel install failed; falling back to SDPA attention"
COPY app/ .