From 1ffe3029ba6616d8b823e3ce636369e8270730ae Mon Sep 17 00:00:00 2001
From: Doug Holt <dholt@nvidia.com>
Date: Tue, 22 Sep 2026 13:11:57 -0600
Subject: [PATCH] slurm: put slurm_install_prefix clients on the slurmd PATH

slurmd runs with systemd's minimal PATH, so with a custom
slurm_install_prefix the Slurm client binaries are not resolvable from
slurmd's children. Enroot's slurm-pmi hook (installed by the pyxis role)
calls scontrol and fails with "Command not found: scontrol", and every
Pyxis container then refuses to start:

    pyxis: container start failed with error code: 1
    [ERROR] Command not found: scontrol
    [ERROR] /etc/enroot/hooks.d/50-slurm-pmi.sh exited with return code 1

The default /usr/local prefix only works because /usr/local/bin is on the
default PATH. Install a slurmd drop-in that prepends the configured prefix.
The existing unconditional slurmd restart with daemon-reload in compute.yml
applies it on every convergence.

Reproduced on a fresh Ubuntu 24.04 single-node deployment with
slurm_install_prefix=/opt/deepops/slurm and Slurm 26.05.3; the same
correction was part of the closed consolidation PR #1396.

Signed-off-by: Doug Holt <dholt@nvidia.com>
---
 roles/slurm/tasks/service-files.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/roles/slurm/tasks/service-files.yml b/roles/slurm/tasks/service-files.yml
index 7ecbff88d..74606e9e0 100644
--- a/roles/slurm/tasks/service-files.yml
+++ b/roles/slurm/tasks/service-files.yml
@@ -21,3 +21,29 @@
     - slurmd.service
   when: is_compute
   failed_when: false
+
+# slurmd inherits systemd's minimal PATH, which does not include a custom
+# slurm_install_prefix. Enroot's slurm-pmi hook (installed by the pyxis role)
+# and any other slurmd child that calls scontrol/squeue then fail with
+# "Command not found: scontrol" and every Pyxis container refuses to start.
+# The unconditional "start slurmd" task in compute.yml restarts the unit with
+# daemon-reload after this file, so the drop-in is active on every convergence.
+- name: create slurmd systemd override directory
+  file:
+    path: /etc/systemd/system/slurmd.service.d
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: is_compute
+
+- name: put slurm_install_prefix clients on the slurmd PATH
+  copy:
+    dest: /etc/systemd/system/slurmd.service.d/10-slurm-path.conf
+    owner: root
+    group: root
+    mode: "0644"
+    content: |
+      [Service]
+      Environment="PATH={{ slurm_install_prefix }}/bin:{{ slurm_install_prefix }}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  when: is_compute