diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index 4d5bda512..d4a1359cc 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -194,6 +194,19 @@ Read through the [slurm usage guide](slurm-usage.md) for more information. The default Slurm deployment includes a collection of prolog and epilog scripts that should be modified to suit a particular system. For more information, see the [prolog/epilog documentation](slurm-prolog-epilog/README.md). +### Lua plugins + +Slurm's Lua plugins (for example `JobSubmitPlugins=lua` or `CliFilterPlugins=lua`) only load when Slurm was built with Lua support. DeepOps does not request Lua support or install its development package by default; Slurm may still detect a preinstalled Lua library. To enable it, set the following in `config/group_vars/slurm-cluster.yml` and run the Slurm playbook: + +```yaml +slurm_build_lua: true +# Needed when Slurm is already installed at the configured version, otherwise +# the build step is skipped and the existing binaries stay in place. +slurm_force_rebuild: true +``` + +This installs the distribution's Lua development package (`liblua5.3-dev` on Ubuntu, `lua-devel` on RHEL-family systems; override the role variable `slurm_lua_deps` through Ansible extra vars to pick another version) and passes `--with-lua` to Slurm's `configure`. Place your `job_submit.lua` or `cli_filter.lua` in the Slurm configuration directory (`/etc/slurm` by default) and enable the plugin in `slurm.conf` as described in the [Slurm documentation](https://slurm.schedmd.com/job_submit_plugins.html). + ## Node Health Check The default Slurm deployment includes setting up [Node Health Check](https://github.com/mej/nhc). This tool will run periodically on idle nodes to validate that the hardware and software is set up as expected. Nodes which fail this check will be automatically drained in Slurm to prevent jobs running on potentially broken nodes. diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 5859e00f4..b96f63491 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -19,6 +19,12 @@ slurm_cuda_prefix: /usr/local/cuda slurm_configure: './configure --prefix={{ slurm_install_prefix }} --disable-dependency-tracking --disable-debug --disable-x11 --enable-really-no-cray --enable-salloc-kill-cmd --with-hdf5=no --sysconfdir={{ slurm_config_dir }} --enable-pam --with-pam_dir={{ slurm_pam_lib_dir }} --with-shared-libslurm --without-rpath --with-pmix={{ pmix_install_prefix }} --with-hwloc={{ hwloc_install_prefix }}' slurm_configure_nvml: './configure --prefix={{ slurm_install_prefix }} --disable-dependency-tracking --disable-debug --disable-x11 --enable-really-no-cray --enable-salloc-kill-cmd --with-hdf5=no --sysconfdir={{ slurm_config_dir }} --enable-pam --with-pam_dir={{ slurm_pam_lib_dir }} --with-shared-libslurm --without-rpath --with-pmix={{ pmix_install_prefix }} --with-hwloc={{ hwloc_install_prefix }} --with-nvml={{ slurm_cuda_prefix }}' slurm_force_rebuild: no +# Build Slurm with Lua support, which Lua plugins such as job_submit/lua and +# cli_filter/lua need. When enabled, the role installs the distro Lua +# development package (slurm_lua_deps in vars/) and passes --with-lua to +# configure, so the build fails loudly if Lua cannot be found. Turning this on +# for an already-installed Slurm also needs slurm_force_rebuild: yes. +slurm_build_lua: no slurm_contain_ssh: yes slurm_cluster_name: deepops diff --git a/roles/slurm/tasks/build.yml b/roles/slurm/tasks/build.yml index ddbf37965..a6c6e8845 100644 --- a/roles/slurm/tasks/build.yml +++ b/roles/slurm/tasks/build.yml @@ -80,6 +80,22 @@ with_items: "{{ slurm_build_deps }}" when: ansible_os_family == 'RedHat' +- name: install lua build dependencies + apt: + name: "{{ slurm_lua_deps }}" + state: present + when: + - slurm_build_lua | bool + - ansible_distribution == 'Ubuntu' + +- name: install lua build dependencies + dnf: + name: "{{ slurm_lua_deps }}" + state: present + when: + - slurm_build_lua | bool + - ansible_os_family == 'RedHat' + - name: remove slurm packages apt: name: "{{ item }}" @@ -162,14 +178,16 @@ failed_when: false when: slurm_build +# --with-lua is appended here rather than baked into slurm_configure so that +# sites overriding the configure line keep working and the flag stays opt-in. - name: configure - command: "{{ slurm_configure }}" + command: "{{ slurm_configure }}{{ ' --with-lua' if slurm_build_lua | bool else '' }}" args: chdir: "{{ slurm_build_dir }}" when: slurm_build and (not slurm_autodetect_nvml) - name: configure - command: "{{ slurm_configure_nvml }}" + command: "{{ slurm_configure_nvml }}{{ ' --with-lua' if slurm_build_lua | bool else '' }}" args: chdir: "{{ slurm_build_dir }}" when: slurm_build and slurm_autodetect_nvml diff --git a/roles/slurm/vars/redhat.yml b/roles/slurm/vars/redhat.yml index 83f3ca7fd..ce337fdbc 100644 --- a/roles/slurm/vars/redhat.yml +++ b/roles/slurm/vars/redhat.yml @@ -15,6 +15,10 @@ slurm_build_deps: - perl-ExtUtils-MakeMaker - libatomic +# Installed only when slurm_build_lua is enabled. +slurm_lua_deps: + - lua-devel + slurm_pmix_deps: - "@Development Tools" - libev-devel diff --git a/roles/slurm/vars/ubuntu.yml b/roles/slurm/vars/ubuntu.yml index ef92f0e16..decc369a7 100644 --- a/roles/slurm/vars/ubuntu.yml +++ b/roles/slurm/vars/ubuntu.yml @@ -24,6 +24,11 @@ slurm_build_deps: - ruby-dev - wget +# Installed only when slurm_build_lua is enabled. Lua 5.3 is available on +# every Ubuntu release DeepOps targets and is supported by Slurm's configure. +slurm_lua_deps: + - liblua5.3-dev + slurm_pmix_deps: - build-essential - libev-dev diff --git a/tests/slurm-lua/test_lua.py b/tests/slurm-lua/test_lua.py new file mode 100644 index 000000000..61762c1e9 --- /dev/null +++ b/tests/slurm-lua/test_lua.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Inert Ansible command/dependency regressions; run with Ansible's Python.""" +import os +from pathlib import Path +import shlex +import unittest + +from ansible.parsing.dataloader import DataLoader +from ansible.playbook.conditional import Conditional +from ansible.template import Templar +import yaml + +ROOT = Path(os.environ.get("DEEPOPS_TEST_ROOT", Path(__file__).resolve().parents[2])) +ROLE = ROOT / "roles/slurm" +TASKS = yaml.safe_load((ROLE / "tasks/build.yml").read_text()) +DEFAULTS = yaml.safe_load((ROLE / "defaults/main.yml").read_text()) + + +class LuaBuild(unittest.TestCase): + def selected(self, task, variables): + loader = DataLoader() + templar = Templar(loader=loader, variables=variables) + condition = Conditional(loader=loader) + when = task.get("when", []) + condition.when = [when] if isinstance(when, str) else when + return condition.evaluate_conditional(templar, variables), templar + + def test_configure_flag_for_both_nvml_paths(self): + for enabled, expected in ((False, False), (True, True), ("false", False), ("true", True)): + for nvml in (False, True): + with self.subTest(enabled=enabled, nvml=nvml): + variables = {**DEFAULTS, "slurm_build_lua": enabled, + "slurm_build": True, "slurm_autodetect_nvml": nvml, + "slurm_configure": "./configure --site-option", + "slurm_configure_nvml": "./configure --site-option --with-nvml=/cuda"} + commands = [] + for task in TASKS: + if task.get("name") != "configure": + continue + active, templar = self.selected(task, variables) + if active: + commands.append(shlex.split(templar.template(task["command"]))) + self.assertEqual(len(commands), 1) + self.assertEqual("--with-lua" in commands[0], expected) + self.assertIn("--site-option", commands[0]) + self.assertEqual("--with-nvml=/cuda" in commands[0], nvml) + + def test_package_is_opt_in_and_matches_distribution(self): + for distro, family, varfile, module, package in ( + ("Ubuntu", "Debian", "ubuntu.yml", "apt", "liblua5.3-dev"), + ("Rocky", "RedHat", "redhat.yml", "dnf", "lua-devel"), + ): + for enabled in (False, True): + with self.subTest(distro=distro, enabled=enabled): + variables = {**DEFAULTS, **yaml.safe_load((ROLE / "vars" / varfile).read_text()), + "slurm_build_lua": enabled, "ansible_distribution": distro, + "ansible_os_family": family} + selected = [] + for task in TASKS: + if task.get("name") != "install lua build dependencies": + continue + active, templar = self.selected(task, variables) + if active: + selected.append(templar.template(task[module])["name"]) + self.assertEqual(selected, [[package]] if enabled else []) + + +if __name__ == "__main__": + unittest.main()