feat(example): add slurm module reference YAMLs

Adds two ci-demo example matrix files for the slurm module:
- job_matrix_slurm.yaml: agent-based example showing the full
  Slurm job lifecycle (allocation, run, stopAllForBuild).
- job_matrix_slurm_k8.yaml: Kubernetes variant running Slurm
  module steps inside a k8s pod with a PVC for job ID persistence.

Signed-off-by: Daniel Pressler <danielpr@nvidia.com>

Author: dpressle

.ci/examples/job_matrix_slurm.yaml

@@ -0,0 +1,59 @@
+---
+job: slurm-example
+
+# Slurm head node and job configuration — set these as Jenkins env vars or override here.
+env:
+  SLURM_HEAD_NODE: scctl
+  SLURM_PARTITION: gpu
+  SLURM_NODES: '1'
+  SLURM_JOB_TIMEOUT: '01:00:00'
+  DOCKER_IMAGE: 'ubuntu:22.04'
+  SLURM_JOB_ID_DIR: '/tmp'
+  SLURM_JOB_ID_FILE: '/tmp/job_id_${BUILD_NUMBER}.txt'
+  # SCCTL_CREDENTIALS_ID must be set to a Jenkins username/password credential id
+  # when SLURM_HEAD_NODE is 'scctl'.
+
+runs_on_agents:
+  - {nodeLabel: master}
+
+# Allocate Slurm nodes before steps run.
+# credentialsId is required when headNode is 'scctl'.
+pipeline_start:
+  shell: action
+  module: slurmCI
+  run: allocation
+  args:
+    partition: '${SLURM_PARTITION}'
+    headNode: '${SLURM_HEAD_NODE}'
+    nodes: '${SLURM_NODES}'
+    jobTimeout: '${SLURM_JOB_TIMEOUT}'
+    jobIdFile: '${SLURM_JOB_ID_FILE}'
+    credentialsId: '${SCCTL_CREDENTIALS_ID}'
+
+steps:
+  # Read the job ID written by allocation() into a Jenkins env var so
+  # subsequent slurm.run() steps can reference it as ${SLURM_JOB_ID}.
+  - name: Read Slurm job ID
+    shell: action
+    module: groovy
+    run: env.SLURM_JOB_ID = readFile(env.SLURM_JOB_ID_FILE).trim()
+
+  - name: Run test script
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobId: '${SLURM_JOB_ID}'
+      headNode: '${SLURM_HEAD_NODE}'
+      testScript: 'echo "Hello from Slurm node $(hostname)"'
+      dockerImage: '${DOCKER_IMAGE}'
+      nodes: '${SLURM_NODES}'
+
+# Cancel all allocations for this build on completion (success or failure).
+pipeline_stop:
+  shell: action
+  module: slurmCI
+  run: stopAllForBuild
+  args:
+    headNode: '${SLURM_HEAD_NODE}'
+    jobIdDir: '${SLURM_JOB_ID_DIR}'

.ci/examples/job_matrix_slurm_k8.yaml

@@ -0,0 +1,79 @@
+---
+job: slurm-k8-example
+
+kubernetes:
+  cloud: swx-k8s
+  namespace: default
+  limits: "{memory: 1Gi, cpu: 500m}"
+  requests: "{memory: 256Mi, cpu: 250m}"
+
+# for step persistance job id file location
+pvc_volumes:
+  - {claimName: nbu-swx-pvc, mountPath: /mnt/pvc, readOnly: false}
+
+empty_volumes:
+  - {mountPath: /root, memory: false}
+
+# Slurm head node and job configuration — set these as Jenkins env vars or override here.
+env:
+  SLURM_HEAD_NODE: headnode
+  SLURM_PARTITION: gpu
+  SLURM_NODES: '1'
+  SLURM_GRES: "gpu:1"
+  SLURM_JOB_TIMEOUT: '01:00:00'
+  DOCKER_IMAGE: 'ubuntu:22.04'
+  SLURM_JOB_ID_DIR: '/mnt/pvc'
+  SLURM_JOB_ID_FILE: '/mnt/pvc/job_id_${BUILD_NUMBER}.txt'
+  # CREDENTIALS_ID: must be set to a Jenkins ssh key credential id with access to the Slurm head node.
+
+# when using scctl use docker image with scctl client installed, otherwise use any image that matches the architecture of the Slurm nodes
+runs_on_dockers:
+  - {name: "ubuntu-x86_64", url: "ubuntu:22.04", arch: "x86_64"}
+
+steps:
+  - name: Allocate Slurm nodes
+    shell: action
+    module: slurmCI
+    run: allocation
+    args:
+      headNode: '${SLURM_HEAD_NODE}'
+      partition: '${SLURM_PARTITION}'
+      nodes: '${SLURM_NODES}'
+      timeout: '${SLURM_JOB_TIMEOUT}'
+      jobIdFile: '${SLURM_JOB_ID_FILE}'
+      credentialsId: '${CREDENTIALS_ID}'
+      extraArgs: [
+        "--gres=${SLURM_GRES}",
+      ]
+
+  # Read the job ID written by allocation() into a Jenkins env var so
+  # subsequent slurm.run() steps can reference it as ${SLURM_JOB_ID}.
+  - name: Read Slurm job ID
+    shell: action
+    module: groovy
+    run: env.SLURM_JOB_ID = readFile(env.SLURM_JOB_ID_FILE).trim()
+
+  - name: Run test script
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobId: '${SLURM_JOB_ID}'
+      headNode: '${SLURM_HEAD_NODE}'
+      testScript: 'env | grep SLURM'
+      dockerImage: '${DOCKER_IMAGE}'
+      credentialsId: '${CREDENTIALS_ID}'
+      nodes: '${SLURM_NODES}'
+      slurmEnv: [
+        "SLURM_TEST_VAR=hello_slurm"
+      ]
+
+# Cancel all allocations for this build on completion (success or failure).
+pipeline_stop:
+  shell: action
+  module: slurmCI
+  run: stop
+  args:
+    credentialsId: '${CREDENTIALS_ID}'
+    headNode: '${SLURM_HEAD_NODE}'
+    jobId: '${SLURM_JOB_ID}'