Concurrency on custom nvidia runners

Author: msaroufim

.github/workflows/nvidia_workflow.yml

@@ -15,11 +15,27 @@ on:
         required: false
         type: string
 
+concurrency:
+  group: nvidia-workflow-global
+  cancel-in-progress: false
+
 run-name: 'NVIDIA Job - ${{ github.event.inputs.run_id }}'
 
 jobs:
+  select-runner:
+    runs-on: ubuntu-latest
+    outputs:
+      runner: ${{ steps.pick.outputs.runner }}
+    steps:
+      - id: pick
+        run: |
+          runners=("b200-02-gpu0" "b200-02-gpu1" "b200-02-gpu2" "b200-02-gpu3" "b200-02-gpu4" "b200-02-gpu5" "b200-02-gpu6" "b200-02-gpu7")
+          index=$(( ${{ github.run_number }} % 8 ))
+          echo "runner=${runners[$index]}" >> $GITHUB_OUTPUT
+
   run:
-    runs-on: [nvidia-docker-b200-8-x86-64]
+    needs: select-runner
+    runs-on: [self-hosted, Linux, X64, nvidia-docker-b200-8-x86-64, "${{ needs.select-runner.outputs.runner }}"]
     timeout-minutes: 10
     steps:
     - uses: actions/checkout@v3