Add PyTorch API tests in CICD

Author: MC952-arch

.github/workflows/test.yml

@@ -1,12 +1,12 @@
 name: Run Container and Execute Tests
 
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
+on: none
+  # push:
+  #   branches:
+  #     - main
+  # pull_request:
+  #   branches:
+  #     - main
 
 jobs:
   test-in-container:

.github/workflows/torch-api-test.yml

@@ -0,0 +1,72 @@
+name: Torch API Tests in Container
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  torch-api-test:
+    runs-on: self-hosted
+    container:
+      image: localhost:5000/flagscale:cuda12.4.1-cudnn9.5.0-python3.12-torch2.6.0-time2505241715
+      options:  --gpus all  --privileged --ipc=host --ulimit memlock=-1 --ulimit stack=67108864
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+          ssh-strict: true
+          ssh-user: git
+          persist-credentials: true
+          clean: true
+          sparse-checkout-cone-mode: true
+          fetch-tags: false
+          show-progress: true
+          lfs: false
+          submodules: false
+          set-safe-directory: true
+
+      - name: Set up Install Dependencies
+        run: |
+          export PATH=/root/miniconda3/bin:$PATH
+          eval "$(/root/miniconda3/bin/conda shell.bash hook)" && conda activate flagscale-train && conda env list
+          apt update -y && apt-get install -y git rdma-core ibverbs-utils perftest infiniband-diags libibverbs-dev librdmacm-dev clang-format
+          pip list | grep torch && which python
+          pip uninstall -y setuptools && pip install setuptools==77.0.1 -i https://pypi.tuna.tsinghua.edu.cn/simple
+          pip install pre-commit -i https://pypi.tuna.tsinghua.edu.cn/simple
+          cd /__w/FlagCX/FlagCX
+          git config --global --add safe.directory /__w/FlagCX/FlagCX && git status
+          pre-commit install
+
+      - name: Run Code Format Check with pre-commit
+        run: |
+          cd /__w/FlagCX/FlagCX
+          git fetch --all
+          from_ref=${GITHUB_HEAD_REF}
+          to_ref=${GITHUB_BASE_REF}
+
+          echo "From branch: $from_ref, To branch: $to_ref"
+
+          pre-commit run --from-ref "origin/$from_ref" --to-ref "origin/$to_ref"
+        continue-on-error: false
+
+      - name: Run `make` to build the project
+        run: |
+          cd /__w/FlagCX/FlagCX
+          export MPI_HOME=/usr/local/mpi
+          make USE_NVIDIA=1 -j$(nproc)
+
+      - name: Enter plugin/torch and install FlagCX PyTorch Plugin
+        run: |
+          cd /__w/FlagCX/FlagCX/plugin/torch
+          python setup.py develop --adaptor nvidia
+
+      - name: Run PyTorch API tests
+        run: |
+          cd /__w/FlagCX/FlagCX/test/script/
+          bash torch_api_test.sh

.github/workflows/unit-test.yml

@@ -1,12 +1,12 @@
 name: Unit Tests in Container
 
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
+on: none
+  # push:
+  #   branches:
+  #     - main
+  # pull_request:
+  #   branches:
+  #     - main
 
 jobs:
   unit-test:

test/script/torch_api_test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export FLAGCX_DEBUG=INFO
+export FLAGCX_DEBUG_SUBSYS=INIT
+
+CMD_BASE='torchrun --nproc_per_node 8 --nnodes=1 --node_rank=0 --master_addr="localhost"'
+PY_SCRIPT='../../plugin/torch/example/example.py'
+
+echo "[INFO] Launching PyTorch API tests in homogeneous mode"
+while true; do
+    PORT=$(shuf -i 20000-65535 -n 1)
+    (echo >/dev/tcp/127.0.0.1/$PORT) &>/dev/null || break
+done
+CMD="$CMD_BASE --master_port=$PORT $PY_SCRIPT"
+echo "$CMD"
+eval "$CMD"
+echo "[INFO] Completed PyTorch API tests in homogeneous mode"
+echo "--------------------------------------------------------"
+
+echo "[INFO] Launching PyTorch API tests in heterogeneous mode"
+export FLAGCX_CLUSTER_SPLIT_LIST=2
+while true; do
+    PORT=$(shuf -i 20000-65535 -n 1)
+    (echo >/dev/tcp/127.0.0.1/$PORT) &>/dev/null || break
+done
+CMD="$CMD_BASE --master_port=$PORT $PY_SCRIPT"
+echo "$CMD"
+eval "$CMD"
+echo "[INFO] Completed PyTorch API tests in heterogeneous mode"
+echo "--------------------------------------------------------"