weekly.yaml

# Copyright 2026 FlagOS Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Weekly full-range test workflow
#
#
# Main config: .github/configs/weekly/weekly-test.yaml
# Per-backend config: .github/configs/weekly/<NAME>.yml

name: weekly

permissions:
  contents: read

on:
  workflow_dispatch:
    inputs:
      vendors:
        description: 'Comma-separated vendor names to run on (e.g. Ascend,Nvidia). Empty = all enabled vendors.'
        required: false
        default: ''
      ops:
        description: 'Comma-separated operators to test (e.g. abs,sum,add). Empty = all operator stages.'
        required: false
        default: ''

  schedule:
    # Wednesday and Saturday 21:30 Beijing time = 13:30 UTC
    - cron: '30 13 * * 3,6'

defaults:
  run:
    shell: bash

jobs:
  prepare:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      date: ${{ steps.get-date.outputs.DATE }}
      matrix_container: ${{ steps.build-matrix.outputs.matrix_container }}
      matrix_native: ${{ steps.build-matrix.outputs.matrix_native }}
      api: ${{ steps.build-matrix.outputs.api }}
      ops: ${{ steps.build-matrix.outputs.ops }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - id: get-date
        run: echo "DATE=$(date -u '+%Y-%m-%d')" >> "$GITHUB_OUTPUT"

      - id: build-matrix
        shell: python3 {0}
        env:
          # Optional workflow_dispatch inputs; empty for scheduled runs.
          VENDORS_INPUT: ${{ github.event.inputs.vendors }}
          OPS_INPUT: ${{ github.event.inputs.ops }}
        run: |
          import json
          import os
          import yaml

          with open('.github/configs/weekly/weekly-test.yaml') as f:
              main = yaml.safe_load(f)

          # Parse optional vendor filter (case-insensitive). Empty -> all.
          vendors_filter = {
              v.strip().lower()
              for v in os.environ.get('VENDORS_INPUT', '').split(',')
              if v.strip()
          }

          container_include = []
          native_include = []
          for name, cfg in main['backends'].items():
              if not cfg.get('enabled'):
                  continue
              with open(f'.github/configs/weekly/{name}.yml') as f:
                  backend = yaml.safe_load(f)
              if vendors_filter and backend.get('vendor', '').lower() not in vendors_filter:
                  continue
              backend['test_env'] = json.dumps(backend.get('test_env') or {})
              if backend.get('use_container', True):
                  container_include.append(backend)
              else:
                  native_include.append(backend)

          # ops: empty input -> "all" operator stages; otherwise the given ops.
          ops = os.environ.get('OPS_INPUT', '').strip() or 'all'

          with open(os.environ['GITHUB_OUTPUT'], 'a') as out:
              out.write(f"matrix_container={json.dumps({'include': container_include})}\n")
              out.write(f"matrix_native={json.dumps({'include': native_include})}\n")
              out.write(f"api={main['server']['api']}\n")
              out.write(f"ops={ops}\n")

  # Backends that run inside a container (use_container: true)
  test-container:
    name: test (${{ matrix.name }})
    needs: prepare
    if: ${{ fromJSON(needs.prepare.outputs.matrix_container).include[0] != null }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.prepare.outputs.matrix_container) }}
    concurrency:
      group: weekly-test-${{ matrix.name }}
      cancel-in-progress: true
    runs-on: ${{ matrix.runner_labels }}
    # Whole-job timeout, evaluated per vendor. This is the ONLY timeout for the
    # test run: the "Run weekly tests" step has no step-level timeout, so it is
    # bounded solely by this value (a step-level timeout can never extend a step
    # beyond its enclosing job, and omitting it avoids configuring the limit
    # twice). Ascend (910B) runs the full operator suite in ~15h so it gets 24h.
    # Self-hosted runners have no hard 6h cap, so these values are honoured.
    timeout-minutes: 1440
    container:
      image: ${{ matrix.container_image }}
      options: ${{ matrix.container_options }}
      volumes: ${{ matrix.container_volumes }}
    env:
      DATE: ${{ needs.prepare.outputs.date }}
      OUTPUT_DIR: ${{ matrix.output_base_dir }}/${{ needs.prepare.outputs.date }}/logs_results_${{ needs.prepare.outputs.date }}
    steps:
      - id: checkout-attempt-1
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        continue-on-error: true
      - id: sleep-1
        if: steps.checkout-attempt-1.outcome == 'failure'
        run: sleep 30s

      - id: checkout-attempt-2
        if: steps.checkout-attempt-1.outcome == 'failure'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        continue-on-error: true
      - id: sleep-2
        if: steps.checkout-attempt-2.outcome == 'failure'
        run: sleep 30s

      - id: checkout-attempt-3
        if: steps.checkout-attempt-2.outcome == 'failure'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Fix git safe directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Copy root profile to GitHub Actions home
        run: |
          for f in /root/.bash_profile /root/.profile /root/.bashrc; do
            [ -f "$f" ] && cp -f "$f" "$HOME/" || true
          done
          # .bashrc may have interactive guards ([ -z "$PS1" ] or case $-)
          # that prevent conda init / backend env setup from running in
          # non-interactive CI shells. Extract those lines (conda init,
          # `source .../set_env.sh`, FLAGTREE_BACKEND export, and the
          # `. /root/.local/bin/env` sourcing) and append to .bash_profile
          # instead, which is sourced by login shells without any
          # interactive check.
          #
          # huawei: set_env.sh
          # mthreads: $HOME/.local/bin/env
          # kunlunxin: conda.sh and conda activate
          #
          if [ -f /root/.bashrc ]; then
            # Pin $HOME -> /root: extracted lines use "$HOME/..." (e.g.
            # . "$HOME/.local/bin/env"), but the login shell that sources
            # .bash_profile has $HOME set to the GHA home, not /root.
            grep -E '(conda\.sh|conda activate|set_env\.sh|\.local/bin/env)' /root/.bashrc \
              | sed 's#\$HOME\/\.local#/root\/\.local#g' >> "$HOME/.bash_profile" || true
          fi
          # GitHub Actions job containers force HOME=/github/home. In the 910B
          # image, the Ascend/MKI libraries loaded by torch_npu derive their
          # default work path from HOME; with /github/home they can print
          # "path string is NULL" to stdout during `import torch`. Huawei's ATB
          # set_env.sh compares that command's full stdout to "True" for ABI
          # auto-detection, so the extra text makes it incorrectly select
          # cxx_abi_0.
          #
          # Important: ASCEND_WORK_PATH must be exported before any profile
          # line sources Ascend/ATB set_env.sh. Keep these lines at the top of
          # $HOME/.bash_profile, ahead of the set_env.sh lines extracted above.
          # Also write it to GITHUB_ENV so later GHA steps have it before their
          # own login shell starts reading system/vendor profiles.
          if [ "${{ matrix.vendor }}" = "Ascend" ] && [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then
            echo "[Ascend env] Configuring ASCEND_WORK_PATH before sourcing Ascend/ATB profiles"
            echo "[Ascend env] before rewrite: ASCEND_WORK_PATH=${ASCEND_WORK_PATH:-<unset>}"
            echo "[Ascend env] before rewrite: ATB_HOME_PATH=${ATB_HOME_PATH:-<unset>}"
            echo "[Ascend env] before rewrite: PATH=${PATH:-}"
            echo "[Ascend env] before rewrite: LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}"
            echo "[Ascend env] .bash_profile before rewrite:"
            nl -ba "$HOME/.bash_profile" || true
            ASCEND_WORK_PATH="${ASCEND_WORK_PATH:-/tmp/ascend_work/${GITHUB_RUN_ID:-manual}}"
            mkdir -p "$ASCEND_WORK_PATH"
            [ -n "${GITHUB_ENV:-}" ] && echo "ASCEND_WORK_PATH=$ASCEND_WORK_PATH" >> "$GITHUB_ENV"
            echo "[Ascend env] exported ASCEND_WORK_PATH=$ASCEND_WORK_PATH"
            {
              echo 'export ASCEND_WORK_PATH="${ASCEND_WORK_PATH:-/tmp/ascend_work/${GITHUB_RUN_ID:-manual}}"'
              echo 'mkdir -p "$ASCEND_WORK_PATH"'
              cat "$HOME/.bash_profile"
            } > "$HOME/.bash_profile.tmp"
            mv "$HOME/.bash_profile.tmp" "$HOME/.bash_profile"
            echo "[Ascend env] .bash_profile after rewrite:"
            nl -ba "$HOME/.bash_profile" || true
            echo "[Ascend env] verifying a fresh login shell after profile source"
            bash --login -c '
              set +e
              echo "[Ascend env] after login source: ASCEND_WORK_PATH=${ASCEND_WORK_PATH:-<unset>}"
              echo "[Ascend env] after login source: ATB_HOME_PATH=${ATB_HOME_PATH:-<unset>}"
              echo "[Ascend env] after login source: PATH=${PATH:-}"
              echo "[Ascend env] after login source: LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}"
              python3 -c "import torch; print(torch.compiled_with_cxx11_abi())" >/tmp/ascend_torch_abi_probe.out 2>/tmp/ascend_torch_abi_probe.err
              probe_rc=$?
              echo "[Ascend env] torch ABI probe rc=${probe_rc}"
              python3 -c "from pathlib import Path; print(\"[Ascend env] torch ABI probe stdout repr=\", repr(Path(\"/tmp/ascend_torch_abi_probe.out\").read_text())); print(\"[Ascend env] torch ABI probe stderr repr=\", repr(Path(\"/tmp/ascend_torch_abi_probe.err\").read_text()))"
            ' || true
          fi

      - name: Install FlagGems dependencies
        env: ${{ fromJSON(matrix.test_env) }}
        shell: bash --login -e -o pipefail {0}
        run: pip install .

      - name: Check GPU availability
        shell: bash --login -e -o pipefail {0}
        run: |
          echo "[GPU check] Running GPU availability check script: ${{ matrix.gpu_check_script }}"
          GPU_OUTPUT=$(bash ${{ matrix.gpu_check_script }})
          echo "[GPU check] GPU availability check output:"
          echo "$GPU_OUTPUT"
          AVAILABLE_GPUS=$(echo "$GPU_OUTPUT" | grep "^Available GPUs:" | cut -d: -f2 | tr -d ' ')
          echo "[GPU check] Available GPUs: ${AVAILABLE_GPUS}"
          echo "AVAILABLE_GPUS=${AVAILABLE_GPUS}" >> $GITHUB_ENV

      # Run weekly tests.
      # OPS is provided by the prepare job: "all" (scheduled runs or an empty
      # "ops" input) runs every operator stage via --stages all; otherwise it
      # runs only the selected operators via --ops <list>.
      - name: Run weekly tests
        env: ${{ fromJSON(matrix.test_env) }}
        shell: bash --login -e -o pipefail {0}
        run: |
          echo "[Run weekly tests] Running on backend: ${{ matrix.name }} (vendor: ${{ matrix.vendor }})"
          echo "[Run weekly tests] Test output directory: ${OUTPUT_DIR}"
          echo "[Run weekly tests] Test GPUs: ${AVAILABLE_GPUS}"
          echo "[Run weekly tests] Test date: ${DATE}"
          echo "[Run weekly tests] Test ops: ${{ needs.prepare.outputs.ops }}"
          # remove cache to avoid potential issues with stale files
          rm -rf ~/.triton/cache && rm -rf ~/.flaggems
          # run tests with specified GPUs and output directory
          # Print env for debugging; drop proxy vars (http(s)_proxy/all_proxy,
          # any case) since they may embed user:password that GitHub Actions
          # does not auto-mask for values from the container/runner env.
          # printenv | grep -viE '^(https?_proxy|all_proxy)=' || true
          OPS="${{ needs.prepare.outputs.ops }}"
          if [ "${OPS}" = "all" ]; then
            OP_ARGS="--stages all"
          else
            OP_ARGS="--ops ${OPS}"
          fi
          # replace ${{ matrix.test_gpus }} with ${AVAILABLE_GPUS}
          python3 tools/run_tests.py \
            --gpus "${AVAILABLE_GPUS}" \
            ${OP_ARGS} \
            --dump-output \
            --output "${OUTPUT_DIR}"
          # process the test results to generate summary files
          python3 tools/add_labels.py ${OUTPUT_DIR}
          python3 tools/psum_text  -o  ${OUTPUT_DIR}/result.csv  ${OUTPUT_DIR}
          python3 tools/psum_html  -o  ${OUTPUT_DIR}/result.html  ${OUTPUT_DIR}

      - name: Package results
        run: |
          ARCHIVE="${{ matrix.output_base_dir }}/weekly-${{ matrix.name }}-${DATE}.tar.gz"
          tar -czf "${ARCHIVE}" -C "$(dirname "${OUTPUT_DIR}")" "$(basename "${OUTPUT_DIR}")"
          echo "ARCHIVE=${ARCHIVE}" >> "$GITHUB_ENV"

      # - name: Upload artifact
      #   uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
      #   with:
      #     name: weekly-${{ matrix.name }}-${{ needs.prepare.outputs.date }}
      #     path: ${{ env.ARCHIVE }}

      - name: Publish results to op-monitor
        timeout-minutes: 15
        continue-on-error: true
        run: |
          # Upload the results archive to the op-monitor server.
          curl -X POST "${{ secrets.OPMON_URL }}api/upload/archive" \
            -H "Authorization: Bearer ${{ secrets.OPMON_UPLOAD_KEY }}" \
            -F "file=@${ARCHIVE}" \
            -F "library_name=FlagGems" \
            -F "backend_name=${BACKEND_NAME}" \
            -F "test_date=${DATE}" \
            --compressed -o - -w "\nHTTP %{http_code}\n"

      - name: Send Feishu notification
        if: always()
        timeout-minutes: 5
        continue-on-error: true
        env:
          CI_STATUS: ${{ job.status }}
          PLATFORM: ${{ matrix.name }}
          FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }}
          FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
          FEISHU_CHAT_ID: ${{ secrets.FEISHU_CHAT_ID }}
        run: python3 .github/scripts/notify_feishu.py

  # Backends that run natively on the runner (use_container: false)
  test-native:
    name: test (${{ matrix.name }})
    needs: prepare
    if: ${{ fromJSON(needs.prepare.outputs.matrix_native).include[0] != null }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.prepare.outputs.matrix_native) }}
    concurrency:
      group: weekly-test-${{ matrix.name }}
      cancel-in-progress: true
    runs-on: ${{ matrix.runner_labels }}
    # See test-container: single per-vendor, job-level timeout (Ascend 24h,
    # others 18h); the "Run weekly tests" step has no step-level timeout.
    timeout-minutes: 1080
    env:
      DATE: ${{ needs.prepare.outputs.date }}
      OUTPUT_DIR: ${{ matrix.output_base_dir }}/${{ needs.prepare.outputs.date }}/logs_results_${{ needs.prepare.outputs.date }}
    steps:
      - id: checkout-attempt-1
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        continue-on-error: true
      - id: sleep-1
        if: steps.checkout-attempt-1.outcome == 'failure'
        run: sleep 30s

      - id: checkout-attempt-2
        if: steps.checkout-attempt-1.outcome == 'failure'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        continue-on-error: true
      - id: sleep-2
        if: steps.checkout-attempt-2.outcome == 'failure'
        run: sleep 30s

      - id: checkout-attempt-3
        if: steps.checkout-attempt-2.outcome == 'failure'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Fix git safe directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Install FlagGems dependencies
        run: pip install .

      - name: Check GPU availability
        shell: bash --login -e -o pipefail {0}
        run: |
          echo "[GPU check] Running GPU availability check script: ${{ matrix.gpu_check_script }}"
          GPU_OUTPUT=$(bash ${{ matrix.gpu_check_script }})
          echo "[GPU check] GPU availability check output:"
          echo "$GPU_OUTPUT"
          AVAILABLE_GPUS=$(echo "$GPU_OUTPUT" | grep "^Available GPUs:" | cut -d: -f2 | tr -d ' ')
          echo "[GPU check] Available GPUs: ${AVAILABLE_GPUS}"
          echo "AVAILABLE_GPUS=${AVAILABLE_GPUS}" >> $GITHUB_ENV

      # Run weekly tests.
      # OPS is provided by the prepare job: "all" (scheduled runs or an empty
      # "ops" input) runs every operator stage via --stages all; otherwise it
      # runs only the selected operators via --ops <list>.
      - name: Run weekly tests
        env: ${{ fromJSON(matrix.test_env) }}
        run: |
          echo "[Run weekly tests] Running on backend: ${{ matrix.name }} (vendor: ${{ matrix.vendor }})"
          echo "[Run weekly tests] Test output directory: ${OUTPUT_DIR}"
          echo "[Run weekly tests] Test GPUs: ${AVAILABLE_GPUS}"
          echo "[Run weekly tests] Test date: ${DATE}"
          echo "[Run weekly tests] Test ops: ${{ needs.prepare.outputs.ops }}"
          # remove cache to avoid potential issues with stale files
          rm -rf ~/.triton/cache && rm -rf ~/.flaggems
          # run tests with specified GPUs and output directory
          OPS="${{ needs.prepare.outputs.ops }}"
          if [ "${OPS}" = "all" ]; then
            OP_ARGS="--stages all"
          else
            OP_ARGS="--ops ${OPS}"
          fi
          # replace ${{ matrix.test_gpus }} with ${AVAILABLE_GPUS}
          python3 tools/run_tests.py \
            --gpus "${AVAILABLE_GPUS}" \
            ${OP_ARGS} \
            --dump-output \
            --output "${OUTPUT_DIR}"
          # process the test results to generate summary files
          python3 tools/add_labels.py ${OUTPUT_DIR}
          python3 tools/psum_text  -o  ${OUTPUT_DIR}/result.csv  ${OUTPUT_DIR}
          python3 tools/psum_html  -o  ${OUTPUT_DIR}/result.html  ${OUTPUT_DIR}

      - name: Package results
        run: |
          ARCHIVE="${{ matrix.output_base_dir }}/weekly-${{ matrix.name }}-${DATE}.tar.gz"
          tar -czf "${ARCHIVE}" -C "$(dirname "${OUTPUT_DIR}")" "$(basename "${OUTPUT_DIR}")"
          echo "ARCHIVE=${ARCHIVE}" >> "$GITHUB_ENV"

      # - name: Upload artifact
      #   uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
      #   with:
      #     name: weekly-${{ matrix.name }}-${{ needs.prepare.outputs.date }}
      #     path: ${{ env.ARCHIVE }}

      - name: Publish results to op-monitor
        timeout-minutes: 15
        continue-on-error: true
        run: |
          # Upload the results archive to the op-monitor server.
          curl -X POST "${{ secrets.OPMON_URL }}api/upload/archive" \
            -H "Authorization: Bearer ${{ secrets.OPMON_UPLOAD_KEY }}" \
            -F "file=@${ARCHIVE}" \
            -F "library_name=FlagGems" \
            -F "backend_name=${BACKEND_NAME}" \
            -F "test_date=${DATE}" \
            --compressed -o - -w "\nHTTP %{http_code}\n"

      - name: Send Feishu notification
        if: always()
        timeout-minutes: 5
        continue-on-error: true
        env:
          CI_STATUS: ${{ job.status }}
          PLATFORM: ${{ matrix.name }}
          FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }}
          FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
          FEISHU_CHAT_ID: ${{ secrets.FEISHU_CHAT_ID }}
        shell: bash
        run: python3 .github/scripts/notify_feishu.py