Merge remote-tracking branch 'origin/main' into chcui/rhea/qwen35-text-pretrain-recipe

Signed-off-by: Chen Cui <chcui@nvidia.com>

# Conflicts:
#	src/megatron/bridge/recipes/nemotronh/__init__.py

Author: cuichenx

.github/CODEOWNERS

@@ -7,5 +7,6 @@ ruff.toml @nvidia-nemo/automation
 
 scripts/performance/argument_parser.py @nvidia-nemo/automation @malay-nagda @erhoo82
 scripts/performance/setup_experiment.py @nvidia-nemo/automation @malay-nagda @erhoo82
+scripts/performance/bootstrap.py @nvidia-nemo/automation @malay-nagda @erhoo82
 scripts/performance/run_script.py @nvidia-nemo/automation @malay-nagda @erhoo82
 scripts/performance/run_recipe.py @nvidia-nemo/automation @malay-nagda @erhoo82

.github/workflows/cicd-main.yml

@@ -305,10 +305,11 @@ jobs:
           REGISTRY_AWS: ${{ needs.pre-flight.outputs.registry }}
           REGISTRY_GCP: ${{ env.container-registry-gb200 }}
           IS_MEMBER: ${{ needs.pre-flight.outputs.is_member }}
+          DISABLE_GB200_TESTS: ${{ vars.DISABLE_GB200_TESTS }}
         run: |
           AWS_ENTRY=$(jq -nc --arg registry "$REGISTRY_AWS" --arg runner "${RUNNER_PREFIX}" \
             '{"cloud": "aws", "registry": $registry, "runner": $runner}')
-          if [[ "$IS_MEMBER" == "true" ]]; then
+          if [[ "$IS_MEMBER" == "true" && "$DISABLE_GB200_TESTS" != "true" ]]; then
             GCP_ENTRY=$(jq -nc --arg registry "$REGISTRY_GCP" --arg runner "nemo-ci-gcp-gpu-x2" \
               '{"cloud": "gcp", "registry": $registry, "runner": $runner}')
             MATRIX=$(jq -nc --argjson aws "$AWS_ENTRY" --argjson gcp "$GCP_ENTRY" \
@@ -410,10 +411,24 @@ jobs:
           # digest on every cold build. Listing more than one cache-from donor lets
           # BuildKit chain from a conflicting copy of such a layer and miss every
           # layer downstream — a ~40 min cold rebuild even when a warm cache exists.
-          # So read EXACTLY ONE donor: PR builds integrate into main, so they seed
-          # from main; main/branch builds seed from their own chain.
+          # So we still read EXACTLY ONE donor; only WHICH donor changes below.
+          #
+          # A PR's first build has no cache of its own and seeds from main. Every
+          # LATER push on the same PR seeds from the PR's OWN buildcache instead of
+          # main: a PR that touches an early layer (Dockerfile.ci, pyproject.toml,
+          # uv.lock, the MCore stubs) never has those layers in main-buildcache, so
+          # seeding from main cold-rebuilt the ~40 min dependency layer on EVERY
+          # push. Seeding from the PR's own cache reuses the layers that same PR
+          # already built and pushed to ${KEY}-buildcache. This stays single-donor,
+          # so the non-reproducible-DeepEP cascade above cannot happen. The probe
+          # falls back to main on any error, so this never fails the build.
           if [[ -n "$PR_NUMBER" && "${{ github.ref }}" != "refs/heads/main" ]]; then
-            SEED="main"
+            PR_CACHE="${{ matrix.registry }}/megatron-bridge:${KEY}-buildcache"
+            if docker buildx imagetools inspect "$PR_CACHE" > /dev/null 2>&1; then
+              SEED="$KEY"
+            else
+              SEED="main"
+            fi
           else
             SEED="$KEY"
           fi
@@ -782,6 +797,7 @@ jobs:
       )
       && !cancelled()
       && needs.pre-flight.outputs.is_member == 'true'
+      && vars.DISABLE_GB200_TESTS != 'true'
     steps:
       - uses: actions/checkout@v6
         with:
@@ -843,6 +859,7 @@ jobs:
       )
       && !cancelled()
       && needs.pre-flight.outputs.is_member == 'true'
+      && vars.DISABLE_GB200_TESTS != 'true'
       && (github.event.inputs.test_suite == '' || github.event.inputs.test_suite == 'all' || github.event.inputs.test_suite == 'functional-only' || contains('L0 L1 L2', github.event.inputs.test_suite))
     name: gb200_${{ matrix.script }}
     environment: ${{ contains(needs.pre-flight.outputs.registry, 'azure') && 'nemo-ci' || '' }}
@@ -890,6 +907,7 @@ jobs:
       )
       && !cancelled()
       && needs.pre-flight.outputs.is_member == 'true'
+      && vars.DISABLE_GB200_TESTS != 'true'
       && (github.ref == 'refs/heads/main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || needs.configure.outputs.needs_more_tests == 'true' || needs.configure.outputs.full_test_suite == 'true')
       && (github.event.inputs.test_suite == '' || github.event.inputs.test_suite == 'all' || github.event.inputs.test_suite == 'functional-only' || contains('L1 L2', github.event.inputs.test_suite))
     name: gb200_${{ matrix.script }}
@@ -938,6 +956,7 @@ jobs:
       )
       && !cancelled()
       && needs.pre-flight.outputs.is_member == 'true'
+      && vars.DISABLE_GB200_TESTS != 'true'
       && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.configure.outputs.full_test_suite == 'true')
       && (github.event.inputs.test_suite == '' || github.event.inputs.test_suite == 'all' || github.event.inputs.test_suite == 'functional-only' || contains('L2', github.event.inputs.test_suite))
     name: gb200_${{ matrix.script }}
@@ -970,7 +989,7 @@ jobs:
           runner: ${{ matrix.runner }}
 
   cicd-functional-tests-gb200-flaky:
-    if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_suite == 'all' && needs.pre-flight.outputs.is_member == 'true'
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_suite == 'all' && needs.pre-flight.outputs.is_member == 'true' && vars.DISABLE_GB200_TESTS != 'true'
     strategy:
       fail-fast: false
       max-parallel: 16
@@ -1038,6 +1057,7 @@ jobs:
           EXPECT_L1: ${{ needs.configure.outputs.expect_l1 }}
           EXPECT_L2: ${{ needs.configure.outputs.expect_l2 }}
           IS_MEMBER: ${{ needs.pre-flight.outputs.is_member }}
+          DISABLE_GB200_TESTS: ${{ vars.DISABLE_GB200_TESTS }}
         run: |
           # NOTE: `gh run view` must NOT be wrapped in `|| echo 0`. Doing so swallows
           # any failure (e.g. "failed to determine base repo" when git context is
@@ -1064,7 +1084,7 @@ jobs:
           check_tier_skips "L0" "$EXPECT_L0"
           check_tier_skips "L1" "$EXPECT_L1"
           check_tier_skips "L2" "$EXPECT_L2"
-          if [[ "$IS_MEMBER" == "true" ]]; then
+          if [[ "$IS_MEMBER" == "true" && "$DISABLE_GB200_TESTS" != "true" ]]; then
             check_tier_skips "gb200_L0" "$EXPECT_L0"
             check_tier_skips "gb200_L1" "$EXPECT_L1"
             check_tier_skips "gb200_L2" "$EXPECT_L2"

.github/workflows/link-check.yml

@@ -0,0 +1,62 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Link check
+
+# Checks external (http/https) links in the docs so broken cross-repo / published
+# links (NVBug 6421302 class) are caught before merge and on a weekly cadence.
+#
+# Scope note: this job checks http/https URLs only (`--scheme`). Internal
+# cross-page links in the Fern docs use a `.md` extension while the source files
+# are `.mdx`, so filesystem-level checking of relative links would report ~600
+# false positives; that convention is out of scope here. GitHub links are NOT
+# excluded (GITHUB_TOKEN is supplied to avoid rate limits).
+
+on:
+  pull_request:
+  schedule:
+    # Weekly, Mondays 06:00 UTC
+    - cron: "0 6 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  link-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check links
+        uses: lycheeverse/lychee-action@v2
+        with:
+          fail: true
+          args: >-
+            --no-progress
+            --max-retries 3
+            --retry-wait-time 10
+            --scheme https
+            --scheme http
+            --accept '200..=299,401,403,429'
+            --root-dir docs/fern
+            --exclude 'github\.com/deepseek-ai/DeepSeek-V4'
+            --exclude 'github\.com/NVIDIA-NeMo/Megatron-Bridge/stargazers'
+            --exclude 'sched\.com'
+            --exclude 'llama\.com'
+            "docs/fern/**/*.mdx"
+            "docs/**/*.md"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.main.commit

@@ -1 +1 @@
-adfb8ec7926215bfd0d5120aaf6b080de224d328
+58bf14e9e68915a5e0c7d70451620e6e713c07d5

3rdparty/Megatron-LM

@@ -21,15 +21,15 @@ Use this checklist-style flow: scaffold → provider mapping → parameter mappi
 
  Most GPT-style models (such as the Qwen and Llama families) can reuse the Megatron-Core GPT model by mapping their configuration. If the model requires custom building blocks (e.g., an attention variant, RoPE variant, or VLM modules), add a lightweight specialization similar to how 🤗 HuggingFace implements `modeling_xxx.py`.
 
-- **Standard GPT-style models**: Implement a `Provider` and a `Bridge`. For example, see the [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_provider.py) and [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py).
+- **Standard GPT-style models**: Implement a `Provider` and a `Bridge`. For example, see the [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py) and [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py).
 - **Models with custom components**: If your model has custom operations or blocks (e.g., a unique attention mechanism), add a minimal modeling module in the same directory and reference it from the `Provider` (example forthcoming).
 
 
 ## 2) Scaffold the model folder
 
 Create a folder under `src/megatron/bridge/models/<your_model>/` and add:
 
-- `<your_model>_provider.py`: builds a `TransformerConfig`-compatible provider (or a subclass of an existing provider) and exposes `.provide_distributed_model()`. For example: [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_provider.py), [Qwen provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen_provider.py), or [Qwen2 provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_provider.py).
+- `<your_model>_provider.py`: builds a `TransformerConfig`-compatible provider (or a subclass of an existing provider) and exposes `.provide_distributed_model()`. For example: [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py), [Qwen provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen3_bridge.py), or [Qwen2 provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_bridge.py).
 - `<your_model>_bridge.py`: architecture-specific bridge that maps HF config → provider and defines parameter mappings. For example: [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py), [Qwen3 bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen3_bridge.py), or [Qwen2 bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_bridge.py).
 - Optional: `README.md` with any model quirks. For example: [Llama README](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/README.md).
 

docs/adding-new-models.md

@@ -9,7 +9,7 @@ Megatron Bridge provides a clean, parallelism-aware path to use 🤗 Hugging Fac
 The examples mirror how NeMo-RL integrates Megatron Bridge:
 
 - [nemo_rl/models/megatron/community_import.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/megatron/community_import.py)
-- [nemo_rl/models/policy/megatron_policy_worker.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/policy/megatron_policy_worker.py)
+- [nemo_rl/models/policy/megatron_policy_worker.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/policy/workers/megatron_policy_worker.py)
 
 - Local example script in this repo: [examples/rl/rlhf_with_bridge.py](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/examples/rl/rlhf_with_bridge.py)
 
@@ -503,4 +503,4 @@ class MegatronBridgeAdapter:
 
 - [Bridge with 🤗 Hugging Face](./bridge-guide.md) for HF↔Megatron conversion overview
 - [nemo_rl/models/megatron/community_import.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/megatron/community_import.py) for import/export helpers
-- [nemo_rl/models/policy/megatron_policy_worker.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/policy/megatron_policy_worker.py) for end-to-end RL integration (training, logprobs, generation, refit)
+- [nemo_rl/models/policy/megatron_policy_worker.py](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/models/policy/workers/megatron_policy_worker.py) for end-to-end RL integration (training, logprobs, generation, refit)

docs/bridge-rl-integration.md

@@ -1,4 +1 @@
 {"uri": "https://www.llama.com/models/llama-3/"}
-{"uri": "https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/context_parallel.html"}
-{"uri": "https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/dist_optimizer.html"}
-{"uri": "https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/tensor_parallel.html"}

docs/broken_links_false_positives.json

@@ -88,19 +88,13 @@ Example:
 
 ```{code-block} json
 :caption: Example: versions1.json 
-:emphasize-lines: 9,10
 
 [
     {
         "preferred": true,
         "version": "latest",
         "url": "https://docs.nvidia.com/nemo/megatron-bridge/latest/"
     },
-    {
-
-        "version": "#.#.#",
-        "url": "https://docs.nvidia.com/nemo/megatron-bridge/#.#.#/"
-    },
     {
 
         "version": "0.1.0",

docs/documentation.md

@@ -19,14 +19,14 @@ Use this checklist-style flow: scaffold → provider mapping → parameter mappi
 
  Most GPT-style models (such as the Qwen and Llama families) can reuse the Megatron-Core GPT model by mapping their configuration. If the model requires custom building blocks (e.g., an attention variant, RoPE variant, or VLM modules), add a lightweight specialization similar to how 🤗 HuggingFace implements `modeling_xxx.py`.
 
-- **Standard GPT-style models**: Implement a `Provider` and a `Bridge`. For example, see the [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_provider.py) and [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py).
+- **Standard GPT-style models**: Implement a `Provider` and a `Bridge`. For example, see the [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py) and [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py).
 - **Models with custom components**: If your model has custom operations or blocks (e.g., a unique attention mechanism), add a minimal modeling module in the same directory and reference it from the `Provider` (example forthcoming).
 
 ## 2) Scaffold the model folder
 
 Create a folder under `src/megatron/bridge/models/<your_model>/` and add:
 
-- `<your_model>_provider.py`: builds a `TransformerConfig`-compatible provider (or a subclass of an existing provider) and exposes `.provide_distributed_model()`. For example: [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_provider.py), [Qwen provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen_provider.py), or [Qwen2 provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_provider.py).
+- `<your_model>_provider.py`: builds a `TransformerConfig`-compatible provider (or a subclass of an existing provider) and exposes `.provide_distributed_model()`. For example: [Llama provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py), [Qwen provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen3_bridge.py), or [Qwen2 provider](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_bridge.py).
 - `<your_model>_bridge.py`: architecture-specific bridge that maps HF config → provider and defines parameter mappings. For example: [Llama bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/llama_bridge.py), [Qwen3 bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen3_bridge.py), or [Qwen2 bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/qwen/qwen2_bridge.py).
 - Optional: `README.md` with any model quirks. For example: [Llama README](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/main/src/megatron/bridge/models/llama/README.md).