Pin FinePDFs revision for long-document seeds

Author: intern_nem_dev_1

src/nemotron/recipes/data/sdg/long-document/01-seed-dataset-preparation.py

@@ -85,6 +85,7 @@
 logging.getLogger("fsspec").setLevel(logging.WARNING)
 
 FINEPDFS_REPO = "HuggingFaceFW/finepdfs"
+FINEPDFS_REVISION = "220bac3acbf07789502c621d2d33952f51ac7f86"
 
 
 class SeedConfig(BaseModel):
@@ -105,6 +106,11 @@ class SeedConfig(BaseModel):
         default="eng_Latn",
         description="FinePDFs language subset (e.g. eng_Latn, fra_Latn).",
     )
+    finepdfs_revision: str = Field(
+        default=FINEPDFS_REVISION,
+        pattern=r"[0-9a-f]{40}",
+        description="Pinned FinePDFs dataset revision SHA for reproducible seed inputs.",
+    )
     timeout: int = Field(
         default=20,
         ge=0,
@@ -195,15 +201,17 @@ def run_seed(cfg: SeedConfig) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
 
     log.info(
-        "Streaming %d documents from %s (subset=%s)",
+        "Streaming %d documents from %s (subset=%s, revision=%s)",
         cfg.num_docs,
         FINEPDFS_REPO,
         cfg.subset,
+        cfg.finepdfs_revision,
     )
 
     ds = load_dataset(
         FINEPDFS_REPO,
         name=cfg.subset,
+        revision=cfg.finepdfs_revision,
         split="train",
         streaming=True,
     )

src/nemotron/recipes/data/sdg/long-document/config/01-seed.yaml

@@ -7,6 +7,7 @@
 output_dir: ./seed_data
 num_docs: 10
 subset: eng_Latn
+finepdfs_revision: 220bac3acbf07789502c621d2d33952f51ac7f86
 timeout: 20
 dpi: 144
 max_pages: 50

tests/recipes/data/test_sdg_long_document_finepdfs_revision_pin.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+SEED_SCRIPT = REPO_ROOT / "src/nemotron/recipes/data/sdg/long-document/01-seed-dataset-preparation.py"
+SEED_CONFIG = REPO_ROOT / "src/nemotron/recipes/data/sdg/long-document/config/01-seed.yaml"
+EXPECTED_REPO = "HuggingFaceFW/finepdfs"
+EXPECTED_REVISION = "220bac3acbf07789502c621d2d33952f51ac7f86"
+
+
+def _module_tree() -> ast.Module:
+    return ast.parse(SEED_SCRIPT.read_text(encoding="utf-8"))
+
+
+def _module_string_constant(tree: ast.Module, name: str) -> str:
+    for node in tree.body:
+        if not isinstance(node, ast.Assign):
+            continue
+        if not any(isinstance(target, ast.Name) and target.id == name for target in node.targets):
+            continue
+        assert isinstance(node.value, ast.Constant)
+        assert isinstance(node.value.value, str)
+        return node.value.value
+    raise AssertionError(f"missing module constant {name}")
+
+
+def _field_default_name(tree: ast.Module, field_name: str) -> str:
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.AnnAssign):
+            continue
+        if not isinstance(node.target, ast.Name) or node.target.id != field_name:
+            continue
+        assert isinstance(node.value, ast.Call)
+        assert isinstance(node.value.func, ast.Name)
+        assert node.value.func.id == "Field"
+        for keyword in node.value.keywords:
+            if keyword.arg != "default":
+                continue
+            assert isinstance(keyword.value, ast.Name)
+            return keyword.value.id
+    raise AssertionError(f"missing SeedConfig field {field_name}")
+
+
+def _load_dataset_call(tree: ast.Module) -> ast.Call:
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        if isinstance(node.func, ast.Name) and node.func.id == "load_dataset":
+            return node
+    raise AssertionError("missing load_dataset call")
+
+
+def _keyword_value(call: ast.Call, name: str) -> ast.expr:
+    for keyword in call.keywords:
+        if keyword.arg == name:
+            return keyword.value
+    raise AssertionError(f"missing load_dataset keyword {name}")
+
+
+def test_finepdfs_seed_source_defaults_to_pinned_revision() -> None:
+    tree = _module_tree()
+
+    assert _module_string_constant(tree, "FINEPDFS_REPO") == EXPECTED_REPO
+    revision = _module_string_constant(tree, "FINEPDFS_REVISION")
+    assert revision == EXPECTED_REVISION
+    assert re.fullmatch(r"[0-9a-f]{40}", revision)
+    assert _field_default_name(tree, "finepdfs_revision") == "FINEPDFS_REVISION"
+
+
+def test_finepdfs_load_dataset_threads_revision_from_config() -> None:
+    call = _load_dataset_call(_module_tree())
+
+    assert isinstance(call.args[0], ast.Name)
+    assert call.args[0].id == "FINEPDFS_REPO"
+
+    name_value = _keyword_value(call, "name")
+    assert isinstance(name_value, ast.Attribute)
+    assert isinstance(name_value.value, ast.Name)
+    assert name_value.value.id == "cfg"
+    assert name_value.attr == "subset"
+
+    revision_value = _keyword_value(call, "revision")
+    assert isinstance(revision_value, ast.Attribute)
+    assert isinstance(revision_value.value, ast.Name)
+    assert revision_value.value.id == "cfg"
+    assert revision_value.attr == "finepdfs_revision"
+
+
+def test_finepdfs_default_config_exposes_same_revision_pin() -> None:
+    config = yaml.safe_load(SEED_CONFIG.read_text(encoding="utf-8"))
+
+    assert config["finepdfs_revision"] == EXPECTED_REVISION
+    assert re.fullmatch(r"[0-9a-f]{40}", config["finepdfs_revision"])

workspace/interns/intern_nem_dev_1/status.md

@@ -1,12 +1,12 @@
 # intern_nem_dev_1 - 状态
 
-<!-- METADATA:STATUS=Working,TASK=task162_long_document_public_docs_path_portability_s1,ROLE=dev,SESSION=1 -->
+<!-- METADATA:STATUS=Working,TASK=task164_long_document_finepdfs_revision_pin_s1,ROLE=dev,SESSION=1 -->
 
 | 字段 | 值 |
 |------|-----|
 | Name | intern_nem_dev_1 |
 | Status | Working |
-| Current Task | task162_long_document_public_docs_path_portability_s1 |
-| PR | https://github.com/songCNMS/Nemotron/pull/269 |
+| Current Task | task164_long_document_finepdfs_revision_pin_s1 |
+| PR | pending |
 | Session | 1 |
-| Recent Progress | Opened PR #269 for task162; public long-document SDG docs now use NEMO_RUN_DIR-relative seed examples and focused static docs checks passed |
+| Recent Progress | Started task164 from origin/main a9b324bf28cd6cb0470b58eec47fd17336fdec0f; adding offline/static FinePDFs revision pin coverage |

workspace/tasks/task164_long_document_finepdfs_revision_pin_s1/README.md

@@ -0,0 +1,20 @@
+# task164_long_document_finepdfs_revision_pin_s1
+
+<!-- METADATA:STATUS=InProgress,ASSIGNEE=intern_nem_dev_1,SESSION=1 -->
+
+## Scope
+
+- Pin long-document SDG FinePDFs seed loading to revision `220bac3acbf07789502c621d2d33952f51ac7f86`.
+- Keep `HuggingFaceFW/finepdfs`, subset behavior, and downstream stage semantics unchanged.
+- Add focused static/AST tests that do not call `load_dataset` or download PDFs.
+
+## Boundaries
+
+- No live `load_dataset`, PDF downloads, seed generation, `--serve`, endpoints, train/eval, W&B, cluster jobs, deploy, artifact operations, main push, or self-merge.
+- Scope is limited to the seed script/config, focused static tests, and task/status docs.
+
+## Status
+
+- Base: `a9b324bf28cd6cb0470b58eec47fd17336fdec0f`
+- Branch: `intern_nem_dev_1/task164_long_document_finepdfs_revision_pin_s1`
+- PR: pending

workspace/tasks/task164_long_document_finepdfs_revision_pin_s1/history_log.md

@@ -0,0 +1,10 @@
+# History Log
+
+<!-- METADATA:SESSION=1 -->
+
+## Session 1 - 2026-05-29
+
+- Started task164 from `origin/main` at `a9b324bf28cd6cb0470b58eec47fd17336fdec0f`.
+- Added a default FinePDFs revision pin and config-threaded `load_dataset(..., revision=...)` path.
+- Added focused AST/static tests for the repo id, exact revision SHA, config field/default, `load_dataset` revision keyword, and YAML default.
+- Ran focused pytest, py_compile, Ruff, structured AST probe, added-line live-surface scan, and `git diff --check` without calling `load_dataset` or running seed generation.

workspace/tasks/task164_long_document_finepdfs_revision_pin_s1/task_knowledge.md

@@ -0,0 +1,6 @@
+# Task Knowledge
+
+<!-- METADATA:SESSION=1 -->
+
+- FinePDFs seed loading must use repo `HuggingFaceFW/finepdfs` pinned to lowercase 40-character SHA `220bac3acbf07789502c621d2d33952f51ac7f86`.
+- Task validation must remain offline/static; do not call `load_dataset`, download PDFs, or run the seed stage.