Merge pull request #3 from lagillenwater/data-downloaders

Data downloaders

Author: lagillenwater

.gitignore

@@ -218,9 +218,13 @@ __marimo__/
 .streamlit/secrets.toml
 
 # fmharness project-specific
-data/tranches/        # cached tranche artifacts (large)
-reports/              # generated reports
-containers/*.sif      # built Apptainer images
-.fmharness/           # local user secrets/state
+# all dataset artifacts (raw, processed, tranches) — never tracked
+data/
+# generated reports
+reports/
+# built Apptainer images
+containers/*.sif
+# local user secrets/state
+.fmharness/
 
 docs/fm-pdo-evaluator-plan.md

docs/adapter_contract.md

@@ -1,8 +1,8 @@
 # Adapter contract
 
-How a model joins the harness. Every model — linear baseline, Tahoe-x1, STATE, and any future addition — implements the same `ModelAdapter` Protocol so the rest of the pipeline (splits, probe, metrics, registry, leakage scan) is model-agnostic.
+How a model joins the harness. Every model — linear baseline, Tahoe-x1, STACK, and any future addition — implements the same `ModelAdapter` Protocol so the rest of the pipeline (splits, probe, metrics, registry, leakage scan) is model-agnostic.
 
-The contract has three required surfaces (`embed`, `metadata`, `version`) and one optional surface (`predict_native`). Day 7 implements the Protocol and the `linear_baseline` and `MockAdapter` reference implementations; later days add `tahoe_x1` (Day 8) and `state` (Day 11) against the same contract.
+The contract has three required surfaces (`embed`, `metadata`, `version`) and one optional surface (`predict_native`). Day 7 implements the Protocol and the `linear_baseline` and `MockAdapter` reference implementations; later days add `tahoe_x1` (Day 8) and `stack` (Day 11) against the same contract.
 
 ## 1. The interface
 
@@ -50,7 +50,7 @@ class ModelAdapter(Protocol):
 
 ## 2. The probe-based prediction pipeline (default path)
 
-All four matrix rows (linear baseline, Tahoe-x1, STATE, plus the metadata-only control) share an identical probe so the comparison isolates "what the encoder captures":
+All four matrix rows (linear baseline, Tahoe-x1, STACK, plus the metadata-only control) share an identical probe so the comparison isolates "what the encoder captures":
 
 ```
   sample (RNA-seq)  --[ encoder ]-->  embedding  --[ concat drug feat ]-->  [ probe ]  --> P(responder)
@@ -59,7 +59,7 @@ All four matrix rows (linear baseline, Tahoe-x1, STATE, plus the metadata-only c
                                                                        trained per split-fold
 ```
 
-- **Encoder** is model-specific. For the linear baseline the encoder is `StandardScaler` (a passthrough; "embedding" == scaled expression). For Tahoe-x1 / STATE it is the pretrained transformer encoder.
+- **Encoder** is model-specific. For the linear baseline the encoder is `StandardScaler` (a passthrough; "embedding" == scaled expression). For Tahoe-x1 / STACK it is the pretrained transformer encoder.
 - **Drug feature** is a one-hot over the drug crosswalk's canonical IDs at MVP; richer drug descriptors (Morgan fingerprint, ATC class) are a deferred extension.
 - **Probe** is a fixed architecture across all models: `StandardScaler → ElasticNetCV` (continuous response) or `LogisticRegressionCV` (binary responder). Declared once in `src/fmharness/probe/linear.py`. The harness — not the adapter — owns the probe.
 
@@ -70,7 +70,7 @@ The adapter's only job is to produce a faithful embedding. Probe training and in
 Foundation models with a drug-aware head can return a prediction directly:
 
 - Tahoe-x1: trained on perturbation-response prediction; may expose a `predict(baseline_state, drug) → post_state` or scalar head.
-- STATE: the ST (state transition) component is exactly this.
+- STACK: `predict_in_context()` consumes a prompt set of cells defining a biological condition and predicts the effect on a target population — no per-task fine-tuning. This is STACK's native zero-shot path.
 
 When `predict_native` returns a value, the harness records it as a separate row in the registry tagged `prediction_mode="native"`. The probe-based row (`prediction_mode="probe"`) is always produced for fair comparison; the native row is supplementary.
 

docs/environment.md

@@ -15,9 +15,9 @@ Foundation-model inference runs inside Apptainer images pinned by digest in
 |---|---|---|---|
 | `fmharness` | `containers/fmharness.def` | Day 1+2 (skeleton); rebuilt Day 8 | core Python deps |
 | `tahoe` | `containers/tahoe.def` | Day 8 | Tahoe-x1 + torch + CUDA |
-| `state` | `containers/state.def` (built only if needed) | Day 11 | STATE + torch + CUDA |
+| `stack` | `containers/stack.def` (built only if needed) | Day 11 | STACK + torch + CUDA |
 
-STATE reuses the Tahoe container unless torch/CUDA conflicts force a split;
+STACK reuses the Tahoe container unless torch/CUDA conflicts force a split;
 the decision (and the reason) is recorded on Day 11 in this document.
 
 Every `PredictionRecord` carries `EnvironmentSnapshot.container_digest`. A

pyproject.toml

@@ -19,15 +19,16 @@ dependencies = [
     "joblib>=1.4",
     "typer>=0.12",
     "pydeseq2>=0.4",
+    "openpyxl>=3.1",
 ]
 
 [project.optional-dependencies]
 ml = [
     "torch>=2.2",
     "transformers>=4.40",
 ]
-state = [
-    "arc-state>=0.1",
+stack = [
+    "arc-stack @ git+https://github.com/ArcInstitute/stack.git",
 ]
 dev = [
     "pytest>=8.0",
@@ -42,6 +43,9 @@ dev = [
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/fmharness"]
 

scripts/download/_utils.py

@@ -0,0 +1,99 @@
+"""Shared helpers for download scripts under scripts/download/.
+
+Common manifest shape both downloaders produce::
+
+    {
+      "dataset": "<short id>",
+      "release": {<dataset-specific release/version keys>},
+      "updated_utc": "<ISO8601>",
+      "files": {
+        "<relative_path>": {
+          "sha256": "...",
+          "bytes": N,
+          "source_uri": "https://... | synapse://syn_id"
+        }
+      }
+    }
+
+Per-file extras (e.g. ``rows`` / ``cols`` / ``columns`` for Synapse Tables) are
+allowed on top of this; downstream consumers ignore unknown keys.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+MANIFEST_NAME = "manifest.json"
+
+
+def sha256_file(path: Path, chunk: int = 1 << 20) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for block in iter(lambda: fh.read(chunk), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def load_manifest(manifest_path: Path, default: dict) -> dict:
+    if manifest_path.exists():
+        return json.loads(manifest_path.read_text())
+    return default
+
+
+def write_manifest(manifest_path: Path, manifest: dict) -> None:
+    manifest["updated_utc"] = datetime.now(UTC).isoformat(timespec="seconds")
+    manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
+
+
+def skip_or_fail_on_hash(name: str, dest: Path, manifest_files: dict) -> bool:
+    """Return True if ``dest`` exists with sha matching the manifest record.
+
+    Returns False when ``dest`` is missing or has no recorded sha — caller
+    should (re-)fetch. Exits non-zero on mismatch so a silently changed
+    upstream file does not slip past unnoticed.
+    """
+    if not dest.exists():
+        return False
+    recorded = manifest_files.get(name, {}).get("sha256")
+    if not recorded:
+        return False
+    actual = sha256_file(dest)
+    if recorded != actual:
+        sys.exit(
+            f"[fail] {name} sha256 mismatch -- expected {recorded}, got {actual}. "
+            "Delete the file to re-download, or investigate upstream change."
+        )
+    return True
+
+
+def verify_manifest(output_dir: Path, manifest_path: Path) -> int:
+    """Re-check every recorded file under ``output_dir`` against its manifest sha.
+
+    Returns the number of errors (0 = clean). Prints one ``[fail]`` line per
+    bad file. Returns 0 silently when the manifest is missing (no records to
+    verify), so callers can iterate over multiple optional manifests.
+    """
+    if not manifest_path.exists():
+        return 0
+    manifest = json.loads(manifest_path.read_text())
+    bad: list[str] = []
+    for name, rec in manifest.get("files", {}).items():
+        path = output_dir / name
+        if not path.exists():
+            bad.append(f"{name}: missing")
+            continue
+        actual = sha256_file(path)
+        if actual != rec["sha256"]:
+            bad.append(f"{name}: sha256 mismatch (expected {rec['sha256']}, got {actual})")
+    if bad:
+        for b in bad:
+            print(f"[fail] {b}")
+        return len(bad)
+    n = len(manifest.get("files", {}))
+    print(f"[ok  ] {manifest_path.parent.name}/{manifest_path.name}: {n} files verified")
+    return 0

scripts/download/download_gdsc2_sarcoma.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Download GDSC2 drug-response + CCLE/DepMap RNA-seq for the sarcoma subset.
+
+Two open data sources, paired on COSMIC ID:
+
+1. GDSC2 (Sanger Institute, release 8.5, dated 27Oct23). Fitted dose-response
+   (AUC, IC50), compound metadata, cell-line metadata. Hosted on the public
+   Sanger CDN; no auth required.
+
+2. DepMap (Broad Institute, latest public release, ~quarterly). RNA-seq raw
+   read counts (per protein-coding gene, RSEM-style; fed through DESeq2
+   median-of-ratios in the loader to match Soragni's normalization scheme)
+   plus model metadata mapping DepMap ACH-XXXXX IDs to COSMIC IDs. DepMap
+   distributes through figshare collections that change per release (26Q1,
+   25Q4, ...). To swap to a newer release:
+
+   Go to https://depmap.org/portal/download/all/, pick the latest release,
+   right-click the file and copy the figshare ndownloader link, then update
+   the corresponding entry in DEPMAP_FILES below.
+
+Outputs land in data/raw/gdsc2_sarcoma/ with sha256 of each file recorded in
+manifest.json. The manifest schema is shared with download_soragni.py via
+scripts/download/_utils.py.
+
+Usage:
+    python scripts/download/download_gdsc2_sarcoma.py
+    python scripts/download/download_gdsc2_sarcoma.py --verify-only
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import urllib.request
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+
+from _utils import (
+    MANIFEST_NAME,
+    load_manifest,
+    sha256_file,
+    skip_or_fail_on_hash,
+    verify_manifest,
+    write_manifest,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+OUTPUT_DIR = REPO_ROOT / "data" / "raw" / "gdsc2_sarcoma"
+MANIFEST_PATH = OUTPUT_DIR / MANIFEST_NAME
+
+# GDSC2 release 8.5 (27Oct23) -- verified live on the Sanger CDN.
+# Bump these when Sanger publishes a new release.
+GDSC2_RELEASE = "8.5"
+GDSC2_BASE = f"https://cog.sanger.ac.uk/cancerrxgene/GDSC_release{GDSC2_RELEASE}"
+GDSC2_FILES: list[tuple[str, str]] = [
+    (
+        "GDSC2_fitted_dose_response_27Oct23.xlsx",
+        f"{GDSC2_BASE}/GDSC2_fitted_dose_response_27Oct23.xlsx",
+    ),
+    ("screened_compounds_rel_8.5.csv", f"{GDSC2_BASE}/screened_compounds_rel_8.5.csv"),
+    ("Cell_Lines_Details.xlsx", f"{GDSC2_BASE}/Cell_Lines_Details.xlsx"),
+]
+
+# DepMap files -- release 26Q1 (canonical-id snapshot public-26q1-5bbf).
+# URLs hit depmap.org which 302s to a signed Google Cloud Storage URL that's
+# regenerated per request, so the depmap.org URL itself is stable across runs.
+#
+# RawReadCount is RSEM expected counts per protein-coding gene (linear,
+# integer-ish); the loader runs DESeq2 median-of-ratios on this to match
+# Soragni's pre-computed normalized counts. The log2(TPM+1) variant at the
+# same canonical-id (.27) is available if a length-normalized view is ever
+# needed for a sensitivity row; not pulled by default.
+DEPMAP_RELEASE = "26Q1"
+DEPMAP_BASE = "https://depmap.org/portal/download/api/download"
+DEPMAP_FILES: list[tuple[str, str]] = [
+    (
+        "OmicsExpressionRawReadCountHumanProteinCodingGenes.csv",
+        f"{DEPMAP_BASE}?file_name=downloads-by-canonical-id%2Fpublic-26q1-5bbf.27%2FOmicsExpressionRawReadCountHumanProteinCodingGenes.csv&dl_name=OmicsExpressionRawReadCountHumanProteinCodingGenes.csv&bucket=depmap-external-downloads",
+    ),
+    (
+        "Model.csv",
+        f"{DEPMAP_BASE}?file_name=downloads-by-canonical-id%2Fpublic-26q1-5bbf.37%2FModel.csv&dl_name=Model.csv&bucket=depmap-external-downloads",
+    ),
+]
+
+
+def default_manifest() -> dict:
+    return {
+        "dataset": "gdsc2_sarcoma",
+        "release": {"gdsc": GDSC2_RELEASE, "depmap": DEPMAP_RELEASE},
+        "files": {},
+    }
+
+
+def download(url: str, dest: Path) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dest.with_suffix(dest.suffix + ".part")
+    req = urllib.request.Request(url, headers={"User-Agent": "fm-pdo-evaluator/0.1"})
+    with urllib.request.urlopen(req) as response, tmp.open("wb") as fh:
+        while True:
+            chunk = response.read(1 << 20)
+            if not chunk:
+                break
+            fh.write(chunk)
+    tmp.replace(dest)
+
+
+def fetch_with_sha(name: str, url: str, manifest: dict) -> None:
+    dest = OUTPUT_DIR / name
+    if skip_or_fail_on_hash(name, dest, manifest["files"]):
+        print(f"[skip] {name} present with matching sha256")
+        return
+    print(f"[get ] {name} <- {url}")
+    try:
+        download(url, dest)
+    except (HTTPError, URLError) as e:
+        sys.exit(f"[fail] {name} download error: {e}")
+    digest = sha256_file(dest)
+    manifest["files"][name] = {
+        "sha256": digest,
+        "bytes": dest.stat().st_size,
+        "source_uri": url,
+    }
+    print(f"       sha256 {digest}")
+
+
+def fetch_gdsc2(manifest: dict) -> None:
+    for name, url in GDSC2_FILES:
+        fetch_with_sha(f"gdsc2/{name}", url, manifest)
+
+
+def fetch_depmap(manifest: dict) -> None:
+    for name, url in DEPMAP_FILES:
+        fetch_with_sha(f"depmap/{name}", url, manifest)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--verify-only", action="store_true", help="re-verify existing files; no download"
+    )
+    args = parser.parse_args()
+
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    manifest = load_manifest(MANIFEST_PATH, default_manifest())
+
+    if args.verify_only:
+        errors = verify_manifest(OUTPUT_DIR, MANIFEST_PATH)
+        sys.exit(1 if errors else 0)
+
+    fetch_gdsc2(manifest)
+    fetch_depmap(manifest)
+    expected = {f"gdsc2/{n}" for n, _ in GDSC2_FILES} | {f"depmap/{n}" for n, _ in DEPMAP_FILES}
+    manifest["files"] = {k: v for k, v in manifest["files"].items() if k in expected}
+    write_manifest(MANIFEST_PATH, manifest)
+    print(f"[done] manifest written to {MANIFEST_PATH.relative_to(REPO_ROOT)}")
+
+
+if __name__ == "__main__":
+    main()