Feat exaspim

pyproject.toml

@@ -23,6 +23,8 @@ dependencies = [
     "jsonschema",
     "aind-data-schema>=2.6.0,<3",
     "aind-metadata-extractor==0.3.11",
+    "aind-metadata-upgrader>=0.13.4,<1",
+    "packaging>=21.0",
 ]
 
 [project.optional-dependencies]

scripts/exaspim/test_exaspim_metadata.py

@@ -0,0 +1,228 @@
+# flake8: noqa: C901
+"""Integration test for the exaSPIM metadata mapper.
+
+Copies ``acquisition.json`` and ``instrument.json`` from a real exaSPIM
+dataset directory into a temp directory, then runs
+:class:`~aind_metadata_mapper.exaspim.mapper.ExaSPIMMapper` to upgrade
+v1 metadata to v2.  No metadata-service calls are made — this purely
+exercises the mapper / upgrader pipeline.
+
+Usage
+-----
+.. code-block:: bash
+
+   python scripts/exaspim/test_exaspim_metadata.py \\
+       --dataset-dir /allen/aind/stage/exaspim/exaSPIM_826507_2026-05-29_16-56-55
+"""
+
+import argparse
+import json
+import logging
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+from packaging import version as pkg_version
+
+from aind_metadata_mapper.base import MapperJobSettings
+from aind_metadata_mapper.exaspim.mapper import ExaSPIMMapper
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+DEFAULT_DATASET_DIR = "/allen/aind/stage/exaspim/1x_screening/" "exaSPIM_821805_2026-05-11_16-32-29"
+
+
+def _print_header(title: str) -> None:
+    """Print a section header."""
+    print("\n" + "=" * 80)
+    print(title)
+    print("=" * 80)
+
+
+def _load(path: Path) -> dict:
+    """Load a JSON file and return the parsed dict."""
+    with open(path) as fh:
+        return json.load(fh)
+
+
+def main() -> int:
+    """Run the exaSPIM integration test.
+
+    Returns
+    -------
+    int
+        Exit code: 0 on success, 1 on failure.
+    """
+    parser = argparse.ArgumentParser(
+        description="Integration test for the exaSPIM metadata mapper.",
+    )
+    parser.add_argument(
+        "--dataset-dir",
+        type=Path,
+        default=Path(DEFAULT_DATASET_DIR),
+        help=(
+            "Path to a real exaSPIM dataset directory containing "
+            "v1 acquisition.json (and optionally instrument.json). "
+            f"Default: {DEFAULT_DATASET_DIR}"
+        ),
+    )
+    args = parser.parse_args()
+    dataset_dir: Path = args.dataset_dir
+
+    # ------------------------------------------------------------------
+    # Validate source directory
+    # ------------------------------------------------------------------
+    _print_header("INTEGRATION TEST: ExaSPIM Metadata Mapper")
+    print(f"Dataset directory: {dataset_dir}")
+
+    if not dataset_dir.is_dir():
+        print(f"\n✗ ERROR: directory does not exist: {dataset_dir}")
+        return 1
+
+    src_acq = dataset_dir / "acquisition.json"
+    src_inst = dataset_dir / "instrument.json"
+    src_inst_yaml = dataset_dir / "derivatives" / "instrument_config.yaml"
+
+    if not src_acq.is_file():
+        print(f"\n✗ ERROR: acquisition.json not found in {dataset_dir}")
+        return 1
+
+    has_instrument = src_inst.is_file()
+    has_inst_yaml = src_inst_yaml.is_file()
+    print(f"  acquisition.json           : found")
+    if has_instrument:
+        print(f"  instrument.json            : found")
+    elif has_inst_yaml:
+        print(f"  instrument.json            : NOT FOUND")
+        print(f"  derivatives/instrument_config.yaml : found (detection fallback)")
+    else:
+        print(f"  instrument.json            : NOT FOUND (will use acq-only upgrade)")
+
+    # ------------------------------------------------------------------
+    # Copy files to a temp directory
+    # ------------------------------------------------------------------
+    temp_dir = tempfile.mkdtemp(prefix="exaspim_integration_")
+    tmp_path = Path(temp_dir)
+
+    try:
+        print(f"\nTemporary directory: {tmp_path}")
+        shutil.copy(src_acq, tmp_path / "acquisition.json")
+        os.chmod(tmp_path / "acquisition.json", 0o644)
+        if has_instrument:
+            shutil.copy(src_inst, tmp_path / "instrument.json")
+            os.chmod(tmp_path / "instrument.json", 0o644)
+        if has_inst_yaml:
+            deriv = tmp_path / "derivatives"
+            deriv.mkdir(exist_ok=True)
+            shutil.copy(src_inst_yaml, deriv / "instrument_config.yaml")
+            os.chmod(deriv / "instrument_config.yaml", 0o644)
+        print("✓ Files copied (permissions fixed to 0644)")
+
+        # ------------------------------------------------------------------
+        # Phase 1 — Detection
+        # ------------------------------------------------------------------
+        _print_header("PHASE 1: Detection")
+
+        detected = ExaSPIMMapper.detect(tmp_path)
+        if detected:
+            print("✓ ExaSPIMMapper.detect() → True")
+        else:
+            print("✗ ExaSPIMMapper.detect() → False")
+            if not has_instrument:
+                print("  (Expected — no instrument.json for detection. " "Continuing with direct upgrade.)")
+            else:
+                print(
+                    "  WARNING: instrument.json is present but "
+                    "detection failed. The upgrade will still be "
+                    "attempted."
+                )
+
+        # ------------------------------------------------------------------
+        # Phase 2 — Pre-upgrade inspection
+        # ------------------------------------------------------------------
+        _print_header("PHASE 2: Pre-Upgrade Inspection")
+
+        acq_before = _load(tmp_path / "acquisition.json")
+        acq_sv_before = acq_before.get("schema_version", "unknown")
+        print(f"  acquisition.json schema_version : {acq_sv_before}")
+        print(f"  subject_id                      : {acq_before.get('subject_id')}")
+        print(f"  instrument_id                   : {acq_before.get('instrument_id')}")
+        print(f"  tiles                           : {len(acq_before.get('tiles', []))}")
+
+        inst_sv_before = None
+        if has_instrument:
+            inst_before = _load(tmp_path / "instrument.json")
+            inst_sv_before = inst_before.get("schema_version", "unknown")
+            print(f"  instrument.json schema_version  : {inst_sv_before}")
+            print(f"  instrument_type                 : {inst_before.get('instrument_type')}")
+
+        # ------------------------------------------------------------------
+        # Phase 3 — Upgrade
+        # ------------------------------------------------------------------
+        _print_header("PHASE 3: Upgrade")
+
+        mapper = ExaSPIMMapper()
+        job_settings = MapperJobSettings(
+            input_filepath=tmp_path / "acquisition.json",
+            output_directory=tmp_path,
+            output_filename_suffix="exaspim",
+        )
+
+        print("Running ExaSPIMMapper.run_job() …")
+        try:
+            mapper.run_job(job_settings)
+            print("✓ Upgrade completed successfully")
+        except Exception as exc:
+            print(f"✗ Upgrade FAILED: {exc}")
+            return 1
+
+        # ------------------------------------------------------------------
+        # Phase 4 — Post-upgrade validation
+        # ------------------------------------------------------------------
+        _print_header("PHASE 4: Post-Upgrade Validation")
+
+        acq_after = _load(tmp_path / "acquisition.json")
+        acq_sv_after = acq_after.get("schema_version", "unknown")
+        acq_ok = pkg_version.parse(acq_sv_after) >= pkg_version.parse("2.0.0")
+        has_streams = "data_streams" in acq_after
+        n_streams = len(acq_after.get("data_streams", []))
+
+        print(f"  acquisition.json schema_version : {acq_sv_before} → {acq_sv_after}")
+        print(f"  schema_version >= 2.0.0         : {'✓' if acq_ok else '✗'}")
+        print(f"  data_streams present            : {'✓' if has_streams else '✗'} ({n_streams} streams)")
+
+        inst_ok = True
+        if has_instrument and (tmp_path / "instrument.json").is_file():
+            inst_after = _load(tmp_path / "instrument.json")
+            inst_sv_after = inst_after.get("schema_version", "unknown")
+            inst_ok = pkg_version.parse(inst_sv_after) >= pkg_version.parse("2.0.0")
+            print(f"  instrument.json schema_version  : {inst_sv_before} → {inst_sv_after}")
+            print(f"  schema_version >= 2.0.0         : {'✓' if inst_ok else '✗'}")
+
+        # ------------------------------------------------------------------
+        # Summary
+        # ------------------------------------------------------------------
+        _print_header("SUMMARY")
+
+        all_passed = acq_ok and has_streams and inst_ok
+        print(f"  Detection          : {'✓' if detected else '⚠ (not critical)'}")
+        print(f"  Acquisition upgrade: {'✓' if acq_ok else '✗'}")
+        print(f"  data_streams       : {'✓' if has_streams else '✗'}")
+        if has_instrument:
+            print(f"  Instrument upgrade : {'✓' if inst_ok else '✗'}")
+        print()
+        print(f"  Overall: {'✓ PASSED' if all_passed else '✗ FAILED'}")
+        print("=" * 80 + "\n")
+
+        return 0 if all_passed else 1
+
+    finally:
+        print(f"Cleaning up temporary directory: {tmp_path}")
+        shutil.rmtree(temp_dir, ignore_errors=True)
+        print("✓ Cleanup complete\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/integration_test.py

@@ -36,7 +36,7 @@
         subject_id="804670",
         project_name="Learning mFISH-V1omFISH",
         modalities=[Modality.POPHYS, Modality.BEHAVIOR_VIDEOS, Modality.BEHAVIOR],
-        metadata_service_url="http://aind-metadata-service-dev", # DEV metadata service for testing
+        metadata_service_url="http://aind-metadata-service-dev",  # DEV metadata service for testing
     )
 
     job = GatherMetadataJob(settings=settings)

src/aind_metadata_mapper/base.py

@@ -30,3 +30,4 @@ def run_job(self, job_settings: MapperJobSettings) -> None:
         within the metadata_directory.
         """
         raise NotImplementedError("Subclasses should implement this method.")
+        raise NotImplementedError

src/aind_metadata_mapper/exaspim/README.md

@@ -0,0 +1,30 @@
+# exaSPIM Metadata Mapper
+
+Pre-processor that detects exaSPIM metadata and upgrades v1 `acquisition.json`
+/ `instrument.json` to aind-data-schema v2.
+
+## Detection
+
+The mapper checks, in order:
+
+1. `instrument.json` — does `instrument_id` or `instrument_type` contain
+   "exaspim" (case-insensitive)?
+2. Fall-back: does a `derivatives/instrument_config.yaml` file in the metadata
+   directory contain "exaspim" (case-insensitive)?
+
+## What it does
+
+* **Already v2 (schema_version ≥ 2.0.0):** no-op.
+* **v1 with instrument.json:** sanitises manufacturer names and deprecated
+  fields, then upgrades both files via `aind-metadata-upgrader`.
+* **v1 without instrument.json:** upgrades acquisition only, using an empty
+  filter / light-source stub.
+
+Upgraded files are written back **in place** in the metadata directory.
+S3 upload is the caller's responsibility (e.g. `aind-data-transfer-service`).
+
+## Integration
+
+Registered in the `pre_processor_registry` of `mapper_registry.py`.
+`GatherMetadataJob` runs all matching pre-processors before the normal
+metadata-gather flow.

src/aind_metadata_mapper/exaspim/__init__.py

@@ -0,0 +1 @@
+"""exaSPIM metadata mapper — upgrades v1 metadata to v2."""

src/aind_metadata_mapper/exaspim/constants.py

@@ -0,0 +1,85 @@
+"""Constants for the exaSPIM metadata mapper."""
+
+# Schema version threshold — anything below this must be upgraded.
+V2_THRESHOLD = "2.0.0"
+
+# Case-insensitive keywords used to detect exaSPIM instruments.
+EXASPIM_INSTRUMENT_KEYWORDS: tuple[str, ...] = ("exaspim",)
+
+# Instrument JSON keys that carry a ``manufacturer`` dict.
+DEVICE_LISTS: tuple[str, ...] = (
+    "objectives",
+    "detectors",
+    "light_sources",
+    "fluorescence_filters",
+    "lenses",
+    "scanning_stages",
+    "motorized_stages",
+    "additional_devices",
+    "daqs",
+    "optical_tables",
+)
+
+# Fields on MotorizedStage that were valid in v1 but removed in v2.
+DEPRECATED_MOTORIZED_FIELDS: tuple[str, ...] = (
+    "stage_axis_direction",
+    "stage_axis_name",
+)
+
+# Allowed ``travel_unit`` values for stages in v2.
+VALID_TRAVEL_UNITS: set[str] = {
+    "meter",
+    "centimeter",
+    "millimeter",
+    "micrometer",
+    "nanometer",
+    "inch",
+    "pixel",
+}
+
+# Canonical ``stage_axis_direction`` values in v2.
+VALID_AXIS_DIRECTIONS: set[str] = {
+    "Detection axis",
+    "Illumination axis",
+    "Perpendicular axis",
+}
+
+# Maps lowercase keywords in free-form direction strings → canonical values.
+# Order matters: more specific keywords must come first (e.g.
+# "perpendicular" before "detection" so that "Perpendicular to detection"
+# maps to "Perpendicular axis", not "Detection axis").
+AXIS_DIRECTION_MAP: dict[str, str] = {
+    "perpendicular": "Perpendicular axis",
+    "illumination": "Illumination axis",
+    "detection": "Detection axis",
+}
+
+# Maps legacy v1 filter_type strings to the v2 canonical forms accepted
+# by aind-data-schema-models.  The upgrader does not normalise these.
+FILTER_TYPE_MAP: dict[str, str] = {
+    "Bandpass": "Band pass",
+    "Band Pass": "Band pass",
+    "Longpass": "Long pass",
+    "Long Pass": "Long pass",
+    "Shortpass": "Short pass",
+    "Short Pass": "Short pass",
+}
+
+# Allowed ``stage_axis_name`` values for scanning stages in v2.
+VALID_STAGE_AXIS_NAMES: set[str] = {
+    "X",
+    "Y",
+    "Z",
+    "AP",
+    "ML",
+    "SI",
+    "Depth",
+}
+
+# Maps ``instrument.id`` from ``instrument_config.yaml`` to the
+# corresponding reference instrument JSON filename bundled in
+# ``src/aind_metadata_mapper/exaspim/instruments/``.
+INSTRUMENT_ID_MAP: dict[str, str] = {
+    "exaspim-01": "beta02_instrument.json",
+    "exaspim-1x": "1x_instrument.json",
+}