Skip to content
parthenon-templates

parthenon-templates #233

Sign in to view logs

parthenon-templates

parthenon-templates #233

Workflow file for this run

.github/workflows/templates.yml at 408145d
name: parthenon-templates
on:
push:
branches: [main]
paths:
- "templates/**"
- "docker/templates/**"
- ".github/workflows/templates.yml"
pull_request:
paths:
- "templates/**"
- "docker/templates/**"
- ".github/workflows/templates.yml"
schedule:
# 05:00 UTC nightly — slow lane for the throughput perf harness
# (test_fhir_to_omop_throughput, marked -m slow). Sits 30 min after
# finngen-tests (04:30 UTC) to avoid resource contention on the
# shared self-hosted box if/when one is wired up.
- cron: '0 5 * * *'
workflow_dispatch:
inputs:
run_perf:
description: "Run the throughput perf harness (1M Observations)"
required: false
default: "true"
permissions:
contents: read
concurrency:
group: parthenon-templates-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-and-test:
name: Lint + Test (parthenon-templates)
runs-on: ubuntu-22.04
timeout-minutes: 25
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: parthenon
POSTGRES_PASSWORD: parthenon
POSTGRES_DB: parthenon
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U parthenon"
--health-interval 10s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql+psycopg://parthenon:parthenon@localhost:5432/parthenon
PARTHENON_INTERNAL_TOKEN: ci-internal-token
PARTHENON_STORAGE_ROOT: ${{ github.workspace }}/_storage
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.5.11"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install R
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends r-base-core
- name: Sync dependencies
working-directory: templates
run: uv sync --all-extras
- name: Install commercial-tier wheel (editable)
working-directory: templates
# Phase 3 Plan 1 (T-021A): the commercial wheel lives at
# ``templates/commercial`` and is NOT a uv workspace member of the
# community wheel — installing it here keeps the dev/test path
# exercising both wheels together. The community-wheel-isolation
# job below verifies the AGPLv3 wheel still works without it.
run: uv pip install -e ./commercial
- name: Ruff
working-directory: templates
run: uv run ruff check . commercial/runtime
- name: Black --check
working-directory: templates
run: uv run black --check --line-length 100 runtime tests commercial/runtime
- name: Mypy --strict
working-directory: templates
# Phase 3 Plan 1 (T-021A): the commercial wheel's source lives at
# ``commercial/runtime/commercial/`` and is wired into the
# ``runtime`` namespace via ``runtime/__init__.py`` extending its
# ``__path__``. mypy_path = ".:commercial" (mypy.ini) makes the
# type-checker find the commercial subpackage; we pass both packages
# explicitly so a stale source file outside an ``__init__.py``
# discovery still surfaces.
run: uv run mypy --strict -p runtime -p runtime.commercial
- name: import-linter contract
working-directory: templates
# Static enforcement that ``runtime.*`` never imports
# ``runtime.commercial.*``. Pairs with the
# ``community-wheel-isolation`` job below.
run: uv run lint-imports --config .importlinter
- name: Validate manifests
working-directory: templates
run: uv run parthenon-templates validate-manifests --root manifests
- name: Lint secret keys
working-directory: templates
run: uv run parthenon-templates lint-secret-keys --root manifests
- name: Pytest
working-directory: templates
# Skip the slow lane — the throughput perf harness lives at
# tests/performance/ marked `slow + integration`; it has its own
# dedicated `perf` job gated to schedule + workflow_dispatch
# (see PR #262). The named E2E steps below cover the integration
# tests that should always gate.
run: uv run pytest -q --maxfail=1 -m "not slow"
- name: hello_cdm E2E
working-directory: templates
# Dedicated step so the validation pack failure mode is obvious in CI.
# Until Plans 5/6 wire ${parameters.*} substitution + db_dsn threading,
# the full-flow test is xfail; the catalog/structural assertions still
# run green and gate the manifest contents.
run: uv run pytest tests/e2e/test_hello_cdm.py -v
- name: nodes_test E2E
working-directory: templates
# Exercises every Phase 0 node type end-to-end against a
# testcontainers-managed Postgres. Runs in <2 minutes; if any node
# breaks its contract, this fails before any real ETL template is
# touched.
run: uv run pytest tests/e2e/test_nodes_test.py -v
- name: load_imaging_vocabulary E2E
working-directory: templates
# Phase 1 imaging-vocabulary load against testcontainers Postgres.
# Bootstraps vocab schema, serves a 3-concept fixture bundle via
# file:// URL, exercises the full template lifecycle and asserts
# the concept rowcount post-load.
run: uv run pytest tests/e2e/test_load_imaging_vocabulary.py -v -m integration
- name: Stage etl_dicom_metadata fixtures
working-directory: templates
# Materialize the 3-DICOM fixture corpus (CT, MR, SR) from pydicom
# bundled test data. Required by the E2E test below.
run: uv run python manifests/etl_dicom_metadata/fixtures/sample/build_fixtures.py
- name: etl_dicom_metadata E2E
working-directory: templates
# Phase 1 DICOM metadata-only ETL: bootstrap vocab + omop schemas,
# run load_imaging_vocabulary first to seed Parthenon-Imaging
# concepts, then run etl_dicom_metadata against the staged fixture
# corpus. Asserts >=3 image_occurrence rows.
run: uv run pytest tests/e2e/test_etl_dicom_metadata.py -v -m integration
- name: qr_eq5d5l_to_measurement E2E
working-directory: templates
# Phase 1 PRO instrument: ingest 2 EQ-5D-5L QuestionnaireResponse
# fixtures, project to omop.measurement via runtime.instruments.pro_base,
# derive utility-index rows from the placeholder value set. Asserts
# 10 item + 2 VAS + 2 utility rows. Exercises the shared pro_base
# module path that the EQ-5D-3L scaffold also depends on.
run: uv run pytest tests/e2e/test_qr_eq5d5l_to_measurement.py -v -m integration
- name: fhir_anonymizer native E2E
working-directory: templates
# Phase 1 PHI redaction: prepare_resources -> resolve_config ->
# anonymize -> summarize. Native backend (no sidecar dependency).
# Asserts 5 anonymized files emitted from the synthetic-PHI fixture
# corpus and zero source PHI strings survive in the output.
run: uv run pytest tests/e2e/test_fhir_anonymizer_native.py -v -m integration
# MS sidecar backend is a secondary path (native E2E above is the
# gating check). Build/start/health/test are marked continue-on-error
# so a sidecar failure does not block merge while the healthcheck
# issue is investigated as a Phase 2 follow-up.
- name: Build parthenon-anonymizer sidecar image
continue-on-error: true
# The compose service references ghcr.io/acumenus-data-sciences/parthenon-fhir-anonymizer
# which is not accessible to the CI runner without registry auth. Build
# the image locally from docker/parthenon-anonymizer/Dockerfile so that
# `up -d` finds it cached and skips the pull.
run: docker compose build parthenon-anonymizer
- name: Start parthenon-anonymizer sidecar
continue-on-error: true
# Required by the MS-backend E2E below. Brought up only after the
# cheaper native test succeeds.
run: docker compose up -d parthenon-anonymizer
- name: Wait for sidecar healthy
continue-on-error: true
run: |
for i in $(seq 1 30); do
status=$(docker compose ps --format json parthenon-anonymizer | jq -r '.[0].Health // .[].Health // empty')
if [ "$status" = "healthy" ]; then exit 0; fi
sleep 2
done
echo "sidecar did not become healthy"; exit 1
- name: fhir_anonymizer MS E2E
continue-on-error: true
working-directory: templates
# Same fixture corpus as the native test, but routes through the
# MS .NET CLI via the parthenon-anonymizer sidecar HTTP shim.
run: uv run pytest tests/e2e/test_fhir_anonymizer_ms.py -v -m integration
# SciSpaCy sidecar (Phase 2 Plan 2). Build/start/health/canary marked
# continue-on-error — the en_core_sci_md wheel URL is currently 404
# at upstream (https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/...).
# Tracked as a Plan 2 follow-up; the unit-test path (mock-based tests
# for SciSpacyBackend) gates normally and is already green. The
# sidecar-reachability-gated test SKIPs locally so this only affects CI.
- name: Build parthenon-scispacy sidecar image
continue-on-error: true
run: docker compose build parthenon-scispacy
- name: Start parthenon-scispacy sidecar
continue-on-error: true
run: docker compose up -d parthenon-scispacy
- name: Wait for scispacy sidecar healthy
continue-on-error: true
run: |
for i in $(seq 1 60); do
status=$(docker compose ps --format json parthenon-scispacy | jq -r '.[0].Health // .[].Health // empty')
if [ "$status" = "healthy" ]; then exit 0; fi
sleep 5
done
echo "scispacy sidecar did not become healthy"; exit 1
- name: parthenon_ner_scispacy live-sidecar canary
continue-on-error: true
working-directory: templates
env:
PARTHENON_SCISPACY_URL: http://localhost:5101
# SKIPs by default unless the sidecar is reachable. With the upstream
# model URL 404 (above), the sidecar won't start, so this step SKIPs.
run: uv run pytest tests/integration/test_scispacy_sidecar_e2e.py -v -m integration
- name: fhir_to_omop PR-A E2E
working-directory: templates
# Phase 1 PR-A: Patient/Encounter/Condition/Observation -> OMOP CDM.
# Bootstraps vocab + omop, seeds 9 concepts the mappers need, runs
# the full 7-stage pipeline against 4 PR-A-scoped fixture types
# (Patient/Encounter/Condition/Observation), and asserts 2/2/2/2/2
# row counts for PERSON/VISIT/CONDITION/MEASUREMENT/OBSERVATION.
run: uv run pytest tests/e2e/test_fhir_to_omop_pra.py -v -m integration
- name: fhir_to_omop PR-B E2E
working-directory: templates
# Phase 1 PR-B: extends PR-A with Procedure/Medication/Immunization.
# Seeds 12 concepts (PR-A 9 + CPT 44950, RxNorm 6809, CVX 141), drops
# and recreates omop.drug_exposure (Phase 0 ships a barebones table
# without SERIAL), runs the 10-stage pipeline against 7 PR-B-scoped
# fixture types, and asserts 2/2/2/2/2/1/2 row counts.
run: uv run pytest tests/e2e/test_fhir_to_omop_prb.py -v -m integration
- name: fhir_to_omop PR-C E2E
working-directory: templates
# Phase 1 PR-C: extends PR-A + PR-B with DiagnosticReport + Consent.
# Seeds 15 concepts (PR-B 12 + LOINC 24331-1 lipid panel, OMOP 4055893
# consent-given, 4054745 consent-withdrawn), creates app.consent_decisions
# inline, runs the 12-stage pipeline against 16 fixture FHIR resources,
# and asserts 2/2/2/2/5/1/2 row counts plus 3 type-32856 OBSERVATION
# rows + 2 consent_decisions with co2 recording 'deny'.
run: uv run pytest tests/e2e/test_fhir_to_omop_prc.py -v -m integration
# Phase 3 Plan 1 (T-021A): community-wheel-isolation lane.
#
# Builds the AGPLv3 community wheel ONLY (no commercial source on PYTHONPATH),
# installs into a clean venv, and runs the smoke-marker subset to prove the
# community surface stays self-contained. This is the runtime canary that
# pairs with the static ``import-linter`` contract — together they prevent
# the community wheel from silently growing a dependency on the proprietary
# commercial wheel as Plans 2/3/4A/4B/4C/6 add more commercial-tier code.
community-wheel-isolation:
name: Community wheel isolation smoke (no commercial source)
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.5.11"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Build community wheel only (no commercial source)
working-directory: templates
# Build the parthenon-templates wheel from templates/pyproject.toml.
# The commercial wheel at templates/commercial/ is NOT built here; the
# ``exclude = ["runtime/commercial", ...]`` rule on the community wheel
# makes the wheel self-contained even if commercial source ever lands
# at templates/runtime/commercial/.
run: |
mkdir -p _isolation/dist
uv build --wheel --out-dir _isolation/dist .
- name: Create clean venv with NO commercial source on PYTHONPATH
working-directory: templates
run: |
python -m venv _isolation/venv
_isolation/venv/bin/pip install --quiet --upgrade pip
_isolation/venv/bin/pip install --quiet _isolation/dist/parthenon_templates-*.whl
_isolation/venv/bin/pip install --quiet pytest==8.3.4 pytest-asyncio==0.25.0
- name: Verify runtime.commercial NOT importable from community wheel
working-directory: templates
# Must fail fast if anyone leaks runtime/commercial/ into the community
# wheel build. The venv above has only the community wheel installed
# and the working-directory is templates/ but pytest discovery is run
# from a tmpdir below to avoid picking up ``commercial/`` on the path.
run: |
_isolation/venv/bin/python -c "
import importlib.util
assert importlib.util.find_spec('runtime') is not None, 'community wheel did not install runtime'
spec = importlib.util.find_spec('runtime.commercial')
assert spec is None, f'runtime.commercial leaked into community wheel: {spec}'
from runtime.registry.manifest import NODE_TYPES
assert 'sql' in NODE_TYPES
assert 'csv_reader' in NODE_TYPES
# Commercial-tier node types MUST NOT register in the community NODE_TYPES.
for forbidden in ('x12_837_reader', 'x12_835_reader', 'ncpdp_reader'):
assert forbidden not in NODE_TYPES, f'{forbidden} leaked into community NODE_TYPES'
print('community-wheel-isolation: smoke imports OK')
"
- name: Run smoke-marker tests against the isolated venv
working-directory: templates
# Copy the smoke tests into a tmpdir so pytest doesn't auto-pick up
# the source tree at templates/ (which contains commercial/). The
# smoke set MUST stay tight — it's the canary that catches commercial
# leakage at runtime.
run: |
mkdir -p _isolation/smoke_tests
cp tests/__init__.py _isolation/smoke_tests/__init__.py 2>/dev/null || true
cat > _isolation/smoke_tests/conftest.py <<'EOF'
# Empty conftest so the smoke harness ignores the parent tree's fixtures.
EOF
cat > _isolation/smoke_tests/test_smoke_isolation.py <<'EOF'
import pytest
@pytest.mark.smoke
def test_registry_loads_without_commercial():
from runtime.orchestration.node_registry import NODE_REGISTRY
from runtime.registry.manifest import NODE_TYPES
assert 'sql' in NODE_TYPES
assert 'csv_reader' in NODE_TYPES
assert 'x12_837_reader' not in NODE_TYPES
assert 'sql' in NODE_REGISTRY
EOF
cat > _isolation/smoke_tests/pytest.ini <<'EOF'
[pytest]
markers =
smoke: minimal canary for community-wheel-isolation
EOF
cd _isolation/smoke_tests && \
../venv/bin/pytest -m smoke -v --rootdir=. .
# Slow lane — nightly + manual dispatch only. Runs the throughput perf
# harness (test_fhir_to_omop_throughput, marked -m slow) against a fresh
# Postgres. Reference hardware is 8 vCPU / 32 GB / NVMe per spec Q5; the
# ubuntu-latest GitHub runner is smaller (4 vCPU / 16 GB) so the wall-time
# number from CI will be slower than the reference benchmark — this job
# exists to detect regression, not to set the reference number. To run on
# the reference box, swap runs-on for a self-hosted runner labeled with
# the appropriate hardware tag.
perf:
name: Throughput perf (parthenon-templates, slow lane)
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 60
needs: lint-and-test
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: parthenon
POSTGRES_PASSWORD: parthenon
POSTGRES_DB: parthenon
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U parthenon"
--health-interval 10s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql+psycopg://parthenon:parthenon@localhost:5432/parthenon
PARTHENON_INTERNAL_TOKEN: ci-internal-token
PARTHENON_STORAGE_ROOT: ${{ github.workspace }}/_storage
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.5.11"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Sync dependencies
working-directory: templates
run: uv sync --all-extras
- name: Throughput E2E (1M Observations, -m slow)
working-directory: templates
# Acceptance criterion: 1M Observation resources processed in
# <10 minutes on the reference box (8 vCPU / 32 GB / NVMe). On the
# smaller ubuntu-latest runner expect 1.5-2x that wall time. The
# harness gates on wall time (hard) and peak RSS (soft warning).
# Tee output to a log so the perf summary is captured as a build
# artifact even on success (where stdout is collapsed in the UI).
run: |
mkdir -p _perf
uv run pytest tests/performance/ -v -m slow --maxfail=1 \
2>&1 | tee _perf/throughput.log
- name: Upload perf log
if: always()
uses: actions/upload-artifact@v7
with:
name: throughput-perf-${{ github.run_id }}
path: templates/_perf/throughput.log
if-no-files-found: warn
retention-days: 90
# Live-LLM canary — runs the parthenon_ner_llm pipeline through the real
# OpenAI API (gated by OPENAI_LLM_ENABLED + OPENAI_API_KEY secret) on the
# 100-note benchmark to detect prompt-drift regressions. Mirrors the perf
# job's gating: schedule + workflow_dispatch only. Per-job spend is
# capped via OPENAI_BUDGET_USD ($1) per Phase 2 spec decision Q11.
ner-live:
name: NER live-LLM canary (parthenon-templates, slow lane)
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 30
needs: lint-and-test
env:
OPENAI_LLM_ENABLED: "true"
OPENAI_BUDGET_USD: "1.00"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.5.11"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Sync dependencies
working-directory: templates
run: uv sync --all-extras
- name: Build the 100-note benchmark fixture
working-directory: templates
run: |
uv run python manifests/parthenon_ner_llm/fixtures/synthetic/build_fixtures.py \
--count 100
- name: Live-LLM canary
working-directory: templates
run: |
mkdir -p _ner_live
uv run pytest tests/e2e/test_parthenon_ner_llm.py -v --maxfail=1 \
2>&1 | tee _ner_live/live.log
- name: Upload canary log
if: always()
uses: actions/upload-artifact@v7
with:
name: ner-live-${{ github.run_id }}
path: templates/_ner_live/live.log
if-no-files-found: warn
retention-days: 90
# Phase 2 Plan 3 (T-018b / Q4): NER backend comparison harness.
# Drives LlmBackend / SciSpacyBackend / LlettuceBackend through the
# 100-note gold standard and produces a markdown report with the
# Phase 3 graduation verdict for Llettuce. Slow-lane only.
ner-eval:
name: NER backend comparison (parthenon-templates, slow lane)
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 90
needs: lint-and-test
env:
OPENAI_LLM_ENABLED: "true"
OPENAI_BUDGET_USD: "1.00"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- uses: actions/checkout@v6
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
version: "0.5.11"
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Sync dependencies
working-directory: templates
run: uv sync --all-extras
- name: NER backend comparison
working-directory: templates
env:
PARTHENON_SCISPACY_URL: http://localhost:5101
# SciSpaCy and Llettuce backends are expected to fail on this CI
# runner (no sidecar / no Llettuce package); the runner records
# those errors in the report rather than aborting. The eval lane
# exists to track LlmBackend metrics + the report shape so prompt
# drift surfaces as a regression in concept_match_rate.
run: |
mkdir -p _eval
uv run pytest tests/eval/ -v -m ner_eval 2>&1 | tee _eval/run.log
- name: Llettuce graduation eval (Phase 3 Plan 7 Section C)
working-directory: templates
env:
PARTHENON_SCISPACY_URL: http://localhost:5101
# Phase 3 Plan 7 Task 17: drives the Llettuce graduation eval +
# threshold lane (`mapping_eval`-marked tests under tests/eval/).
# The two graduation tests SKIP cleanly when the curated benchmark
# CSVs are absent (gitignored, generated locally) or when the
# SciSpaCy / Llettuce backends can't load — same pattern as the
# ner_eval lane above. The verdict markdown is uploaded as a 90-
# day artifact below regardless of skip/pass.
run: |
uv run pytest \
tests/eval/test_llettuce_graduation_against_curated_benchmark.py \
-v -m mapping_eval 2>&1 | tee -a _eval/run.log || true
- name: Upload eval report + log
if: always()
uses: actions/upload-artifact@v7
with:
name: ner-eval-${{ github.run_id }}
path: |
templates/_eval/ner_backend_comparison.md
templates/_eval/llettuce_graduation_report.md
templates/_eval/llettuce_graduation_verdict.md
templates/_eval/run.log
if-no-files-found: warn
retention-days: 90