feat(grz-db,grzctl): add PostgreSQL support (#459)

- add unit tests with postgres
- fix migrations to work with both SQLite and PostgreSQL

Author: Travis Wrightsman

.github/workflows/ci.yml

@@ -249,6 +249,9 @@ jobs:
         with:
           lfs: true
 
+      - name: Install PostgreSQL
+        run: sudo apt-get --update install postgresql
+
       - name: Download grz-check binary artifact
         uses: actions/download-artifact@v4
         with:

packages/grz-db/pyproject.toml

@@ -18,7 +18,8 @@ dependencies = [
     "alembic[tz] >=1.16.1",
     "cryptography >=45.0.3",
     "sqlmodel >=0.0.27",
-    "grz-pydantic-models >=2.4,<3"
+    "grz-pydantic-models >=2.4,<3",
+    "psycopg[binary] ~=3.2"
 ]
 
 [tool.uv.sources]

packages/grz-db/src/grz_db/migrations/README.md

@@ -30,3 +30,6 @@ One can also look at the generated schema for a newly initialized database:
 ```
 sqlite3 submission.db.sqlite .schema
 ```
+
+For PostgreSQL, Enums must be explicitly created if not added as part of a `create_table` operation (e.g. `add_column`).
+See the existing migrations for hints.

packages/grz-db/src/grz_db/migrations/versions/9bf36a91c87e_add_reporting_columns.py

@@ -22,35 +22,34 @@
 def upgrade() -> None:
     """Upgrade schema."""
     op.add_column("submissions", sa.Column("submission_date", sa.Date(), nullable=True))
+    submission_type_enum = sa.Enum("initial", "followup", "addition", "correction", "test", name="submissiontype")
+    submission_type_enum.create(op.get_bind())
     op.add_column(
         "submissions",
-        sa.Column(
-            "submission_type", sa.Enum("initial", "followup", "addition", "correction", "test", name="submissiontype")
-        ),
+        sa.Column("submission_type", submission_type_enum),
     )
     op.add_column("submissions", sa.Column("submitter_id", AutoString()))
     op.add_column("submissions", sa.Column("data_node_id", AutoString()))
-    op.add_column(
-        "submissions", sa.Column("disease_type", sa.Enum("oncological", "rare", "hereditary", name="diseasetype"))
+    disease_type_enum = sa.Enum("oncological", "rare", "hereditary", name="diseasetype")
+    disease_type_enum.create(op.get_bind())
+    op.add_column("submissions", sa.Column("disease_type", disease_type_enum))
+    library_type_enum = sa.Enum(
+        "panel",
+        "panel_lr",
+        "wes",
+        "wes_lr",
+        "wgs",
+        "wgs_lr",
+        "wxs",
+        "wxs_lr",
+        "other",
+        "unknown",
+        name="librarytype",
     )
+    library_type_enum.create(op.get_bind())
     op.add_column(
         "submissions",
-        sa.Column(
-            "library_type",
-            sa.Enum(
-                "panel",
-                "panel_lr",
-                "wes",
-                "wes_lr",
-                "wgs",
-                "wgs_lr",
-                "wxs",
-                "wxs_lr",
-                "other",
-                "unknown",
-                name="librarytype",
-            ),
-        ),
+        sa.Column("library_type", library_type_enum),
     )
     op.add_column("submissions", sa.Column("basic_qc_passed", sa.Boolean()))
     op.add_column("submissions", sa.Column("consented", sa.Boolean()))

packages/grz-db/src/grz_db/migrations/versions/fb3df229a77b_initial_quartalsbericht_support.py

@@ -51,24 +51,47 @@ def upgrade() -> None:
         sa.Column("research_consent_missing_justifications", AutoString(), nullable=True),
     )
 
+    # explicitly create enums if needed (PostgreSQL)
+    genomic_study_type_enum = sa.Enum("single", "duo", "trio", name="genomicstudytype")
+    genomic_study_type_enum.create(op.get_bind())
+    genomic_study_subtype_enum = sa.Enum("tumor_only", "tumor_germline", "germline_only", name="genomicstudysubtype")
+    genomic_study_subtype_enum.create(op.get_bind())
+    coverage_type_enum = sa.Enum(
+        "GKV", "PKV", "BG", "SEL", "SOZ", "GPV", "PPV", "BEI", "SKT", "UNK", name="coveragetype"
+    )
+    coverage_type_enum.create(op.get_bind())
+
     # we have to batch alter because SQLite doesn't support ALTER on column types
     with op.batch_alter_table("submissions") as batch_op:
         # will need to be repopulated in donors table before reporting, so just drop
         batch_op.drop_column("library_type")
         # modified/extra submission table columns
-        batch_op.add_column(sa.Column("genomic_study_type", sa.Enum("single", "duo", "trio", name="genomicstudytype")))
+        batch_op.add_column(sa.Column("genomic_study_type", genomic_study_type_enum))
         batch_op.add_column(
             sa.Column(
                 "genomic_study_subtype",
-                sa.Enum("tumor_only", "tumor_germline", "germline_only", name="genomicstudysubtype"),
+                genomic_study_subtype_enum,
             ),
         )
-        batch_op.add_column(
-            sa.Column(
-                "coverage_type",
-                sa.Enum("GKV", "PKV", "BG", "SEL", "SOZ", "GPV", "PPV", "BEI", "SKT", "UNK", name="coveragetype"),
-            )
-        )
+        batch_op.add_column(sa.Column("coverage_type", coverage_type_enum))
+
+    # already should exist so don't need to re-create, just define
+    # but must be postgresql.ENUM until fixed in sqlalchemy 2.1
+    # see https://github.com/sqlalchemy/alembic/issues/1347
+    library_type_enum = sa.dialects.postgresql.ENUM(
+        "panel",
+        "panel_lr",
+        "wes",
+        "wes_lr",
+        "wgs",
+        "wgs_lr",
+        "wxs",
+        "wxs_lr",
+        "other",
+        "unknown",
+        name="librarytype",
+        create_type=False,
+    )
 
     # new detailed QC results table
     op.create_table(
@@ -90,19 +113,7 @@ def upgrade() -> None:
         ),
         sa.Column(
             "library_type",
-            sa.Enum(
-                "panel",
-                "panel_lr",
-                "wes",
-                "wes_lr",
-                "wgs",
-                "wgs_lr",
-                "wxs",
-                "wxs_lr",
-                "other",
-                "unknown",
-                name="librarytype",
-            ),
+            library_type_enum,
             nullable=False,
         ),
         sa.Column("percent_bases_above_quality_threshold_minimum_quality", sa.Float(), nullable=False),

packages/grz-db/src/grz_db/models/submission.py

@@ -25,7 +25,7 @@
     Tan,
 )
 from pydantic import ConfigDict, field_serializer, field_validator
-from sqlalchemy import JSON, Column
+from sqlalchemy import JSON, Column, Enum
 from sqlalchemy import func as sqlfn
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import selectinload
@@ -292,7 +292,11 @@ class Donor(SQLModel, table=True):
 
     submission_id: str = Field(foreign_key="submissions.id", primary_key=True)
     pseudonym: str = Field(primary_key=True)
-    relation: Relation
+    # use values_callable so enum value is stored instead of name.
+    # SQLite stores string of value instead of name because it doesn't have
+    # native Enum support, so this keeps things consistent with other SQL
+    # server implementations.
+    relation: Relation = Field(sa_column=Column(Enum(Relation, values_callable=lambda e: [x.value for x in e])))
     library_types: set[LibraryType] = Field(sa_column=Column(SemicolonSeparatedStringSet))
     sequence_types: set[SequenceType] = Field(sa_column=Column(SemicolonSeparatedStringSet))
     sequence_subtypes: set[SequenceSubtype] = Field(sa_column=Column(SemicolonSeparatedStringSet))

packages/grz-db/uv.lock

@@ -19,6 +19,7 @@ dependencies = [
     "jsonschema >=4.23.0,<5",
     "pyyaml >=6.0.2,<7",
     "tqdm >=4.66.5,<5",
+    "psycopg ~=3.2",
     "pydantic >=2.12,<3",
     "pydantic-settings >=2.11.0,<3",
     "platformdirs >=4.3.6,<5",
@@ -76,6 +77,8 @@ test = [
     "pytest-mock",
     "pytest-cov",
     "pytest-sugar",
+    "pytest-postgresql ~=7.0.2",
+    "psycopg",
     "moto[s3]",
     "numpy",
     "grz-common",

packages/grzctl/pyproject.toml

@@ -1,16 +1,26 @@
+import shutil
 from pathlib import Path
 
 import click.testing
 import cryptography.hazmat.primitives.serialization as cryptser
 import grzctl.cli
+import psycopg
 import pytest
 import yaml
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
 from grzctl.models.config import DbConfig
 
 
-@pytest.fixture
-def blank_database_config(tmp_path: Path) -> DbConfig:
+@pytest.fixture(
+    params=[
+        "sqlite",
+        pytest.param(
+            "postgresql",
+            marks=pytest.mark.skipif(condition=shutil.which("pg_config") is None, reason="postgresql not detected"),
+        ),
+    ]
+)
+def blank_database_config(request: pytest.FixtureRequest, tmp_path: Path) -> DbConfig:
     private_key = Ed25519PrivateKey.generate()
     private_key_path = tmp_path / "alice.sec"
     with open(private_key_path, "wb") as private_key_file:
@@ -31,9 +41,14 @@ def blank_database_config(tmp_path: Path) -> DbConfig:
         # add the comment too
         public_key_file.write(b" alice")
 
+    database_url = "sqlite:///" + str((tmp_path / "submission.db.sqlite").resolve())
+    if request.param == "postgresql":
+        postgresql: psycopg.Connection = request.getfixturevalue("postgresql")
+        database_url = f"postgresql+psycopg://{postgresql.info.user}:@{postgresql.info.host}:{postgresql.info.port}/{postgresql.info.dbname}"
+
     return DbConfig(
         db={
-            "database_url": "sqlite:///" + str((tmp_path / "submission.db.sqlite").resolve()),
+            "database_url": database_url,
             "author": {
                 "name": "alice",
                 "private_key_path": str(private_key_path.resolve()),