feat(grz-cli, grz-db): require & record QC workflow version; log grzctl runtime version (closes #532) (#561)

## Summary
Require a QC workflow version for new `populate-qc` runs (CLI flag or
env var), persist `qc_workflow_version` on Detailed QC results, and
record the `grzctl` dependency runtime versions on submission state
transitions for auditability.

## What changed
- CLI
- `populate-qc` now requires `--qc-workflow-version` and also reads
`GRZCTL_QC_WORKFLOW_VERSION`.
See
[packages/grzctl/src/grzctl/commands/db/cli.py](packages/grzctl/src/grzctl/commands/db/cli.py).
- Model / DB
- Added `qc_workflow_version` to `DetailedQCResult` (nullable column for
backward compatibility).
  - Alembic migration added to create the column.
- Tests
- Updated and added tests in
[packages/grzctl/tests/cli/test_db.py](packages/grzctl/tests/cli/test_db.py):
- Updated existing `test_populate_qc` to pass `--qc-workflow-version`.
- Added tests for flag usage, env var usage, and missing-version
failure.
- Version logging
- Cache and store `grzctl` dependy runtime versions with submission
state logs so historical logs are traceable to the runtime that wrote
them.

## DB / upgrade
Run DB migrations before using the new CLI:

```bash
# Using the CLI helper
grzctl db upgrade

# or directly with alembic (repo-specific)
alembic upgrade head
```

The migration adds a nullable qc_workflow_version column so existing
rows remain NULL.
  
Closes #532

---------

Co-authored-by: Florian R. Hölzlwimmer <Hoeze@users.noreply.github.com>

Author: jblesch

packages/grz-db/src/grz_db/migrations/versions/c5f6a8b9d2e1_add_grzctl_and_qc_workflow_versions.py

@@ -0,0 +1,35 @@
+"""add grzctl_version and qc_workflow_version columns
+
+Revision ID: c5f6a8b9d2e1
+Revises: 834bd50b8734
+Create Date: 2026-05-06 00:00:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "c5f6a8b9d2e1"
+down_revision: str | Sequence[str] | None = "834bd50b8734"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Add grzctl_versions to submission_states
+    op.add_column(
+        "submission_states",
+        sa.Column("grzctl_versions", sa.JSON(), nullable=True),
+    )
+
+    # Add qc_workflow_version to detailed_qc_results
+    op.add_column(
+        "detailed_qc_results",
+        sa.Column("qc_workflow_version", sa.String(length=64), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    raise RuntimeError("Downgrades not supported.")

packages/grz-db/src/grz_db/models/submission/__init__.py

@@ -257,6 +257,11 @@ class SubmissionStateLogBase(SQLModel):
 
     state: SubmissionStateEnum
     data: dict[str, Any] | None = Field(default=None, sa_column=Column(JSON))
+    grzctl_versions: dict[str, str] | None = Field(
+        default=None,
+        description="grzctl versions that created this state log (nullable for backward compatibility with old state logs)",
+        sa_column=Column(JSON),
+    )
     timestamp: datetime.datetime = Field(
         default_factory=lambda: datetime.datetime.now(datetime.UTC),
         sa_column=Column(DateTime(timezone=True), nullable=False),
@@ -465,6 +470,11 @@ class DetailedQCResult(SQLModel, table=True):
     targeted_regions_above_min_coverage: float
     targeted_regions_above_min_coverage_passed_qc: bool
     targeted_regions_above_min_coverage_percent_deviation: float
+    qc_workflow_version: str | None = Field(
+        default=None,
+        sa_column=Column(sa.String(length=64), nullable=True),
+        description="QC workflow version (nullable for backward compatibility with old results)",
+    )
 
     model_config = ConfigDict(  # type: ignore
         populate_by_name=True,
@@ -737,6 +747,7 @@ def update_submission_state(
         submission_id: str,
         state: SubmissionStateEnum,
         data: dict | None = None,
+        grzctl_versions: dict[str, str] | None = None,
     ) -> SubmissionStateLog:
         """
         Updates a submission's state to the specified state.
@@ -745,6 +756,7 @@ def update_submission_state(
             submission_id: Submission ID of the submission to update.
             state: New state of the submission.
             data: Optional data to attach to the update.
+            grzctl_versions: Optional dictionary of grzctl dependency versions.
 
         Returns:
             An instance of SubmissionStateLog.
@@ -757,7 +769,11 @@ def update_submission_state(
                 raise ValueError("No author defined")
 
             state_log_payload = SubmissionStateLogPayload(
-                submission_id=submission_id, author_name=self._author.name, state=state, data=data
+                submission_id=submission_id,
+                author_name=self._author.name,
+                state=state,
+                data=data,
+                grzctl_versions=grzctl_versions,
             )
             signature = state_log_payload.sign(self._author.private_key())
 

packages/grzctl/src/grzctl/__init__.py

@@ -2,4 +2,23 @@
 GRZ Control CLI for GRZ administrators.
 """
 
-__version__ = "1.4.0"
+from importlib.metadata import PackageNotFoundError, version
+
+__version__ = "1.4.0"  # This version is managed by release-please
+
+
+def get_versions() -> dict[str, str | None]:
+    def get_version(package_name: str) -> str | None:
+        try:
+            return version(package_name)
+        except PackageNotFoundError:
+            return None
+
+    return {
+        "grzctl": get_version("grzctl"),
+        "grz-cli": get_version("grz-cli"),
+        "grz-common": get_version("grz-common"),
+        "grz-db": get_version("grz-db"),
+        "grz-pydantic-models": get_version("grz-pydantic-models"),
+        "grz-check": get_version("grz-check"),
+    }

packages/grzctl/src/grzctl/cli.py

@@ -3,11 +3,7 @@
 """
 
 import logging.config
-import shutil
-import subprocess
-from importlib.metadata import version
 from pathlib import Path
-from textwrap import dedent
 
 import click
 import grz_common.cli as grzcli
@@ -16,6 +12,7 @@
 from grz_common.cli.dump_config import dump_config
 from grz_common.logging import setup_cli_logging
 
+from . import get_versions
 from .commands.archive import archive
 from .commands.clean import clean
 from .commands.consent import consent
@@ -45,26 +42,16 @@ def build_cli():
     """
     Factory for building the CLI application.
     """
+    versions = get_versions()
 
     @click.group(
         cls=OrderedGroup,
         help="GRZ Control CLI for GRZ administrators.",
     )
     @click.version_option(
-        version=version("grzctl"),
+        version=versions["grzctl"],
         prog_name="grzctl",
-        message=dedent(f"""\
-        %(prog)s v%(version)s
-        grz-cli v{version("grz-cli")}
-        grz-common v{version("grz-common")}
-        grz-db v{version("grz-db")}
-        grz-pydantic-models v{version("grz-pydantic-models")}
-        """)
-        + (
-            subprocess.run(["grz-check", "--version"], capture_output=True, text=True).stdout.strip()  # noqa: S607
-            if shutil.which("grz-check") is not None
-            else ""
-        ),
+        message="\n".join(f"{k} {'v' + v if v else 'unknown'}" for k, v in versions.items()),
     )
     @click.option("--log-file", metavar="FILE", type=str, help="Path to log file")
     @click.option(

packages/grzctl/src/grzctl/commands/db/cli.py

@@ -58,6 +58,7 @@
 from pydantic import Field
 from tqdm.auto import tqdm
 
+from ... import get_versions
 from ...models.config import DbConfig, ListConfig
 from .. import limit
 from . import SignatureStatus, _verify_signature
@@ -503,7 +504,12 @@ def update(ctx: click.Context, submission_id: str, state_str: str, data_json: st
             console_err.print(f"[yellow]Not modifying state of errored submission '{submission_id}'.[/yellow]")
             ctx.exit()
 
-        new_state_log = db_service.update_submission_state(submission_id, state_enum, parsed_data)
+        new_state_log = db_service.update_submission_state(
+            submission_id,
+            state_enum,
+            parsed_data,
+            grzctl_versions={k: (v if v is not None else "unknown") for k, v in get_versions().items()},
+        )
         console_err.print(
             f"[green]Submission '{submission_id}' updated to state '{new_state_log.state.value}'. Log ID: {new_state_log.id}[/green]"
         )
@@ -757,13 +763,20 @@ class QCReportRow(StrictBaseModel):
 @submission.command()
 @click.argument("submission_id", type=str)
 @click.argument("report_csv_path", metavar="path/to/report.csv", type=grzcli.FILE_R_E)
+@click.option(
+    "--qc-workflow-version",
+    type=str,
+    required=True,
+    envvar="GRZCTL_QC_WORKFLOW_VERSION",
+    help="QC workflow version to store with detailed QC results. Can also be provided via GRZCTL_QC_WORKFLOW_VERSION.",
+)
 @click.option(
     "--confirm/--no-confirm",
     default=True,
     help="Whether to confirm changes before committing to database. (Default: confirm)",
 )
 @click.pass_context
-def populate_qc(ctx: click.Context, submission_id: str, report_csv_path: str, confirm: bool):
+def populate_qc(ctx: click.Context, submission_id: str, report_csv_path: str, qc_workflow_version: str, confirm: bool):
     """Populate the submission database from a detailed QC pipeline report."""
     db = ctx.obj["db_url"]
     db_service = get_submission_db_instance(db, author=ctx.obj["author"])
@@ -800,6 +813,7 @@ def populate_qc(ctx: click.Context, submission_id: str, report_csv_path: str, co
                 targeted_regions_above_min_coverage_passed_qc=report.targeted_regions_above_min_coverage_qc_status
                 == QCStatus.PASS,
                 targeted_regions_above_min_coverage_percent_deviation=report.targeted_regions_above_min_coverage_deviation,
+                qc_workflow_version=qc_workflow_version,
             )
         )
     table = rich.table.Table(
@@ -900,7 +914,9 @@ def show(ctx: click.Context, submission_id: str, output_json: bool):
             signature_status, verifying_key_comment = _verify_signature(
                 ctx.obj["public_keys"], state_log.author_name, state_log
             )
-            state_dict = state_log.model_dump(mode="json", include={"id", "timestamp", "state", "data"})
+            state_dict = state_log.model_dump(
+                mode="json", include={"id", "timestamp", "state", "data", "grzctl_versions"}
+            )
             state_dict["data_steward"] = state_log.author_name
             state_dict["data_steward_signature"] = signature_status
             state_dict["signature_key_comment"] = verifying_key_comment
@@ -936,11 +952,12 @@ def show(ctx: click.Context, submission_id: str, output_json: bool):
 
     renderables: list[rich.console.RenderableType] = [rich.padding.Padding(attribute_table, (1, 0))]
     if submission.states:
-        state_table = rich.table.Table(title="State History")
+        state_table = rich.table.Table(title="State History", show_header=True)
         state_table.add_column("Log ID", style="dim", width=12)
         state_table.add_column("Timestamp (UTC)", style="yellow")
         state_table.add_column("State", style="green")
         state_table.add_column("Data", style="cyan", overflow="ellipsis")
+        state_table.add_column("Dependency Versions", style="blue")
         state_table.add_column("Data Steward", style="magenta")
         state_table.add_column("Signature Status")
 
@@ -960,6 +977,7 @@ def show(ctx: click.Context, submission_id: str, output_json: bool):
                 state_log.timestamp.isoformat(),
                 state_str,
                 data_str,
+                json.dumps(state_log.grzctl_versions) if state_log.grzctl_versions else _TEXT_MISSING,
                 data_steward_str,
                 signature_status_str,
             )

packages/grzctl/src/grzctl/commands/db/sync.py

@@ -8,6 +8,8 @@
 from grz_db.models.author import Author
 from grz_db.models.submission import SubmissionDb, SubmissionStateEnum
 
+from ... import get_versions
+
 log = logging.getLogger(__name__)
 
 
@@ -84,4 +86,8 @@ def _determine_target_state(s3_state: InboxSubmissionState) -> SubmissionStateEn
 
 
 def _update_state(db: SubmissionDb, submission_id: str, state: SubmissionStateEnum, author: Author):
-    db.update_submission_state(submission_id, state)
+    db.update_submission_state(
+        submission_id,
+        state,
+        grzctl_versions={k: (v if v is not None else "unknown") for k, v in get_versions().items()},
+    )

packages/grzctl/src/grzctl/dbcontext.py

@@ -8,6 +8,7 @@
 from grz_db.models.submission import SubmissionDb, SubmissionStateEnum
 from pydantic import ValidationError
 
+from . import get_versions
 from .commands.db.cli import get_submission_db_instance
 from .models.config import DbConfig
 
@@ -79,6 +80,12 @@ def __init__(
         self.enabled = enabled
         self.db: SubmissionDb | None = None
 
+    @cached_property
+    def grzctl_versions(self) -> dict[str, str]:
+        """Get version information."""
+        versions = get_versions()
+        return {k: v or "unknown" for k, v in versions.items()}
+
     @cached_property
     def expected_prior_states(self) -> set[SubmissionStateEnum | None]:
         # determine expected prior state based on order of enums
@@ -106,7 +113,7 @@ def __enter__(self):
             self._check_prerequisites()
 
             log.debug(f"Updating submission {self.submission_id} state to {self.start_state.name}")
-            self.db.update_submission_state(self.submission_id, self.start_state)
+            self.db.update_submission_state(self.submission_id, self.start_state, grzctl_versions=self.grzctl_versions)
 
         except SubmissionNotFoundError:
             raise
@@ -127,7 +134,9 @@ def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException
             error_state = SubmissionStateEnum.ERROR
             log.error(f"Operation failed for {self.submission_id}. Updating DB to {error_state.name}.")
             try:
-                self.db.update_submission_state(self.submission_id, error_state, data={"error": error_message})
+                self.db.update_submission_state(
+                    self.submission_id, error_state, data={"error": error_message}, grzctl_versions=self.grzctl_versions
+                )
             except Exception as db_exc:
                 log.error(f"Failed to write error state to DB: {db_exc}")
 
@@ -136,7 +145,9 @@ def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException
         else:
             log.info(f"Operation successful. Updating DB to {self.end_state.name}.")
             try:
-                self.db.update_submission_state(self.submission_id, self.end_state)
+                self.db.update_submission_state(
+                    self.submission_id, self.end_state, grzctl_versions=self.grzctl_versions
+                )
             except Exception as db_exc:
                 log.error(f"Failed to write success state to DB: {db_exc}")