fix(migration): give each job its own credential tracker (#1295 follow-up, #793 collision) (#1325)

Follow-up to #1324, which routed a blocked run to a `provider-credentials` tracker
so a drained provider account would stop being filed as a Langflow migration
failure. That tracker was **shared by both jobs**, which made it a fresh instance
of the collision PR #793 found on run #101: the two jobs of `migration-test.yml`
run in parallel, and the one that goes green closes whatever open issue carries the
label — including one the other job filed a minute earlier on a failure that is
still live. Reaching the provider from the compose job says nothing about what the
API job saw, and vice versa.

- The label is now per job (`provider-credentials-api` / `provider-credentials-compose`,
  both self-provisioned on first use), composed in one place by `issue_label(job)`.
- `--job` is REQUIRED on the CLI with `choices=("api", "compose")`: a default would
  hand both jobs one tracker, and a typo would open a third nobody watches.
- Each `Close issue on success` step closes only its own job's credential label. The
  `migration-test` half keeps its current shared behaviour — that is #793's scope,
  not this change's.

The cost is that a drained account files two issues, one per job. That is noise
which self-clears on the next good run, where the alternative silently closes a
live blocker.

Validation: 69 tests in the Python unit lane (3 new), including a structural guard
that pins an ABSENCE — neither job's half of the workflow may reference the other's
credential label or pass the other's `--job` — because these two jobs are
near-copies and copy-paste between them is exactly how a per-job label degrades
back into a shared one. All 4 mutations attempted were killed: a shared label, an
unvalidated job name, the compose job closing the API job's tracker, and the compose
pre-flight claiming to be the API job. Probed live against the still-drained key:
both jobs exit 1 with their own label.

Co-authored-by: Rafael <rafael@oriontech.me>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: 3 people

.github/workflows/migration-test.yml

@@ -71,7 +71,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           python tests/github-workflows/migration/provider_credentials.py \
-            --probe --phase "pre-flight/pip"
+            --probe --phase "pre-flight/pip" --job api
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
@@ -324,9 +324,14 @@ jobs:
             // old version said "Migration test passed" on whatever open
             // `migration-test` issue it found, which is how a billing outage filed
             // there would have been closed as a fixed migration bug.
+            //
+            // The credential label is THIS job's own (`-api`), never the compose job's:
+            // the two run in parallel, and closing the other one's issue is the
+            // collision PR #793 found on run #101. The `migration-test` half keeps its
+            // existing (shared) behaviour — that is #793's scope, not this PR's.
             const CLOSING = {
               'migration-test': 'Migration test passed. Closing this issue.',
-              'provider-credentials':
+              'provider-credentials-api':
                 'This run reached the model provider and executed the witness flow, so the ' +
                 'credential that blocked it is usable again. Closing.',
             };
@@ -405,7 +410,7 @@ jobs:
                   repo: context.repo.repo,
                   name: label,
                   color: 'd93f0b',
-                  description: 'Provider credential/billing state blocked a scheduled run (no product signal)',
+                  description: `Provider credential/billing state blocked the migration test's ${credential.job} job (no product signal)`,
                 });
               } catch (e) {
                 console.log(`Label ${label} already exists or could not be created:`, e.message);
@@ -492,7 +497,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           python3 tests/github-workflows/migration/provider_credentials.py \
-            --probe --phase "pre-flight/compose"
+            --probe --phase "pre-flight/compose" --job compose
 
       - name: Generate ephemeral SECRET_KEY
         run: |
@@ -687,7 +692,7 @@ jobs:
             # deliberately does not change the exit code below.
             python3 tests/github-workflows/migration/provider_credentials.py \
               --classify-file /tmp/run-source.json --status "$HTTP" \
-              --phase "compose/source-execute" || true
+              --phase "compose/source-execute" --job compose || true
             exit 1
           fi
           echo "Source flow execution OK"
@@ -763,7 +768,7 @@ jobs:
             # here would have filed exactly that against Langflow (#1295).
             python3 tests/github-workflows/migration/provider_credentials.py \
               --classify-file /tmp/run-target.json --status "$HTTP" \
-              --phase "compose/target-execute" || true
+              --phase "compose/target-execute" --job compose || true
             exit 1
           fi
           if grep -iE 'api[ _-]?key.{0,30}(required|missing|not[ _]?found)|fernet|cannot[ _]decrypt' /tmp/run-target.json; then
@@ -849,10 +854,11 @@ jobs:
         uses: actions/github-script@v9
         with:
           script: |
-            // Two trackers, same reasoning as the API job above (#1295).
+            // Two trackers, same reasoning as the API job above (#1295) — and the
+            // credential label is this job's own (`-compose`), for the #793 reason.
             const CLOSING = {
               'migration-test': 'docker-compose migration test passed. Closing.',
-              'provider-credentials':
+              'provider-credentials-compose':
                 'This run reached the model provider and executed the witness flow, so the ' +
                 'credential that blocked it is usable again. Closing.',
             };
@@ -914,7 +920,7 @@ jobs:
                   repo: context.repo.repo,
                   name: label,
                   color: 'd93f0b',
-                  description: 'Provider credential/billing state blocked a scheduled run (no product signal)',
+                  description: `Provider credential/billing state blocked the migration test's ${credential.job} job (no product signal)`,
                 });
               } catch (e) {
                 console.log(`Label ${label} already exists or could not be created:`, e.message);

tests/github-workflows/migration/README.md

@@ -50,7 +50,9 @@ On failure, the workflow opens or updates an issue in the repository, including
 | Verdict | Tracker | Meaning |
 |---|---|---|
 | `FAILED` | `migration-test` | Something about the migration is broken — a claim about Langflow. |
-| `BLOCKED (provider credentials)` | `provider-credentials` | The witness flow never reached the model provider, so **nothing about the migration was measured**. Not a claim about Langflow; fix the account or the key. |
+| `BLOCKED (provider credentials)` | `provider-credentials-api` / `provider-credentials-compose` | The witness flow never reached the model provider, so **nothing about the migration was measured**. Not a claim about Langflow; fix the account or the key. |
+
+The credential tracker is **per job**, and each job closes only its own. The two jobs run in parallel, so a shared label lets the one that goes green close the issue the other just filed — the collision PR #793 found on run #101, where the API job's pass closed the compose job's live failure. The cost is that a drained account files two issues; they self-clear on the next good run, whereas the alternative silently closes a live blocker.
 
 The split exists because both are red and only one is about the product. A billing outage filed under `migration-test` gets closed by the next green run with *"Migration test passed"*, leaving a migration bug in the history that never existed. A green run closes **both** trackers, each with wording about what it actually proved.
 

tests/github-workflows/migration/provider_credentials.py

@@ -139,7 +139,26 @@
 # The label the report is routed to when the verdict is blocking. Kept off
 # `migration-test` so a later green run cannot close it with "Migration test
 # passed", which would read as a migration bug that never existed.
-ISSUE_LABEL = "provider-credentials"
+#
+# **Per job, not shared** — the collision PR #793 found on run #101: the two jobs of
+# this workflow run in parallel and used one label, so the job that went green closed
+# the issue the other had just filed (its `migration-test` failure survived as
+# "resolved"). A single `provider-credentials` label would have been a fresh instance
+# of that same bug: the compose job reaching the provider says nothing about what the
+# API job saw a minute earlier, and vice versa. The cost is that a drained account
+# files two issues — one per job — which is noise, but noise that self-clears on the
+# next good run, where the alternative silently closes a live blocker.
+ISSUE_LABEL_BASE = "provider-credentials"
+
+# The two jobs of `migration-test.yml`, spelled as they appear in `--job`. A closed
+# set so a typo in the workflow cannot quietly open a third tracker nobody watches.
+JOBS = ("api", "compose")
+
+
+def issue_label(job: str) -> str:
+    if job not in JOBS:
+        raise ValueError(f"unknown job {job!r} — expected one of {JOBS}")
+    return f"{ISSUE_LABEL_BASE}-{job}"
 
 
 def classify(status: Optional[int], body: str) -> Tuple[str, str]:
@@ -239,18 +258,23 @@ def _send(request: urllib.request.Request, timeout: int) -> Tuple[int, str]:
         return err.code, err.read().decode("utf-8", "replace")
 
 
-def marker_payload(verdict: str, reason: str, phase: str) -> dict:
+def marker_payload(verdict: str, reason: str, phase: str, job: str) -> dict:
     return {
         "verdict": verdict,
         "reason": reason,
         "phase": phase,
+        "job": job,
         "title": _TITLES.get(verdict, ""),
         "action": _ACTIONS.get(verdict, ""),
-        "label": ISSUE_LABEL,
+        # The workflow's issue step reads this rather than composing it, so the
+        # per-job split lives in one place.
+        "label": issue_label(job),
     }
 
 
-def write_marker(verdict: str, reason: str, phase: str, path: str = MARKER_FILE) -> Optional[str]:
+def write_marker(
+    verdict: str, reason: str, phase: str, job: str, path: str = MARKER_FILE
+) -> Optional[str]:
     """Record a blocking verdict for the workflow's issue-routing step.
 
     Only `BLOCKING` verdicts are written: the file's **presence** is the signal, and
@@ -260,11 +284,13 @@ def write_marker(verdict: str, reason: str, phase: str, path: str = MARKER_FILE)
     if verdict not in BLOCKING:
         return None
     with open(path, "w") as handle:
-        json.dump(marker_payload(verdict, reason, phase), handle, indent=2)
+        json.dump(marker_payload(verdict, reason, phase, job), handle, indent=2)
     return path
 
 
-def step_status(detail: str, phase: str, marker: Optional[str] = None) -> str:
+def step_status(
+    detail: str, phase: str, job: str = "api", marker: Optional[str] = None
+) -> str:
     """The status a failed flow execution should record: `"blocked"` or `"fail"`.
 
     The mid-run half of #1295, shared by `setup_latest.py` and
@@ -273,12 +299,15 @@ def step_status(detail: str, phase: str, marker: Optional[str] = None) -> str:
     string, where the provider's own error is quoted inside Langflow's — hence no
     status to pass. A blocking verdict is announced and recorded on the way out; any
     other failure keeps its original `"fail"` and this function stays silent.
+
+    `job` defaults to `"api"` because both callers are steps of that job — the compose
+    job has no Python and classifies through the CLI.
     """
     verdict, reason = classify(None, detail)
     if verdict not in BLOCKING:
         return "fail"
     print(announce(verdict, reason, phase))
-    write_marker(verdict, reason, phase, MARKER_FILE if marker is None else marker)
+    write_marker(verdict, reason, phase, job, MARKER_FILE if marker is None else marker)
     return "blocked"
 
 
@@ -310,6 +339,10 @@ def main(argv=None) -> int:
     parser.add_argument("--status", type=int, default=None, help="HTTP status for --classify-file")
     parser.add_argument("--model", default=DEFAULT_MODEL)
     parser.add_argument("--phase", default="pre-flight", help="where the verdict was reached")
+    # Required, with no default: which job filed this decides which issue a green run
+    # is allowed to close, and a default would hand both jobs the same tracker — the
+    # #793 collision (see ISSUE_LABEL_BASE).
+    parser.add_argument("--job", required=True, choices=JOBS, help="which job reached the verdict")
     parser.add_argument("--marker", default=MARKER_FILE)
     args = parser.parse_args(argv)
 
@@ -329,7 +362,7 @@ def main(argv=None) -> int:
         verdict, reason = classify(args.status, body)
 
     print(announce(verdict, reason, args.phase))
-    written = write_marker(verdict, reason, args.phase, args.marker)
+    written = write_marker(verdict, reason, args.phase, args.job, args.marker)
     if written:
         print(f"Recorded credential verdict in {written}")
 

tests/github-workflows/migration/setup_latest.py

@@ -112,7 +112,7 @@ def main():
     # the common case before this job installs anything; this covers the account
     # draining mid-run, and records `blocked` — a state the report renders apart from
     # FAILED, and the workflow routes to the credentials tracker, not `migration-test`.
-    status = "pass" if success else credentials.step_status(detail, "latest/execute_flow")
+    status = "pass" if success else credentials.step_status(detail, "latest/execute_flow", job="api")
     print(f"   {status.upper()}: {detail[:120]}")
     phase["steps"]["execute_flow"] = {"status": status, "detail": detail}
 

tests/github-workflows/migration/test_provider_credentials.py

@@ -258,21 +258,50 @@ def send(request, timeout):
 
 def test_marker_is_written_only_for_blocking_verdicts(tmp_path):
     path = tmp_path / "verdict.json"
-    assert pc.write_marker(pc.LIVE, "fine", "pre-flight", str(path)) is None
-    assert pc.write_marker(pc.INCONCLUSIVE, "unclear", "pre-flight", str(path)) is None
+    assert pc.write_marker(pc.LIVE, "fine", "pre-flight", "api", str(path)) is None
+    assert pc.write_marker(pc.INCONCLUSIVE, "unclear", "pre-flight", "api", str(path)) is None
     assert not path.exists(), "absence of the marker is what keeps the default routing"
 
-    assert pc.write_marker(pc.BILLING, "drained", "pre-flight", str(path)) == str(path)
+    assert pc.write_marker(pc.BILLING, "drained", "pre-flight", "api", str(path)) == str(path)
     payload = json.loads(path.read_text())
     assert payload["verdict"] == pc.BILLING
-    assert payload["label"] == pc.ISSUE_LABEL
+    assert payload["label"] == "provider-credentials-api"
     assert payload["label"] != "migration-test", (
         "routing a billing state to the migration tracker is #1295 — a later green "
         "run would close it with 'Migration test passed'"
     )
+    assert payload["job"] == "api"
     assert payload["title"] and payload["action"]
 
 
+def test_the_two_jobs_get_two_different_trackers(tmp_path):
+    """The #793 collision, which a single shared label would have reintroduced.
+
+    Both jobs run in parallel; the one that goes green must not be able to close the
+    issue the other just filed, because reaching the provider from the compose job
+    says nothing about what the API job saw a minute earlier.
+    """
+    labels = {job: pc.issue_label(job) for job in pc.JOBS}
+
+    assert labels["api"] != labels["compose"]
+    assert set(labels.values()) == {"provider-credentials-api", "provider-credentials-compose"}
+    assert "migration-test" not in labels.values()
+
+    for job, label in labels.items():
+        path = tmp_path / f"{job}.json"
+        pc.write_marker(pc.BILLING, "drained", f"pre-flight/{job}", job, str(path))
+        assert json.loads(path.read_text())["label"] == label
+
+
+def test_an_unknown_job_is_refused_rather_than_given_a_tracker():
+    # A typo in the workflow must not open a third tracker nobody watches.
+    import pytest
+
+    for job in ("", "API", "pip", "compose-2", None):
+        with pytest.raises(ValueError):
+            pc.issue_label(job)
+
+
 def test_a_blocking_announcement_states_there_was_no_migration_signal():
     line = pc.announce(pc.BILLING, "drained", "pre-flight")
     assert line.startswith("::error::")
@@ -316,6 +345,39 @@ def test_both_api_scripts_route_their_execution_failure_through_step_status():
         )
 
 
+def test_neither_job_can_touch_the_other_jobs_credential_tracker():
+    """Structural, and it pins an ABSENCE — which is the realistic regression here.
+
+    The two jobs of `migration-test.yml` are near-copies of each other, so the way a
+    per-job label degrades back into a shared one is a copy-paste between them (that
+    is how the collision PR #793 found reached both halves in the first place). Text,
+    not YAML parsing: the PR-validation lane installs `pytest` and nothing else, so
+    `pyyaml` is not available here.
+
+    It cannot tell whether the routing is *correct* — the tests above do that.
+    """
+    import pathlib
+
+    workflow = (
+        pathlib.Path(__file__).resolve().parents[3] / ".github/workflows/migration-test.yml"
+    ).read_text()
+
+    marker = "\n  migration-test-compose:"
+    assert marker in workflow, "the compose job header moved — this guard cannot split the file"
+    api_half, compose_half = workflow.split(marker, 1)
+
+    for half, own, other in (
+        (api_half, "api", "compose"),
+        (compose_half, "compose", "api"),
+    ):
+        assert f"provider-credentials-{other}" not in half, (
+            f"the {own} job references the {other} job's credential tracker — a green run "
+            f"would close an issue about a failure it never observed (#793)"
+        )
+        assert f"--job {own}" in half, f"the {own} job does not tell the classifier which job it is"
+        assert f"--job {other}" not in half
+
+
 def test_any_other_failure_still_records_fail_and_says_nothing(tmp_path, capsys):
     marker = tmp_path / "verdict.json"
 
@@ -340,7 +402,7 @@ def test_probe_mode_exits_nonzero_on_billing(monkeypatch, tmp_path, capsys):
     monkeypatch.setattr(pc, "_send", _transport(429, RUN_124_BODY))
     marker = tmp_path / "verdict.json"
 
-    code = pc.main(["--probe", "--marker", str(marker)])
+    code = pc.main(["--probe", "--job", "api", "--marker", str(marker)])
 
     assert code == 1, "a blocking pre-flight must stop the job before it installs anything"
     assert "::error::" in capsys.readouterr().out
@@ -351,7 +413,7 @@ def test_probe_mode_exits_zero_when_inconclusive(monkeypatch, tmp_path, capsys):
     monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
     monkeypatch.setattr(pc, "_send", _transport(503, "bad gateway"))
 
-    code = pc.main(["--probe", "--marker", str(tmp_path / "verdict.json")])
+    code = pc.main(["--probe", "--job", "api", "--marker", str(tmp_path / "verdict.json")])
 
     assert code == 0, "fail-open: the run itself is the authoritative verdict"
     assert "::warning::" in capsys.readouterr().out
@@ -370,6 +432,8 @@ def test_classify_file_mode_attributes_without_changing_the_exit_code(tmp_path,
             "500",
             "--phase",
             "compose-source",
+            "--job",
+            "compose",
             "--marker",
             str(marker),
         ]
@@ -383,7 +447,7 @@ def test_classify_file_mode_attributes_without_changing_the_exit_code(tmp_path,
 
 
 def test_an_unreadable_body_file_never_masks_the_real_failure(tmp_path, capsys):
-    code = pc.main(["--classify-file", str(tmp_path / "absent.json")])
+    code = pc.main(["--classify-file", str(tmp_path / "absent.json"), "--job", "compose"])
     assert code == 0
     out = capsys.readouterr().out
     assert "::warning::" in out and "no credential verdict" in out
@@ -393,6 +457,11 @@ def test_probe_and_classify_file_are_mutually_exclusive(tmp_path):
     import pytest
 
     with pytest.raises(SystemExit):
-        pc.main(["--probe", "--classify-file", str(tmp_path / "x.json")])
+        pc.main(["--probe", "--job", "api", "--classify-file", str(tmp_path / "x.json")])
+    with pytest.raises(SystemExit):
+        pc.main(["--job", "api"])
+    # `--job` itself is required: a default would hand both jobs one tracker (#793).
+    with pytest.raises(SystemExit):
+        pc.main(["--probe"])
     with pytest.raises(SystemExit):
-        pc.main([])
+        pc.main(["--probe", "--job", "pip"])

tests/github-workflows/migration/verify_migration_api.py

@@ -85,7 +85,9 @@ def main():
     # migration-specific checks above (flow preserved, variables preserved) still ran
     # and still fail on their own terms if they are broken.
     status = (
-        "pass" if success else credentials.step_status(detail, "nightly_api/execute_flow_api")
+        "pass"
+        if success
+        else credentials.step_status(detail, "nightly_api/execute_flow_api", job="api")
     )
     print(f"   {status.upper()}: {detail[:120]}")
     phase["steps"]["execute_flow_api"] = {"status": status, "detail": detail}