chore(dashboard): Drop redundant CI dashboard columns (#484)

dillon-zheng · web-flow · commit 28522eb9b2f2 · 2026-06-13T20:52:35.000+08:00
## Summary

- Stop reading and writing the redundant `ci_l1_builds.url` column,
using `normalized_build_url` as the canonical Jenkins build URL.
- Stop persisting unused `ci_l1_pod_lifecycle.jenkins_label_digest` and
`jenkins_controller` metadata.
- Add follow-up schema migrations to drop the redundant build URL
column, one redundant build index, and the unused pod lifecycle metadata
columns.

## Rollout notes

This PR intentionally keeps code and schema compatible for a two-step
rollout:

1. Deploy the new dashboard image first so workers no longer reference
the dropped columns.
2. After confirming old `sync_pods`, `sync_builds`, and Jenkins worker
jobs are no longer running, execute the schema migrations:
   - `025_drop_ci_l1_builds_url.sql`
   - `026_drop_redundant_ci_l1_build_indexes.sql`
   - `027_drop_unused_pod_lifecycle_metadata_columns.sql`

Do not run the drop-column migrations before all old workers are
drained, because old workers still reference these columns.

## Validation

- `git diff --check`
- `.venv/bin/python -m ruff check src tests`
- `PYTHONPATH=src .venv/bin/python -m pytest -q
tests/jobs/test_sync_pods.py tests/api/test_runtime_insights.py
tests/jobs/test_archive_error_logs.py tests/jobs/test_jenkins_worker.py
tests/test_models.py`
diff --git a/ci-dashboard/sql/006_create_ci_l1_pod_lifecycle.sql b/ci-dashboard/sql/006_create_ci_l1_pod_lifecycle.sql
@@ -17,8 +17,6 @@ CREATE TABLE IF NOT EXISTS ci_l1_pod_lifecycle (
   pod_org VARCHAR(255) NULL,
   pod_repo VARCHAR(255) NULL,
   jenkins_label VARCHAR(255) NULL,
-  jenkins_label_digest VARCHAR(255) NULL,
-  jenkins_controller VARCHAR(255) NULL,
   ci_job VARCHAR(255) NULL,
   source_prow_job_id VARCHAR(128) NULL,
   normalized_build_url VARCHAR(1024) NULL,
diff --git a/ci-dashboard/sql/012_alter_ci_l1_pod_lifecycle_add_pod_metadata_fields.sql b/ci-dashboard/sql/012_alter_ci_l1_pod_lifecycle_add_pod_metadata_fields.sql
@@ -20,10 +20,4 @@ ALTER TABLE ci_l1_pod_lifecycle
   ADD COLUMN IF NOT EXISTS jenkins_label VARCHAR(255) NULL AFTER pod_repo;
 
 ALTER TABLE ci_l1_pod_lifecycle
-  ADD COLUMN IF NOT EXISTS jenkins_label_digest VARCHAR(255) NULL AFTER jenkins_label;
-
-ALTER TABLE ci_l1_pod_lifecycle
-  ADD COLUMN IF NOT EXISTS jenkins_controller VARCHAR(255) NULL AFTER jenkins_label_digest;
-
-ALTER TABLE ci_l1_pod_lifecycle
-  ADD COLUMN IF NOT EXISTS ci_job VARCHAR(255) NULL AFTER jenkins_controller;
+  ADD COLUMN IF NOT EXISTS ci_job VARCHAR(255) NULL AFTER jenkins_label;
diff --git a/ci-dashboard/sql/025_drop_ci_l1_builds_url.sql b/ci-dashboard/sql/025_drop_ci_l1_builds_url.sql
@@ -0,0 +1,2 @@
+ALTER TABLE ci_l1_builds
+  DROP COLUMN IF EXISTS url;
diff --git a/ci-dashboard/sql/026_drop_redundant_ci_l1_build_indexes.sql b/ci-dashboard/sql/026_drop_redundant_ci_l1_build_indexes.sql
@@ -0,0 +1,2 @@
+ALTER TABLE ci_l1_builds
+  DROP INDEX IF EXISTS idx_ci_l1_builds_build_system_state_start_id;
diff --git a/ci-dashboard/sql/027_drop_unused_pod_lifecycle_metadata_columns.sql b/ci-dashboard/sql/027_drop_unused_pod_lifecycle_metadata_columns.sql
@@ -0,0 +1,5 @@
+ALTER TABLE ci_l1_pod_lifecycle
+  DROP COLUMN IF EXISTS jenkins_label_digest;
+
+ALTER TABLE ci_l1_pod_lifecycle
+  DROP COLUMN IF EXISTS jenkins_controller;
diff --git a/ci-dashboard/src/ci_dashboard/api/queries/runtime.py b/ci-dashboard/src/ci_dashboard/api/queries/runtime.py
@@ -1556,7 +1556,6 @@ def get_error_builds(
                 SELECT
                   b.id AS build_row_id,
                   b.build_id,
-                  b.url,
                   b.normalized_build_url,
                   b.completion_time,
                   b.start_time
@@ -2148,7 +2147,7 @@ def _job_ranking_item(row: Any, *, value_key: str, extra_keys: tuple[str, ...] =
 
 
 def _error_build_item(row: Any, *, include_completion_time: bool = False) -> dict[str, Any]:
-    build_url = normalize_build_url(row.get("normalized_build_url")) or normalize_build_url(row.get("url"))
+    build_url = normalize_build_url(row.get("normalized_build_url"))
     build_number_from_url = _build_number_from_url(build_url)
     raw_build_number = str(row.get("build_id") or "").strip()
     if build_number_from_url:
diff --git a/ci-dashboard/src/ci_dashboard/common/models.py b/ci-dashboard/src/ci_dashboard/common/models.py
@@ -22,7 +22,6 @@ class NormalizedBuildRow:
     pr_number: int | None
     is_pr_build: bool
     context: str | None
-    url: str
     normalized_build_url: str | None
     author: str | None
     retest: bool | None
diff --git a/ci-dashboard/src/ci_dashboard/jobs/analyze_errors.py b/ci-dashboard/src/ci_dashboard/jobs/analyze_errors.py
@@ -105,7 +105,8 @@
 FETCH_CANDIDATE_BUILDS_BY_IDS = text(
     """
     SELECT b.id, b.source_prow_job_id, b.job_name, b.job_type, b.repo_full_name,
-           b.pr_number, b.head_sha, b.url, b.normalized_build_url, b.pod_name, b.log_gcs_uri,
+           b.pr_number, b.head_sha, b.normalized_build_url,
+           b.pod_name, b.log_gcs_uri,
            b.error_l1_category, b.error_l2_subcategory,
            b.revise_error_l1_category, b.revise_error_l2_subcategory,
            pj.state AS prow_state, pj.status AS prow_status,
@@ -142,7 +143,8 @@
 FETCH_CANDIDATE_BUILD_BY_ID = text(
     """
     SELECT b.id, b.source_prow_job_id, b.job_name, b.job_type, b.repo_full_name,
-           b.pr_number, b.head_sha, b.url, b.normalized_build_url, b.pod_name, b.log_gcs_uri,
+           b.pr_number, b.head_sha, b.normalized_build_url,
+           b.pod_name, b.log_gcs_uri,
            b.error_l1_category, b.error_l2_subcategory,
            b.revise_error_l1_category, b.revise_error_l2_subcategory,
            pj.state AS prow_state, pj.status AS prow_status,
@@ -677,7 +679,7 @@ def _append_live_signal_excerpts(
     if not callable(fetch_console_signal_excerpts):
         return log_text
 
-    build_url = str(build.get("url") or "").strip()
+    build_url = str(build.get("normalized_build_url") or "").strip()
     if not build_url:
         return log_text
 
diff --git a/ci-dashboard/src/ci_dashboard/jobs/archive_error_logs.py b/ci-dashboard/src/ci_dashboard/jobs/archive_error_logs.py
@@ -19,7 +19,7 @@
 
 FETCH_CANDIDATE_BUILD_BY_ID = text(
     """
-    SELECT id, url, normalized_build_url, log_gcs_uri, state, build_system,
+    SELECT id, normalized_build_url, log_gcs_uri, state, build_system,
            start_time, completion_time
     FROM ci_l1_builds
     WHERE id = :build_id
@@ -31,7 +31,7 @@
 
 FETCH_CANDIDATE_BUILD_BY_ID_FORCE = text(
     """
-    SELECT id, url, normalized_build_url, log_gcs_uri, state, build_system,
+    SELECT id, normalized_build_url, log_gcs_uri, state, build_system,
            start_time, completion_time
     FROM ci_l1_builds
     WHERE id = :build_id
@@ -42,7 +42,7 @@
 
 FETCH_CANDIDATE_BUILDS_SCAN = text(
     """
-    SELECT id, url, normalized_build_url, log_gcs_uri, state, build_system,
+    SELECT id, normalized_build_url, log_gcs_uri, state, build_system,
            start_time, completion_time
     FROM ci_l1_builds
     WHERE build_system = 'JENKINS'
@@ -55,7 +55,7 @@
 
 FETCH_CANDIDATE_BUILDS_SCAN_FORCE = text(
     """
-    SELECT id, url, normalized_build_url, log_gcs_uri, state, build_system,
+    SELECT id, normalized_build_url, log_gcs_uri, state, build_system,
            start_time, completion_time
     FROM ci_l1_builds
     WHERE build_system = 'JENKINS'
@@ -236,7 +236,7 @@ def _archive_single_build(
     if not force and build.get("log_gcs_uri"):
         return False
 
-    build_url = str(build.get("url") or "").strip()
+    build_url = str(build.get("normalized_build_url") or "").strip()
     if not build_url:
         raise ValueError(f"build {build['id']} is missing Jenkins build URL")
 
diff --git a/ci-dashboard/src/ci_dashboard/jobs/jenkins_timings.py b/ci-dashboard/src/ci_dashboard/jobs/jenkins_timings.py
@@ -261,9 +261,11 @@ def run_backfill_jenkins_timings(
     limit_clause = "LIMIT :limit_value" if limit is not None else ""
     query = text(
         f"""
-        SELECT id, COALESCE(normalized_build_url, url) AS build_url
+        SELECT id, normalized_build_url AS build_url
         FROM ci_l1_builds
         WHERE build_system = 'JENKINS'
+          AND normalized_build_url IS NOT NULL
+          AND normalized_build_url <> ''
           AND completion_time IS NOT NULL
           AND completion_time >= :cutoff
           AND jenkins_queue_total_subtasks_sum IS NULL
diff --git a/ci-dashboard/src/ci_dashboard/jobs/jenkins_worker.py b/ci-dashboard/src/ci_dashboard/jobs/jenkins_worker.py
@@ -115,7 +115,6 @@
       pr_number,
       is_pr_build,
       context,
-      url,
       normalized_build_url,
       author,
       retest,
@@ -148,7 +147,6 @@
       :pr_number,
       :is_pr_build,
       :context,
-      :url,
       :normalized_build_url,
       :author,
       :retest,
@@ -174,7 +172,6 @@
     UPDATE ci_l1_builds
     SET source_prow_job_id = COALESCE(source_prow_job_id, :source_prow_job_id),
         state = :state,
-        url = COALESCE(url, :url),
         normalized_build_url = COALESCE(normalized_build_url, :normalized_build_url),
         job_name = CASE
           WHEN :job_name IS NULL THEN job_name
@@ -270,7 +267,6 @@ def as_build_row_params(self) -> dict[str, Any]:
             "pr_number": self.pr_number,
             "is_pr_build": int(self.is_pr_build),
             "context": self.context,
-            "url": self.build_url,
             "normalized_build_url": self.normalized_build_url,
             "author": self.author,
             "retest": None,
diff --git a/ci-dashboard/src/ci_dashboard/jobs/sync_builds.py b/ci-dashboard/src/ci_dashboard/jobs/sync_builds.py
@@ -72,7 +72,6 @@ def _build_window_fetch_source_rows_query(has_end: bool):
     "pr_number",
     "is_pr_build",
     "context",
-    "url",
     "normalized_build_url",
     "author",
     "retest",
@@ -208,7 +207,6 @@ def map_build_row(row: Mapping[str, Any]) -> NormalizedBuildRow:
         pr_number=pr_number,
         is_pr_build=is_pr_build,
         context=_coerce_str(_get_first(row, "context"), allow_none=True),
-        url=url,
         normalized_build_url=normalize_build_url(url),
         author=_coerce_str(_get_first(row, "author"), allow_none=True),
         retest=_coerce_optional_bool(_get_first(row, "retest")),
diff --git a/ci-dashboard/src/ci_dashboard/jobs/sync_pods.py b/ci-dashboard/src/ci_dashboard/jobs/sync_pods.py
@@ -148,8 +148,6 @@ def as_lifecycle_fields(self) -> dict[str, Any]:
             "pod_org": _coerce_str(self.labels.get("org")),
             "pod_repo": _coerce_str(self.labels.get("repo")),
             "jenkins_label": _coerce_str(self.labels.get("jenkins/label")),
-            "jenkins_label_digest": _coerce_str(self.labels.get("jenkins/label-digest")),
-            "jenkins_controller": _coerce_str(self.labels.get("kubernetes.jenkins.io/controller")),
             "ci_job": _coerce_str(self.annotations.get("ci_job")),
         }
 
@@ -1219,7 +1217,7 @@ def _load_direct_build_metadata_map(
               repo_full_name,
               job_name,
               author,
-              url,
+              build_system,
               start_time
             FROM ci_l1_builds
             WHERE pod_name IN ({", ".join(placeholders)})
@@ -1235,7 +1233,7 @@ def _load_direct_build_metadata_map(
         if pod_name is None or pod_name in metadata_by_pod_name:
             continue
         payload = dict(row)
-        build_system = classify_build_system(payload.get("url"))
+        build_system = payload.get("build_system") or classify_build_system(payload.get("normalized_build_url"))
         metadata_by_pod_name[pod_name] = {
             "build_system": build_system,
             "source_prow_job_id": payload.get("source_prow_job_id"),
@@ -1534,8 +1532,6 @@ def _empty_pod_metadata_fields() -> dict[str, Any]:
         "pod_org": None,
         "pod_repo": None,
         "jenkins_label": None,
-        "jenkins_label_digest": None,
-        "jenkins_controller": None,
         "ci_job": None,
     }
 
@@ -2035,15 +2031,15 @@ def _build_pod_lifecycle_upsert_statement(connection: Connection):
               source_project, cluster_name, location, namespace_name, pod_name, pod_uid,
               build_system, pod_labels_json, pod_annotations_json, metadata_observed_at, pod_created_at,
               abnormal_reason, abnormal_message,
-              pod_author, pod_org, pod_repo, jenkins_label, jenkins_label_digest, jenkins_controller, ci_job,
+              pod_author, pod_org, pod_repo, jenkins_label, ci_job,
               source_prow_job_id, normalized_build_url, repo_full_name, job_name,
               scheduled_at, first_pulling_at, first_pulled_at, first_created_at, first_started_at,
               last_failed_scheduling_at, failed_scheduling_count, last_event_at, schedule_to_started_seconds
             ) VALUES (
               :source_project, :cluster_name, :location, :namespace_name, :pod_name, :pod_uid,
               :build_system, :pod_labels_json, :pod_annotations_json, :metadata_observed_at, :pod_created_at,
               :abnormal_reason, :abnormal_message,
-              :pod_author, :pod_org, :pod_repo, :jenkins_label, :jenkins_label_digest, :jenkins_controller, :ci_job,
+              :pod_author, :pod_org, :pod_repo, :jenkins_label, :ci_job,
               :source_prow_job_id, :normalized_build_url, :repo_full_name, :job_name,
               :scheduled_at, :first_pulling_at, :first_pulled_at, :first_created_at, :first_started_at,
               :last_failed_scheduling_at, :failed_scheduling_count, :last_event_at, :schedule_to_started_seconds
@@ -2062,8 +2058,6 @@ def _build_pod_lifecycle_upsert_statement(connection: Connection):
               pod_org = excluded.pod_org,
               pod_repo = excluded.pod_repo,
               jenkins_label = excluded.jenkins_label,
-              jenkins_label_digest = excluded.jenkins_label_digest,
-              jenkins_controller = excluded.jenkins_controller,
               ci_job = excluded.ci_job,
               source_prow_job_id = excluded.source_prow_job_id,
               normalized_build_url = excluded.normalized_build_url,
@@ -2087,15 +2081,15 @@ def _build_pod_lifecycle_upsert_statement(connection: Connection):
           source_project, cluster_name, location, namespace_name, pod_name, pod_uid,
           build_system, pod_labels_json, pod_annotations_json, metadata_observed_at, pod_created_at,
           abnormal_reason, abnormal_message,
-          pod_author, pod_org, pod_repo, jenkins_label, jenkins_label_digest, jenkins_controller, ci_job,
+          pod_author, pod_org, pod_repo, jenkins_label, ci_job,
           source_prow_job_id, normalized_build_url, repo_full_name, job_name,
           scheduled_at, first_pulling_at, first_pulled_at, first_created_at, first_started_at,
           last_failed_scheduling_at, failed_scheduling_count, last_event_at, schedule_to_started_seconds
         ) VALUES (
           :source_project, :cluster_name, :location, :namespace_name, :pod_name, :pod_uid,
           :build_system, :pod_labels_json, :pod_annotations_json, :metadata_observed_at, :pod_created_at,
           :abnormal_reason, :abnormal_message,
-          :pod_author, :pod_org, :pod_repo, :jenkins_label, :jenkins_label_digest, :jenkins_controller, :ci_job,
+          :pod_author, :pod_org, :pod_repo, :jenkins_label, :ci_job,
           :source_prow_job_id, :normalized_build_url, :repo_full_name, :job_name,
           :scheduled_at, :first_pulling_at, :first_pulled_at, :first_created_at, :first_started_at,
           :last_failed_scheduling_at, :failed_scheduling_count, :last_event_at, :schedule_to_started_seconds
@@ -2114,8 +2108,6 @@ def _build_pod_lifecycle_upsert_statement(connection: Connection):
           pod_org = VALUES(pod_org),
           pod_repo = VALUES(pod_repo),
           jenkins_label = VALUES(jenkins_label),
-          jenkins_label_digest = VALUES(jenkins_label_digest),
-          jenkins_controller = VALUES(jenkins_controller),
           ci_job = VALUES(ci_job),
           source_prow_job_id = VALUES(source_prow_job_id),
           normalized_build_url = VALUES(normalized_build_url),
diff --git a/ci-dashboard/tests/conftest.py b/ci-dashboard/tests/conftest.py
@@ -65,7 +65,7 @@ def _create_test_schema(engine: Engine) -> None:
           pr_number INTEGER NULL,
           is_pr_build INTEGER NOT NULL DEFAULT 0,
           context TEXT NULL,
-          url TEXT NOT NULL,
+          url TEXT NULL,
           normalized_build_url TEXT NULL,
           author TEXT NULL,
           retest INTEGER NULL,
@@ -262,8 +262,6 @@ def _create_test_schema(engine: Engine) -> None:
           pod_org TEXT NULL,
           pod_repo TEXT NULL,
           jenkins_label TEXT NULL,
-          jenkins_label_digest TEXT NULL,
-          jenkins_controller TEXT NULL,
           ci_job TEXT NULL,
           source_prow_job_id TEXT NULL,
           normalized_build_url TEXT NULL,
diff --git a/ci-dashboard/tests/jobs/test_analyze_errors.py b/ci-dashboard/tests/jobs/test_analyze_errors.py
@@ -1147,17 +1147,17 @@ def test_append_live_signal_excerpts_handles_missing_fetcher_url_and_signal() ->
     assert (
         _append_live_signal_excerpts(
             "tail",
-            build={"url": "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_check2/1/"},
+            build={"normalized_build_url": "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_check2/1/"},
             jenkins_fetcher=object(),
         )
         == "tail"
     )
     fetcher = _FakeJenkinsFetcher({})
-    assert _append_live_signal_excerpts("tail", build={"url": ""}, jenkins_fetcher=fetcher) == "tail"
+    assert _append_live_signal_excerpts("tail", build={"normalized_build_url": ""}, jenkins_fetcher=fetcher) == "tail"
     assert (
         _append_live_signal_excerpts(
             "tail",
-            build={"url": "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_check2/1/"},
+            build={"normalized_build_url": "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_check2/1/"},
             jenkins_fetcher=fetcher,
         )
         == "tail"
diff --git a/ci-dashboard/tests/jobs/test_archive_error_logs.py b/ci-dashboard/tests/jobs/test_archive_error_logs.py
@@ -172,7 +172,7 @@ def test_run_archive_error_logs_archives_failed_jenkins_build(sqlite_engine) ->
     assert summary.builds_archived == 1
     assert summary.builds_failed == 0
     assert fetcher.calls == [
-        ("https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/101/display/redirect", 1024)
+        ("https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/101/", 1024)
     ]
     assert uploader.calls[0][0] == "ci-dashboard-test"
     assert uploader.calls[0][1] == "2604/101.log"
@@ -208,7 +208,7 @@ def test_run_archive_error_logs_appends_failed_pipeline_node_logs(sqlite_engine)
 
     assert summary.builds_archived == 1
     assert fetcher.failed_node_calls == [
-        "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/151/display/redirect"
+        "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/151/"
     ]
     assert "Connection reset by peer" in uploader.calls[0][2]
     assert "Build completed, 1 test FAILED" in uploader.calls[0][2]
@@ -236,7 +236,7 @@ def test_run_archive_error_logs_appends_live_signal_excerpts_for_remoting_tail(s
 
     assert summary.builds_archived == 1
     assert fetcher.signal_calls == [
-        "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/161/display/redirect"
+        "https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/161/"
     ]
     assert "Timeout waiting for agent to come back" in uploader.calls[0][2]
     assert "Container [golang] terminated [OOMKilled]" in uploader.calls[0][2]
@@ -289,6 +289,35 @@ def test_run_archive_error_logs_force_overwrites_existing_path(sqlite_engine) ->
     assert uploader.calls[0][1] == "custom/path/build-301.log"
 
 
+def test_run_archive_error_logs_force_works_after_url_column_drop(sqlite_engine) -> None:
+    _insert_build(
+        sqlite_engine,
+        build_id=302,
+        log_gcs_uri="gcs://ci-dashboard-test/custom/path/build-302.log",
+    )
+    with sqlite_engine.begin() as connection:
+        connection.execute(text("ALTER TABLE ci_l1_builds DROP COLUMN url"))
+
+    fetcher = _FakeFetcher("password=very-secret\n")
+    uploader = _FakeUploader()
+
+    summary = run_archive_error_logs(
+        sqlite_engine,
+        _settings(),
+        build_id=302,
+        force=True,
+        fetcher=fetcher,
+        uploader=uploader,
+    )
+
+    assert summary.builds_scanned == 1
+    assert summary.builds_archived == 1
+    assert fetcher.calls == [
+        ("https://prow.tidb.net/jenkins/job/pingcap/job/tidb/job/ghpr_unit_test/302/", 1024)
+    ]
+    assert uploader.calls[0][1] == "custom/path/build-302.log"
+
+
 def test_build_archive_object_ref_reuses_existing_uri_on_force(sqlite_engine) -> None:
     _insert_build(
         sqlite_engine,
diff --git a/ci-dashboard/tests/jobs/test_jenkins_worker.py b/ci-dashboard/tests/jobs/test_jenkins_worker.py
diff --git a/ci-dashboard/tests/test_models.py b/ci-dashboard/tests/test_models.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+ALTER TABLE ci_l1_builds`
	`2`	`+ DROP COLUMN IF EXISTS url;`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+ALTER TABLE ci_l1_builds`
	`2`	`+ DROP INDEX IF EXISTS idx_ci_l1_builds_build_system_state_start_id;`