[codex] Polish DAG viewer metadata and previews (#1340)

Polish DAG viewer metadata and previews

Author: aneiderhiser

DAG_viewer/public/app.js

@@ -120,7 +120,8 @@ pre {
 }
 
 .dag-launcher {
-  flex: 0 1 360px;
+  flex: 1 1 680px;
+  max-width: 820px;
   min-width: 0;
   position: relative;
 }
@@ -559,6 +560,30 @@ body.is-panning {
   z-index: 0;
 }
 
+.system-overview-group-frame {
+  background: rgba(255, 255, 255, 0.2);
+  border: 1.5px dashed rgba(39, 38, 35, 0.46);
+  border-radius: 28px;
+  pointer-events: none;
+  position: absolute;
+  z-index: 0;
+}
+
+.system-overview-group-label {
+  color: rgba(36, 35, 32, 0.78);
+  font-size: 0.76rem;
+  font-weight: 700;
+  letter-spacing: 0.08em;
+  line-height: 1.2;
+  pointer-events: none;
+  position: absolute;
+  text-align: center;
+  text-transform: uppercase;
+  transform: translateX(-50%);
+  width: 240px;
+  z-index: 1;
+}
+
 .system-overview-entry {
   align-items: center;
   appearance: none;
@@ -843,6 +868,7 @@ body.is-panning {
   display: grid;
   gap: 12px;
   grid-auto-flow: row;
+  min-height: 0;
   overflow: auto;
   padding: 8px 12px 12px;
 }
@@ -1076,7 +1102,7 @@ body.is-panning {
 }
 
 .seed-table-wrap {
-  max-height: 420px;
+  max-height: none;
 }
 
 .seed-preview-table th,

DAG_viewer/public/styles.css

@@ -1741,8 +1741,7 @@ function buildSeedViewer(manifestNode) {
   }
 
   if (normalizedPath.includes("seeds/value_sets/")) {
-    const datasetKey = buildValueSetSeedPublicKey({ manifestNode, normalizedPath });
-    const fileName = datasetKey ? `${datasetKey}.csv_0_0_0.csv.gz` : null;
+    const fileName = buildValueSetSeedObjectFileName({ manifestNode, normalizedPath });
 
     return {
       sourceType: "seed_preview",
@@ -1758,7 +1757,10 @@ function buildSeedViewer(manifestNode) {
 }
 
 function buildTerminologySeedDownloadUrl({ manifestNode, baseDomain }) {
-  const datasetKey = (manifestNode?.name || "").replace(/^terminology__/, "");
+  const seedName = manifestNode?.name || "";
+  const datasetKey = seedName === "terminology__icd10_pcs_cms_ontology"
+    ? "icd_10_pcs_cms_ontology"
+    : seedName.replace(/^terminology__/, "");
 
   if (!datasetKey || datasetKey === "provider") {
     return null;
@@ -1767,18 +1769,74 @@ function buildTerminologySeedDownloadUrl({ manifestNode, baseDomain }) {
   return `${baseDomain}/versioned_terminology/latest/${datasetKey}.csv_0_0_0.csv.gz`;
 }
 
-function buildValueSetSeedPublicKey({ manifestNode, normalizedPath }) {
+function buildValueSetSeedObjectFileName({ manifestNode, normalizedPath }) {
   const seedName = manifestNode?.name || path.posix.basename(normalizedPath).replace(/\.csv$/i, "");
 
+  if (!seedName) {
+    return null;
+  }
+
+  if (seedName === "encounter_group_sk" || seedName === "encounter_type_sk" || seedName === "predictor_encounter_xwalk") {
+    return `${seedName}.csv.gz`;
+  }
+
+  if (seedName.startsWith("pqi__")) {
+    return null;
+  }
+
   if (seedName === "ed_classification__categories") {
-    return "ed_classification_categories";
+    return "ed_classification_categories.csv_0_0_0.csv.gz";
   }
 
   if (seedName.startsWith("ed_classification__")) {
-    return seedName.replace(/^ed_classification__/, "");
+    return `${seedName.replace(/^ed_classification__/, "")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName.startsWith("ccsr__")) {
+    return `${seedName.replace(/^ccsr__/, "")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName.startsWith("chronic_conditions__")) {
+    return `${seedName.replace(/^chronic_conditions__/, "")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName === "cms_hcc__disease_hierarchy_flat") {
+    return null;
+  }
+
+  if (seedName.startsWith("cms_hcc__")) {
+    return `${seedName.replace(/^cms_hcc__/, "cms_hcc_")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName.startsWith("data_quality__")) {
+    return `${seedName.replace(/^data_quality__/, "data_quality_")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName === "hcc_suspecting__hcc_descriptions") {
+    return "hcc_suspecting_descriptions.csv_0_0_0.csv.gz";
+  }
+
+  if (seedName.startsWith("hcc_suspecting__")) {
+    return `${seedName.replace(/^hcc_suspecting__/, "hcc_suspecting_")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName.startsWith("pharmacy__")) {
+    return `${seedName.replace(/^pharmacy__/, "")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName === "quality_measures__value_sets") {
+    return "quality_measures_value_set_codes.csv_0_0_0.csv.gz";
+  }
+
+  if (seedName.startsWith("quality_measures__")) {
+    return `${seedName.replace(/^quality_measures__/, "quality_measures_")}.csv_0_0_0.csv.gz`;
+  }
+
+  if (seedName.startsWith("readmissions__")) {
+    return `${seedName.replace(/^readmissions__/, "")}.csv_0_0_0.csv.gz`;
   }
 
-  return seedName || null;
+  return `${seedName}.csv_0_0_0.csv.gz`;
 }
 
 function sortNodeType(nodeType) {

DAG_viewer/scripts/build-lineage.mjs

@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { cp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { gunzipSync } from "node:zlib";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
 import { parse as parseYaml } from "yaml";
@@ -15,6 +16,7 @@ const manifestPath = path.join(cacheRoot, "manifest.json");
 const lineageCacheRoot = path.join(cacheRoot, "lineage");
 const defaultRepoUrl = "https://github.com/tuva-health/tuva.git";
 const defaultGithubRef = "main";
+const staticSeedPreviewRowLimit = Number(process.env.TUVA_DAG_SEED_PREVIEW_ROW_LIMIT || 1000) || 1000;
 
 async function main() {
   const sourceRoot = await resolveSourceRoot();
@@ -281,14 +283,18 @@ async function exportLineage(sourceRoot) {
 
   const lineageModule = await import(pathToFileURL(path.join(scriptDir, "build-lineage.mjs")).href);
   const targets = await lineageModule.listTargetConfigs();
+  const seedPreviewNodes = new Map();
 
   for (const target of targets) {
     const payload = await lineageModule.buildLineagePayload({ targetKey: target.key });
     const staticResponse = buildStaticResponse({ payload, targets });
     const outputPath = path.join(distRoot, "data", `${target.key}-lineage.json`);
 
     await writeFile(outputPath, `${JSON.stringify(staticResponse, null, 2)}\n`, "utf8");
+    collectSeedPreviewNodes(seedPreviewNodes, payload);
   }
+
+  await exportStaticSeedPreviews(seedPreviewNodes);
 }
 
 function buildStaticResponse({ payload, targets }) {
@@ -314,6 +320,151 @@ function buildStaticResponse({ payload, targets }) {
   };
 }
 
+function collectSeedPreviewNodes(seedPreviewNodes, payload) {
+  for (const node of payload.nodes || []) {
+    if (node?.resourceType !== "seed" || seedPreviewNodes.has(node.id)) {
+      continue;
+    }
+
+    seedPreviewNodes.set(node.id, {
+      nodeId: node.id,
+      name: node.name,
+      csvPath: node.paths?.sql || null,
+      seedViewer: node.seedViewer || null,
+      columns: node.columns || []
+    });
+  }
+}
+
+async function exportStaticSeedPreviews(seedPreviewNodes) {
+  const outputRoot = path.join(distRoot, "data", "seed-previews");
+  await mkdir(outputRoot, { recursive: true });
+
+  for (const seedNode of seedPreviewNodes.values()) {
+    const snapshot = await buildStaticSeedPreviewSnapshot(seedNode);
+    const outputPath = path.join(outputRoot, `${seedNode.nodeId}.json`);
+    await writeFile(outputPath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
+  }
+}
+
+async function buildStaticSeedPreviewSnapshot(seedNode) {
+  const fallbackHeaders = seedNode.columns.map((column) => column.name).filter(Boolean);
+  let preview = null;
+  let sourceError = null;
+
+  if (seedNode.seedViewer?.downloadUrl) {
+    try {
+      preview = await readCsvPreviewFromUrl(seedNode.seedViewer.downloadUrl, staticSeedPreviewRowLimit, fallbackHeaders);
+    } catch (error) {
+      sourceError = error instanceof Error ? error.message : String(error);
+    }
+  }
+
+  if (!preview && seedNode.csvPath && existsSync(seedNode.csvPath)) {
+    try {
+      preview = await readCsvPreviewFromFile(seedNode.csvPath, staticSeedPreviewRowLimit, fallbackHeaders);
+    } catch (error) {
+      sourceError = sourceError || (error instanceof Error ? error.message : String(error));
+    }
+  }
+
+  return {
+    nodeId: seedNode.nodeId,
+    name: seedNode.name,
+    generatedAt: new Date().toISOString(),
+    sourceUrl: seedNode.seedViewer?.downloadUrl || null,
+    sourceError,
+    headers: preview?.headers?.length ? preview.headers : fallbackHeaders,
+    rows: preview?.rows || [],
+    cachedRows: preview?.rows?.length || 0,
+    totalRows: preview?.totalRows ?? null,
+    rowLimit: staticSeedPreviewRowLimit,
+    truncated: Boolean(preview?.truncated)
+  };
+}
+
+async function readCsvPreviewFromUrl(url, rowLimit, preferredHeaders = []) {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(`CSV source returned ${response.status}`);
+  }
+
+  const sourceBuffer = Buffer.from(await response.arrayBuffer());
+  const csvBuffer = isGzipBuffer(sourceBuffer) ? gunzipSync(sourceBuffer) : sourceBuffer;
+  return readCsvPreviewFromText(csvBuffer.toString("utf8"), rowLimit, preferredHeaders);
+}
+
+async function readCsvPreviewFromFile(filePath, rowLimit, preferredHeaders = []) {
+  return readCsvPreviewFromText(await readFile(filePath, "utf8"), rowLimit, preferredHeaders);
+}
+
+function readCsvPreviewFromText(csvText, rowLimit, preferredHeaders = []) {
+  const lines = csvText.split(/\r?\n/);
+  const firstLine = lines.length ? parseCsvLine(lines[0].replace(/^\uFEFF/, "")) : [];
+  const hasPreferredHeaders = preferredHeaders.length > 0;
+  const headers = hasPreferredHeaders ? preferredHeaders : firstLine;
+  const firstLineIsHeader = hasPreferredHeaders
+    ? doCsvHeadersMatch(firstLine, preferredHeaders)
+    : firstLine.length > 0;
+  const dataLines = lines.slice(firstLineIsHeader ? 1 : 0).filter((line) => line.length);
+  const rows = dataLines.slice(0, rowLimit).map(parseCsvLine);
+
+  return {
+    headers,
+    rows,
+    totalRows: dataLines.length,
+    truncated: dataLines.length > rowLimit
+  };
+}
+
+function doCsvHeadersMatch(csvHeaders, preferredHeaders) {
+  if (csvHeaders.length !== preferredHeaders.length) {
+    return false;
+  }
+
+  return csvHeaders.every((header, index) => normalizeCsvHeader(header) === normalizeCsvHeader(preferredHeaders[index]));
+}
+
+function normalizeCsvHeader(header) {
+  return String(header || "").trim().toLowerCase();
+}
+
+function isGzipBuffer(buffer) {
+  return buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b;
+}
+
+function parseCsvLine(line) {
+  const cells = [];
+  let cell = "";
+  let inQuotes = false;
+
+  for (let index = 0; index < line.length; index += 1) {
+    const character = line[index];
+
+    if (character === "\"") {
+      if (inQuotes && line[index + 1] === "\"") {
+        cell += "\"";
+        index += 1;
+      } else {
+        inQuotes = !inQuotes;
+      }
+      continue;
+    }
+
+    if (character === "," && !inQuotes) {
+      cells.push(cell);
+      cell = "";
+      continue;
+    }
+
+    cell += character;
+  }
+
+  cells.push(cell);
+  return cells;
+}
+
 async function walkSelectedSourceFiles(sourceRoot) {
   const roots = ["models", "seeds"];
   const files = [];

DAG_viewer/scripts/build-static-data.mjs

@@ -5,7 +5,7 @@ models:
       The appointment table contains information related to appointments at a healthcare facility. This table may include canceled, completed, or scheduled appointments.
     grain: One record per appointment per patient.
     transformation_steps: |-
-      This model brings the input layer appointment table into the Tuva package -- it does not do any transformation.
+      This model brings the input layer appointment table into the Tuva package. It does not do any transformation; it's just `select *`.
     mapping_instructions: 'No explicit mapping instructions provided.'
     config:
       schema: |
@@ -121,3 +121,4 @@ models:
         config:
           meta:
             data_type: varchar
+            is_primary_key: true

models/input_layer/input_layer__appointment.yml

@@ -6,6 +6,8 @@ models:
       patients have been officially diagnosed with or are suspected of having.
       This includes problems, admitting diagnosis codes, and billable diagnoses.
     grain: Intended to be one record per condition record per data source.
+    transformation_steps: |-
+      This model brings the input layer condition table into the Tuva package. It does not do any transformation; it's just `select *`.
     mapping_instructions: 'No explicit mapping instructions provided.'
     config:
       schema: "{%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_input_layer{% else %}input_layer{%- endif -%}\n"

models/input_layer/input_layer__condition.yml

@@ -9,6 +9,8 @@ models:
       Eligibility data is also used to calculate member months, which are
       needed for measures such as PMPM.
     grain: Intended to be one record per person, member, enrollment span, payer, plan, and data source.
+    transformation_steps: |-
+      This model brings the input layer eligibility table into the Tuva package. It does not do any transformation; it's just `select *`.
     mapping_instructions: ''
     tests:
       - dbt_utils.unique_combination_of_columns:

models/input_layer/input_layer__eligibility.yml

@@ -8,6 +8,8 @@ models:
       encounter_type field correctly for clinical data; in claims data this
       field is determined automatically.
     grain: Intended to be one record per encounter per data source.
+    transformation_steps: |-
+      This model brings the input layer encounter table into the Tuva package. It does not do any transformation; it's just `select *`.
     mapping_instructions: 'No explicit mapping instructions provided.'
     config:
       schema: |

models/input_layer/input_layer__encounter.yml

@@ -5,6 +5,8 @@ models:
       The immunization table contains information about immunizations
       administered to patients.
     grain: Intended to be one record per immunization event per data source.
+    transformation_steps: |-
+      This model brings the input layer immunization table into the Tuva package. It does not do any transformation; it's just `select *`.
     mapping_instructions: 'No explicit mapping instructions provided.'
     config:
       schema: |