Extract Jinja scanner (#46)

Author: sabino

PLAN.md

@@ -569,7 +569,7 @@ Exit criteria:
 
 - M0 is complete as the Zig `0.16.0` runtime scaffold.
 - M1 has started on stacked branches with native Zig artifact-first parser slices.
-- M1A has started with behavior-preserving `src/project/types.zig`, `src/project/selector.zig`, `src/project/manifest.zig`, `src/project/util.zig`, `src/project/config.zig`, `src/project/fs.zig`, `src/project/jinja.zig`, `src/project/resolve.zig`, and `src/project/parse.zig` extractions. `src/project/manifest.zig` owns selected-resource JSON and partial `manifest.json` writing with native tests for selected JSON shape, JSON escaping, exposure dependency ordering, disabled-resource filtering, and graph-map output. `src/project/util.zig` owns shared display, membership, append-dedup, string sorting, and narrow YAML scalar/list helpers. `src/project/config.zig` owns `dbt_project.yml` loading, project path/docs config parsing, and applying parsed project path/docs configs to graph nodes. `src/project/fs.zig` owns deterministic resource file discovery, Linux directory traversal helpers, and resource path/name helpers. `src/project/jinja.zig` owns lexical Jinja call, parenthesis, quoted string, and literal argument helpers while graph-aware scanning remains in `src/project.zig`. `src/project/resolve.zig` owns graph lookup/count helpers, canonical graph resource ordering, duplicate resource validation, macro unique-id package extraction, low-level ref/source resolution helpers, and dependency-map mutation for refs, sources, and known macro dependencies. `src/project/parse.zig` owns narrow parser scalar helpers for YAML booleans, JSON-compatible scalar classification, generic-test YAML item names, generic-test definition construction/cloning, generic-test relationship target ref parsing, and generic-test identity/name/hash helpers with native tests. `src/project.zig` remains the public parser/list facade and still owns most loader, resource parser, scanner, and resolver orchestration code until follow-up extractions move those pieces behind focused internal modules.
+- M1A has started with behavior-preserving `src/project/types.zig`, `src/project/selector.zig`, `src/project/manifest.zig`, `src/project/util.zig`, `src/project/config.zig`, `src/project/fs.zig`, `src/project/jinja.zig`, `src/project/resolve.zig`, and `src/project/parse.zig` extractions. `src/project/manifest.zig` owns selected-resource JSON and partial `manifest.json` writing with native tests for selected JSON shape, JSON escaping, exposure dependency ordering, disabled-resource filtering, and graph-map output. `src/project/util.zig` owns shared display, membership, append-dedup, string sorting, and narrow YAML scalar/list helpers. `src/project/config.zig` owns `dbt_project.yml` loading, project path/docs config parsing, and applying parsed project path/docs configs to graph nodes. `src/project/fs.zig` owns deterministic resource file discovery, Linux directory traversal helpers, and resource path/name helpers. `src/project/jinja.zig` owns lexical Jinja call, parenthesis, quoted string, literal argument helpers, supported model SQL scanning, inline config/tag parsing, and known macro-call scanning with native tests. `src/project/resolve.zig` owns graph lookup/count helpers, canonical graph resource ordering, duplicate resource validation, macro unique-id package extraction, low-level ref/source resolution helpers, and dependency-map mutation for refs, sources, and known macro dependencies. `src/project/parse.zig` owns narrow parser scalar helpers for YAML booleans, JSON-compatible scalar classification, generic-test YAML item names, generic-test definition construction/cloning, generic-test relationship target ref parsing, and generic-test identity/name/hash helpers with native tests. `src/project.zig` remains the public parser/list facade and still owns most loader, resource parser, and resolver orchestration code until follow-up extractions move those pieces behind focused internal modules.
 - `dxt parse` now targets the supported Tier 0 subset: project name/model paths/seed paths/macro paths, project and package model path configs for literal `+materialized`, `+tags`, and model/seed `+docs.node_color`, root-project model config overrides for installed packages, SQL model discovery, CSV seed discovery, installed package SQL model and CSV seed discovery from `dbt_packages`, source discovery, installed package source discovery, exposure discovery, installed package exposure discovery, project macro discovery, installed package macro discovery from `dbt_packages`, macro property YAML for project macro descriptions and arguments, project and package docs block discovery, literal `ref` to models or seeds, two-argument package refs, package-local refs in installed package models and exposures, unique installed-package fallback for unqualified refs, literal `source`, package-local sources in installed package models, unique installed-package fallback for unqualified sources, literal `doc` in project and package descriptions, inline `config(materialized=..., tags=...)`, known project/package-qualified/package-local macro call dependencies, narrow project and package YAML model properties for scalar descriptions, simple columns, tags, materialization, disabled SQL models, dbt-shaped `unique`, `not_null`, `accepted_values`, and `relationships` generic test nodes, model/test `refs` and `sources` artifact fields, dependency maps, and deterministic partial `manifest.json`. The manifest includes the v12 top-level maps needed by the M1 artifact shape and is covered by a pinned local dbt Manifest v12 schema slice. YAML generic test arguments are currently supported for scalar values plus inline and block lists required by public Jaffle Shop DuckDB-style tests.
 - `dxt ls` now lists dbt-selectable resources from the same parser graph with stable text/JSON output and basic name/FQN wildcards, tag wildcards, slash-aware `path:` wildcards, exact `package:`/`package:this`, `source:` wildcards including package-qualified source selectors, `exposure:` wildcards, `resource_type:`, `test_type:generic`, config materialization, comma intersection, whitespace union, multi-argument selector lists, repeated selector flags, leading/trailing `+` graph expansion, and exact exclude filters; macros are emitted in artifacts but not exposed as `ls` resources.
 - Synthetic fixtures cover one model, model refs, seed refs, source refs, exposure refs to models and sources, combined source/model YAML, inline config/tag selection, config materialization selection, comma-intersection selection, YAML model properties and columns, emitted `unique`, `not_null`, `accepted_values`, and `relationships` generic test nodes, project macro artifacts and macro properties, configured `macro-paths` replacing the default macro directory, installed package macros with package-qualified calls and package-local macro calls, installed package models, seeds, sources, docs, exposures, package YAML model properties, root package config overrides, and package-qualified/package-local refs/sources, macro calls recorded in model and macro `depends_on.macros`, docs blocks with literal `doc` descriptions, disabled models, disabled ref diagnostics, unmatched model-property warnings, duplicate model and docs diagnostics, unsupported dynamic ref/doc diagnostics, missing doc diagnostics, malformed docs block diagnostics, unresolved package macro diagnostics, and unsupported unknown macro-call diagnostics.

src/project.zig

@@ -58,27 +58,18 @@ const parseJsonScalar = project_parse.parseJsonScalar;
 const refDepFromValue = project_parse.refDepFromValue;
 const synthesizeGenericTestNames = project_parse.synthesizeGenericTestNames;
 const testNameFromYamlItem = project_parse.testNameFromYamlItem;
-const findKeyword = project_jinja.findKeyword;
 const findMatchingParen = project_jinja.findMatchingParen;
-const findValueStart = project_jinja.findValueStart;
 const isIdentChar = project_jinja.isIdentChar;
 const isIdentStart = project_jinja.isIdentStart;
 const parseLiteralArgs = project_jinja.parseLiteralArgs;
-const parseQuoted = project_jinja.parseQuoted;
-const readJinjaCall = project_jinja.readJinjaCall;
-const skipQuotedSpan = project_jinja.skipQuotedSpan;
 const skipWs = project_jinja.skipWs;
 const appendUnique = util.appendUnique;
 const countActiveExposures = project_resolve.countActiveExposures;
 const countActiveNodes = project_resolve.countActiveNodes;
 const countActiveSeeds = project_resolve.countActiveSeeds;
 const findDoc = project_resolve.findDoc;
-const findMacroIdByPackageAndName = project_resolve.findMacroIdByPackageAndName;
-const findMacroIdForUnqualifiedCall = project_resolve.findMacroIdForUnqualifiedCall;
 const findMacroIndexByPackageAndName = project_resolve.findMacroIndexByPackageAndName;
 const findModelIndexByName = project_resolve.findModelIndexByName;
-const hasMacroPackage = project_resolve.hasMacroPackage;
-const packageNameFromMacroUniqueId = project_resolve.packageNameFromMacroUniqueId;
 const rejectDuplicateDocs = project_resolve.rejectDuplicateDocs;
 const rejectDuplicateExposures = project_resolve.rejectDuplicateExposures;
 const rejectDuplicateMacroProperties = project_resolve.rejectDuplicateMacroProperties;
@@ -982,7 +973,7 @@ fn parseModel(runtime: Runtime, project_dir: []const u8, model_root: []const u8,
     errdefer {
         deinitNode(runtime.allocator, &node);
     }
-    try scanSql(runtime.allocator, sql, &node, graph);
+    try project_jinja.scanSql(runtime.allocator, sql, &node, graph);
     try graph.nodes.append(runtime.allocator, node);
 }
 
@@ -1155,7 +1146,7 @@ fn appendColumnClone(graph: *Graph, package_name: []const u8, columns: *std.Arra
 
 fn resolveMacroDependencies(graph: *Graph) !void {
     for (graph.macros.items) |*macro| {
-        try scanMacroSqlForKnownMacroCalls(graph.allocator, macro.macro_sql, graph, macro.unique_id, &macro.macro_depends_on);
+        try project_jinja.scanMacroSqlForKnownMacroCalls(graph.allocator, macro.macro_sql, graph, macro.unique_id, &macro.macro_depends_on);
         sortStrings(macro.macro_depends_on.items);
     }
 }
@@ -1182,195 +1173,6 @@ fn resolveDocDescription(graph: *Graph, package_name: []const u8, description: [
     return doc.block_contents;
 }
 
-fn scanSql(allocator: std.mem.Allocator, sql: []const u8, node: *Node, graph: ?*const Graph) !void {
-    var index: usize = 0;
-    while (index + 1 < sql.len) {
-        if (sql[index] != '{') {
-            index += 1;
-            continue;
-        }
-        if (sql[index + 1] == '#') {
-            const end = std.mem.indexOfPos(u8, sql, index + 2, "#}") orelse return error.UnsupportedJinja;
-            index = end + 2;
-            continue;
-        }
-        const close = if (sql[index + 1] == '{')
-            std.mem.indexOfPos(u8, sql, index + 2, "}}")
-        else if (sql[index + 1] == '%')
-            std.mem.indexOfPos(u8, sql, index + 2, "%}")
-        else
-            null;
-        if (close) |end| {
-            try scanJinjaSpan(allocator, sql[index + 2 .. end], node, graph);
-            index = end + 2;
-            continue;
-        }
-        index += 1;
-    }
-}
-
-fn scanJinjaSpan(allocator: std.mem.Allocator, span: []const u8, node: *Node, graph: ?*const Graph) !void {
-    var i: usize = 0;
-    while (i < span.len) {
-        if (span[i] == '"' or span[i] == '\'') {
-            i = skipQuotedSpan(span, i) orelse return error.UnsupportedJinja;
-            continue;
-        }
-        if (!isIdentStart(span[i])) {
-            i += 1;
-            continue;
-        }
-        const start = i;
-        i += 1;
-        while (i < span.len and isIdentChar(span[i])) i += 1;
-        const ident = span[start..i];
-        const call = (try readJinjaCall(span, ident, i)) orelse continue;
-        const args = span[call.open + 1 .. call.close];
-
-        if (call.package_name) |package_name| {
-            if (graph) |known_graph| {
-                if (findMacroIdByPackageAndName(known_graph, package_name, call.name)) |macro_id| {
-                    try appendUnique(allocator, &node.macro_depends_on, macro_id);
-                    i = call.close + 1;
-                    continue;
-                }
-                if (hasMacroPackage(known_graph, package_name)) return error.UnresolvedMacro;
-            }
-            return error.UnsupportedJinja;
-        } else if (std.mem.eql(u8, call.name, "ref")) {
-            var strings = try parseLiteralArgs(allocator, args, error.UnsupportedDynamicRef);
-            defer strings.deinit(allocator);
-            if (!(strings.items.len == 1 or strings.items.len == 2)) return error.UnsupportedDynamicRef;
-            try node.refs.append(allocator, .{
-                .package = if (strings.items.len == 2) strings.items[0] else null,
-                .name = if (strings.items.len == 2) strings.items[1] else strings.items[0],
-            });
-        } else if (std.mem.eql(u8, call.name, "source")) {
-            var strings = try parseLiteralArgs(allocator, args, error.UnsupportedDynamicSource);
-            defer strings.deinit(allocator);
-            if (strings.items.len != 2) return error.UnsupportedDynamicSource;
-            try node.source_refs.append(allocator, .{
-                .source_name = strings.items[0],
-                .table_name = strings.items[1],
-            });
-        } else if (std.mem.eql(u8, call.name, "config")) {
-            try parseConfig(allocator, args, node);
-        } else {
-            if (graph) |known_graph| {
-                if (findMacroIdForUnqualifiedCall(known_graph, node.package_name, call.name)) |macro_id| {
-                    try appendUnique(allocator, &node.macro_depends_on, macro_id);
-                    i = call.close + 1;
-                    continue;
-                }
-            }
-            return error.UnsupportedJinja;
-        }
-        i = call.close + 1;
-    }
-}
-
-fn scanMacroSqlForKnownMacroCalls(allocator: std.mem.Allocator, sql: []const u8, graph: *const Graph, current_macro_id: []const u8, macro_depends_on: *std.ArrayList([]const u8)) !void {
-    var index: usize = 0;
-    while (index + 1 < sql.len) {
-        if (sql[index] != '{') {
-            index += 1;
-            continue;
-        }
-        if (sql[index + 1] == '#') {
-            const end = std.mem.indexOfPos(u8, sql, index + 2, "#}") orelse break;
-            index = end + 2;
-            continue;
-        }
-        const close = if (sql[index + 1] == '{')
-            std.mem.indexOfPos(u8, sql, index + 2, "}}")
-        else if (sql[index + 1] == '%')
-            std.mem.indexOfPos(u8, sql, index + 2, "%}")
-        else
-            null;
-        if (close) |end| {
-            try scanMacroSpanForKnownMacroCalls(allocator, sql[index + 2 .. end], graph, current_macro_id, macro_depends_on);
-            index = end + 2;
-            continue;
-        }
-        index += 1;
-    }
-}
-
-fn scanMacroSpanForKnownMacroCalls(allocator: std.mem.Allocator, span: []const u8, graph: *const Graph, current_macro_id: []const u8, macro_depends_on: *std.ArrayList([]const u8)) !void {
-    var i: usize = 0;
-    const current_package = packageNameFromMacroUniqueId(current_macro_id) orelse graph.project_name;
-    while (i < span.len) {
-        if (span[i] == '"' or span[i] == '\'') {
-            i = skipQuotedSpan(span, i) orelse break;
-            continue;
-        }
-        if (!isIdentStart(span[i])) {
-            i += 1;
-            continue;
-        }
-        const start = i;
-        i += 1;
-        while (i < span.len and isIdentChar(span[i])) i += 1;
-        const ident = span[start..i];
-        const call = (readJinjaCall(span, ident, i) catch break) orelse continue;
-        const macro_id = if (call.package_name) |package_name| blk: {
-            const resolved = findMacroIdByPackageAndName(graph, package_name, call.name);
-            if (resolved == null and hasMacroPackage(graph, package_name)) return error.UnresolvedMacro;
-            break :blk resolved;
-        } else findMacroIdForUnqualifiedCall(graph, current_package, call.name);
-        if (macro_id) |resolved_macro_id| {
-            if (std.mem.eql(u8, resolved_macro_id, current_macro_id)) {
-                i = call.close + 1;
-                continue;
-            }
-            try appendUnique(allocator, macro_depends_on, resolved_macro_id);
-        }
-        i = call.close + 1;
-    }
-}
-
-fn parseConfig(allocator: std.mem.Allocator, args: []const u8, node: *Node) !void {
-    if (findKeyword(args, "materialized")) |pos| {
-        if (findValueStart(args, pos + "materialized".len)) |value_pos| {
-            if (args[value_pos] != '"' and args[value_pos] != '\'') return error.UnsupportedJinja;
-            const parsed = try parseQuoted(allocator, args, value_pos);
-            node.materialized = parsed.value;
-            node.inline_materialized = true;
-        }
-    }
-    if (findKeyword(args, "tags")) |pos| {
-        if (findValueStart(args, pos + "tags".len)) |value_pos| {
-            try parseTagList(allocator, args[value_pos..], &node.tags);
-            node.inline_tags = true;
-            sortStrings(node.tags.items);
-        }
-    }
-}
-
-fn parseTagList(allocator: std.mem.Allocator, text: []const u8, tags: *std.ArrayList([]const u8)) !void {
-    var i = skipWs(text, 0);
-    if (i >= text.len) return;
-    if (text[i] == '"' or text[i] == '\'') {
-        const parsed = try parseQuoted(allocator, text, i);
-        try appendUnique(allocator, tags, parsed.value);
-        return;
-    }
-    if (text[i] != '[') return error.UnsupportedJinja;
-    i += 1;
-    while (i < text.len) {
-        i = skipWs(text, i);
-        if (i >= text.len or text[i] == ']') break;
-        if (text[i] == ',') {
-            i += 1;
-            continue;
-        }
-        if (text[i] != '"' and text[i] != '\'') return error.UnsupportedJinja;
-        const parsed = try parseQuoted(allocator, text, i);
-        try appendUnique(allocator, tags, parsed.value);
-        i = parsed.next;
-    }
-}
-
 fn appendMacroArgumentClone(graph: *Graph, arguments: *std.ArrayList(MacroArgument), source: MacroArgument) !void {
     for (arguments.items) |*existing| {
         if (std.mem.eql(u8, existing.name, source.name)) {
@@ -1425,41 +1227,3 @@ fn sortColumns(columns: []ColumnDef) void {
         }
     }.lessThan);
 }
-
-test "sql scanner extracts refs sources and config tags from jinja spans" {
-    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
-    defer arena.deinit();
-    const allocator = arena.allocator();
-
-    var node = Node{
-        .package_name = "demo",
-        .unique_id = "model.demo.customers",
-        .name = "customers",
-        .path = "customers.sql",
-        .original_file_path = "models/customers.sql",
-        .raw_code = "",
-    };
-    defer {
-        node.tags.deinit(allocator);
-        node.refs.deinit(allocator);
-        node.source_refs.deinit(allocator);
-        node.depends_on.deinit(allocator);
-        node.macro_depends_on.deinit(allocator);
-    }
-
-    try scanSql(allocator,
-        \\{{ config(materialized="table", tags=["nightly", 'core']) }}
-        \\select * from {{ ref("stg_customers") }}
-        \\union all select * from {{ source('raw', "customers") }}
-        \\select {{ "ref('not_a_dependency')" }} as literal_ref
-        \\{# {{ ref("ignored") }} #}
-    , &node, null);
-
-    try std.testing.expectEqual(@as(usize, 1), node.refs.items.len);
-    try std.testing.expectEqualStrings("stg_customers", node.refs.items[0].name);
-    try std.testing.expectEqual(@as(usize, 1), node.source_refs.items.len);
-    try std.testing.expectEqualStrings("raw", node.source_refs.items[0].source_name);
-    try std.testing.expectEqualStrings("customers", node.source_refs.items[0].table_name);
-    try std.testing.expectEqualStrings("table", node.materialized);
-    try std.testing.expectEqual(@as(usize, 2), node.tags.items.len);
-}