Add adapter-free docs generate artifacts (#59)

Author: sabino

PLAN.md

@@ -278,13 +278,15 @@ Every compatibility slice must record:
 
 The next source-grounded M1/M2 slices after macro block variant support are:
 
-1. Establish the first render-only compile boundary for selected SQL models:
-   route `dxt compile` through the Zig parser graph, apply `--select` and
-   `--exclude`, render supported `config`, literal `ref`, and literal `source`
-   calls without executing SQL, write compiled SQL under
-   `target/compiled/<project>/...`, and emit compile fields only for compiled
-   model nodes. Stop before macro execution, materializations, tests, DuckDB
-   connections, `run_results.json`, `catalog.json`, or docs generation.
+1. Extend the render-only artifact boundary to adapter-free docs generation:
+   route `dxt docs generate` through the Zig parser graph, apply `--select` and
+   `--exclude` to compiled SQL model output, render supported `config`, literal
+   `ref`, and literal `source` calls without executing SQL, write compiled SQL
+   under `target/compiled/<package>/...`, emit compile fields only for compiled
+   model nodes, write `manifest.json`, and write an empty dbt-shaped
+   `catalog.json` until adapter introspection exists. Stop before macro
+   execution, materializations, tests, DuckDB connections, non-empty catalog
+   introspection, `run_results.json`, or docs serving.
 2. Finish macro patch and namespace parity beyond the current macro artifact
    surface: parser-controlled macro argument extraction when the dbt
    `validate_macro_args` behavior is exposed, macro patch validation, macro
@@ -624,7 +626,8 @@ Exit criteria:
 - CI format validation now covers every tracked Zig source file under `src/`, including extracted `src/project/*.zig` modules, so M1A module splits remain under the same formatting gate as the root CLI files.
 - `dxt parse` now targets the supported Tier 0 subset: project name/model paths/seed paths/macro paths, project and package model path configs for literal `+materialized`, `+tags`, and model/seed `+docs.node_color`, root-project model config overrides for installed packages, SQL model discovery, CSV seed discovery, installed package SQL model and CSV seed discovery from `dbt_packages`, source discovery, installed package source discovery, exposure discovery, installed package exposure discovery, project macro discovery, dbt-shaped generic test macro and materialization macro block discovery, installed package macro discovery from `dbt_packages`, macro property YAML for project macro descriptions and arguments, project and package docs block discovery, literal `ref` to models or seeds, two-argument package refs, package-local refs in installed package models and exposures, unique installed-package fallback for unqualified refs, literal `source`, package-local sources in installed package models, unique installed-package fallback for unqualified sources, literal `doc` in project and package descriptions, inline `config(materialized=..., tags=...)`, known project/package-qualified/package-local macro call dependencies, narrow project and package YAML model properties for scalar descriptions, simple columns, tags, materialization, disabled SQL models, dbt-shaped `unique`, `not_null`, `accepted_values`, and `relationships` generic test nodes, model/test `refs` and `sources` artifact fields, dependency maps, and deterministic partial `manifest.json`. The manifest includes the v12 top-level maps needed by the M1 artifact shape and is covered by a pinned local dbt Manifest v12 schema slice. YAML generic test arguments are currently supported for scalar values plus inline and block lists required by public Jaffle Shop DuckDB-style tests.
 - `dxt ls` now lists dbt-selectable resources from the same parser graph with stable text/JSON output and basic name/FQN wildcards, tag wildcards, slash-aware `path:` wildcards, exact `package:`/`package:this`, `source:` wildcards including package-qualified source selectors, `exposure:` wildcards, `resource_type:`, `test_type:generic`, config materialization, comma intersection, whitespace union, multi-argument selector lists, repeated selector flags, leading/trailing `+` graph expansion, and exact exclude filters; macros are emitted in artifacts but not exposed as `ls` resources.
-- `dxt compile` has started as a render-only M2 boundary for the current graph subset. It loads and resolves the same Zig parser graph, applies `--select` and `--exclude`, compiles selected enabled SQL model nodes, writes compiled SQL under `target/compiled/<project>/...`, and emits `compiled`, `compiled_code`, `compiled_path`, `relation_name`, `extra_ctes`, and `extra_ctes_injected` only for compiled model nodes. The current compiler renders `config` to empty text and literal `ref`/`source` calls to deterministic quoted relation names without opening a database connection. Macro execution, materializations, tests, profiles-derived relation identity, adapters, `run_results.json`, `catalog.json`, `build`, and `docs generate` remain out of scope.
+- `dxt compile` has started as a render-only M2 boundary for the current graph subset. It loads and resolves the same Zig parser graph, applies `--select` and `--exclude`, compiles selected enabled SQL model nodes, writes compiled SQL under `target/compiled/<package>/...`, and emits `compiled`, `compiled_code`, `compiled_path`, `relation_name`, `extra_ctes`, and `extra_ctes_injected` only for compiled model nodes. The current compiler renders `config` to empty text and literal `ref`/`source` calls to deterministic quoted relation names without opening a database connection.
+- `dxt docs generate` has started as an adapter-free docs artifact boundary. It loads and resolves the same Zig parser graph, applies `--select` and `--exclude` to compiled model output, writes compiled SQL, writes `manifest.json`, and writes an empty dbt-shaped `catalog.json` because adapter relation introspection is not implemented yet. Macro execution, materializations, tests, profiles-derived relation identity, adapters, `run_results.json`, non-empty `catalog.json`, `run`, `build`, and `docs serve` remain out of scope.
 - Synthetic fixtures cover one model, model refs, seed refs, source refs, exposure refs to models and sources, combined source/model YAML, inline config/tag selection, config materialization selection, comma-intersection selection, YAML model properties and columns, emitted `unique`, `not_null`, `accepted_values`, and `relationships` generic test nodes, project macro artifacts, macro block variants, macro materialization `supported_languages`, and macro properties, configured `macro-paths` replacing the default macro directory, installed package macros with package-qualified calls and package-local macro calls, installed package models, seeds, sources, docs, exposures, package YAML model properties, root package config overrides, and package-qualified/package-local refs/sources, macro calls recorded in model and macro `depends_on.macros`, docs blocks with literal `doc` descriptions, disabled models, disabled ref diagnostics, unmatched model-property warnings, duplicate model and docs diagnostics, unsupported dynamic ref/doc diagnostics, missing doc diagnostics, malformed docs block diagnostics, unresolved package macro diagnostics, and unsupported unknown macro-call diagnostics.
 - The committed M1 public Jaffle gate lives in `scripts/check_jaffle_shop_duckdb_parse.py`. It clones a pinned public Jaffle Shop DuckDB ref into a temporary directory by default, runs the Zig `dxt` binary, validates the current M1 manifest schema slice, asserts the supported partial manifest shape with five SQL models, three CSV seeds, two docs blocks, twenty supported generic test nodes, model/test `refs` artifact fields, dependency maps, materialization/docs config, and checks representative `dxt ls` selector behavior for resource types, materialization config, wildcards, path selectors, and graph expansion. It is a developer-side Python compatibility harness only; product parse/list behavior remains implemented in Zig. Remaining M1 work includes package-provided generic tests/macros beyond the current narrow macro call surface and deeper Jaffle artifact parity.
 - Selector wildcard behavior is currently pinned to observed dbt Core 1.10 behavior. dbt Fusion preview currently differs for resource-type-prefixed wildcard selectors such as `model.<package>.*` and filename-suffix path selectors such as `path:*orders.sql`; a future Fusion-compatibility slice must decide whether to support a selector dialect switch or a compatible superset.

README.md

@@ -34,9 +34,10 @@ Implemented pre-alpha commands:
 ./zig-out/bin/dxt ls --project-dir tests/fixtures/model_ref
 ./zig-out/bin/dxt ls --project-dir tests/fixtures/model_ref --output json
 ./zig-out/bin/dxt compile --project-dir tests/fixtures/compile_basic --target-path target-dxt
+./zig-out/bin/dxt docs generate --project-dir tests/fixtures/docs_blocks --target-path target-dxt
 ```
 
-`parse`, `ls`, and `compile` currently support only the documented M1/M2 parser and render-only subset: `dbt_project.yml` name/model paths/seed paths/macro paths/target path, SQL model discovery, CSV seed discovery, installed package SQL model and CSV seed discovery from `dbt_packages`, source discovery, installed package source discovery, exposure discovery, installed package exposure discovery, project macro discovery, installed package macro discovery from `dbt_packages`, project and package docs block discovery, literal `ref` to models or seeds, two-argument package refs, package-local refs in installed package models and exposures, dbt-style fallback from unqualified refs to a unique installed-package model or seed, literal `source`, package-local sources in installed package models, dbt-style fallback from unqualified sources to a unique installed-package source, literal `doc` in descriptions, package-qualified and package-local macro dependencies, basic inline `config`, narrow project and package YAML model and macro properties, project and package model/seed `+docs.node_color`, root-project model config overrides for installed packages, simple columns, tags, materialization and disabled SQL models, dbt-shaped generic test nodes for `unique`, `not_null`, `accepted_values`, and `relationships`, model/test `refs` and `sources` artifact fields, deterministic partial `manifest.json`, and basic name/tag/path/package/resource/config materialization selectors with exact `package:`/`package:this`, comma intersections, whitespace unions, multi-argument selector lists, repeated selector flags, and graph expansion. `compile` applies selectors/excludes to enabled SQL models, writes supported compiled SQL to `target/compiled/<project>/...`, and emits compile fields for compiled models. It currently renders only `config`, literal `ref`, and literal `source` calls; macro execution, materializations, tests, `build`, and `docs generate` remain planned.
+`parse`, `ls`, `compile`, and `docs generate` currently support only the documented M1/M2 parser and render-only subset: `dbt_project.yml` name/model paths/seed paths/macro paths/target path, SQL model discovery, CSV seed discovery, installed package SQL model and CSV seed discovery from `dbt_packages`, source discovery, installed package source discovery, exposure discovery, installed package exposure discovery, project macro discovery, installed package macro discovery from `dbt_packages`, project and package docs block discovery, literal `ref` to models or seeds, two-argument package refs, package-local refs in installed package models and exposures, dbt-style fallback from unqualified refs to a unique installed-package model or seed, literal `source`, package-local sources in installed package models, dbt-style fallback from unqualified sources to a unique installed-package source, literal `doc` in descriptions, package-qualified and package-local macro dependencies, basic inline `config`, narrow project and package YAML model and macro properties, project and package model/seed `+docs.node_color`, root-project model config overrides for installed packages, simple columns, tags, materialization and disabled SQL models, dbt-shaped generic test nodes for `unique`, `not_null`, `accepted_values`, and `relationships`, model/test `refs` and `sources` artifact fields, deterministic partial `manifest.json`, and basic name/tag/path/package/resource/config materialization selectors with exact `package:`/`package:this`, comma intersections, whitespace unions, multi-argument selector lists, repeated selector flags, and graph expansion. `compile` applies selectors/excludes to enabled SQL models, writes supported compiled SQL to `target/compiled/<package>/...`, and emits compile fields for compiled models. `docs generate` uses the same render-only compiler, writes `manifest.json`, writes compiled SQL for selected enabled SQL models, and emits an adapter-free empty `catalog.json` until relation introspection exists. The compiler currently renders only `config`, literal `ref`, and literal `source` calls; macro execution, materializations, tests, `run`, `build`, non-empty catalog introspection, and `docs serve` remain planned.
 
 ## Development
 

src/project.zig

@@ -1,5 +1,6 @@
 const std = @import("std");
 const Io = std.Io;
+const catalog = @import("project/catalog.zig");
 const compiler = @import("project/compiler.zig");
 const project_fs = @import("project/fs.zig");
 const project_jinja = @import("project/jinja.zig");
@@ -125,22 +126,73 @@ pub fn compile(runtime: Runtime, options: Options, stdout: *Io.Writer, stderr: *
     const exclude = if (options.exclude) |value| try runtime.allocator.dupe(u8, value) else null;
     const selected = try selector.selectResources(runtime.allocator, &graph, null, select, exclude);
 
+    const target_dir = try targetDir(runtime, options);
+    const compile_result = try compileSelectedModels(runtime, &graph, selected, target_dir);
+    if (selected.len != 0 and !compile_result.saw_model) return error.UnsupportedCompileSelection;
+
+    const manifest_path = try pathJoin(runtime.allocator, &.{ target_dir, "manifest.json" });
+    const manifest_json = try manifest.renderManifest(runtime.allocator, &graph);
+    try std.Io.Dir.cwd().createDirPath(runtime.io, target_dir);
+    try std.Io.Dir.cwd().writeFile(runtime.io, .{ .sub_path = manifest_path, .data = manifest_json });
+    try stdout.print("Compiled {d} model(s) into {s}\n", .{
+        compile_result.count,
+        util.normalizeForDisplay(compile_result.compiled_base),
+    });
+}
+
+pub fn docsGenerate(runtime: Runtime, options: Options, stdout: *Io.Writer, stderr: *Io.Writer) !void {
+    var graph = try project_loader.loadGraph(runtime, options.project_dir, loader_callbacks);
+    defer graph.deinit();
+
+    try resolveDependencies(&graph);
+    try writeWarnings(stderr, &graph);
+
+    const select = if (options.select) |value| try runtime.allocator.dupe(u8, value) else null;
+    const exclude = if (options.exclude) |value| try runtime.allocator.dupe(u8, value) else null;
+    const selected = try selector.selectResources(runtime.allocator, &graph, null, select, exclude);
+
+    const target_dir = try targetDir(runtime, options);
+    const compile_result = try compileSelectedModels(runtime, &graph, selected, target_dir);
+
+    const manifest_path = try pathJoin(runtime.allocator, &.{ target_dir, "manifest.json" });
+    const manifest_json = try manifest.renderManifest(runtime.allocator, &graph);
+    try std.Io.Dir.cwd().createDirPath(runtime.io, target_dir);
+    try std.Io.Dir.cwd().writeFile(runtime.io, .{ .sub_path = manifest_path, .data = manifest_json });
+
+    const catalog_path = try pathJoin(runtime.allocator, &.{ target_dir, "catalog.json" });
+    const catalog_json = try catalog.renderCatalog(runtime.allocator);
+    try std.Io.Dir.cwd().writeFile(runtime.io, .{ .sub_path = catalog_path, .data = catalog_json });
+
+    try stdout.print("Generated docs artifacts for {d} compiled model(s) into {s}\n", .{
+        compile_result.count,
+        util.normalizeForDisplay(target_dir),
+    });
+}
+
+const CompileResult = struct {
+    count: usize,
+    saw_model: bool,
+    compiled_base: []const u8,
+};
+
+fn targetDir(runtime: Runtime, options: Options) ![]const u8 {
     const target_path = options.target_path orelse project_loader.graphDefaultTarget(runtime, options.project_dir) catch "target";
-    const target_dir = if (std.fs.path.isAbsolute(target_path))
-        target_path
-    else
-        try pathJoin(runtime.allocator, &.{ options.project_dir, target_path });
+    if (std.fs.path.isAbsolute(target_path)) return target_path;
+    return try pathJoin(runtime.allocator, &.{ options.project_dir, target_path });
+}
+
+fn compileSelectedModels(runtime: Runtime, graph: *Graph, selected: []const selector.SelectedResource, target_dir: []const u8) !CompileResult {
     const compiled_base = try pathJoin(runtime.allocator, &.{ target_dir, "compiled" });
     try std.Io.Dir.cwd().createDirPath(runtime.io, compiled_base);
 
     var compiled_count: usize = 0;
-    var saw_selected_resource = false;
+    var saw_selected_model = false;
     for (graph.nodes.items) |*node| {
         if (!node.enabled or !std.mem.eql(u8, node.resource_type, "model")) continue;
         if (!selectionContains(selected, node.unique_id)) continue;
-        saw_selected_resource = true;
+        saw_selected_model = true;
 
-        const compiled_code = try compiler.compileModel(runtime.allocator, &graph, node);
+        const compiled_code = try compiler.compileModel(runtime.allocator, graph, node);
         const compiled_path = try pathJoin(runtime.allocator, &.{ compiled_base, node.package_name, node.original_file_path });
         if (std.fs.path.dirname(compiled_path)) |parent| {
             try std.Io.Dir.cwd().createDirPath(runtime.io, parent);
@@ -153,19 +205,10 @@ pub fn compile(runtime: Runtime, options: Options, stdout: *Io.Writer, stderr: *
         compiled_count += 1;
     }
 
-    if (selected.len != 0 and !saw_selected_resource) return error.UnsupportedCompileSelection;
-
-    const manifest_path = try pathJoin(runtime.allocator, &.{ target_dir, "manifest.json" });
-    const manifest_json = try manifest.renderManifest(runtime.allocator, &graph);
-    try std.Io.Dir.cwd().createDirPath(runtime.io, target_dir);
-    try std.Io.Dir.cwd().writeFile(runtime.io, .{ .sub_path = manifest_path, .data = manifest_json });
-    try stdout.print("Compiled {d} model(s) into {s}\n", .{
-        compiled_count,
-        util.normalizeForDisplay(compiled_base),
-    });
+    return .{ .count = compiled_count, .saw_model = saw_selected_model, .compiled_base = compiled_base };
 }
 
-fn selectionContains(selected: []selector.SelectedResource, unique_id: []const u8) bool {
+fn selectionContains(selected: []const selector.SelectedResource, unique_id: []const u8) bool {
     for (selected) |item| {
         if (std.mem.eql(u8, item.unique_id, unique_id)) return true;
     }