Workflows graph extractor (#455)

---------

Signed-off-by: Karthik Kalyanaraman <karthik@scale3labs.com>

Author: karthikscale3

.changeset/huge-rabbits-travel.md

@@ -0,0 +1,14 @@
+---
+"@workflow/world-postgres": patch
+"@workflow/world-local": patch
+"@workflow/sveltekit": patch
+"@workflow/builders": patch
+"@workflow/nitro": patch
+"@workflow/utils": patch
+"@workflow/world": patch
+"@workflow/core": patch
+"@workflow/next": patch
+"@workflow/web": patch
+---
+
+Added Control Flow Graph extraction from Workflows and extended manifest.json's schema to incorporate the graph structure into it. Refactored manifest generation to pass manifest as a parameter instead of using instance state. Add e2e tests for manifest validation across all builders.

packages/builders/src/base-builder.ts

@@ -13,6 +13,7 @@ import { createDiscoverEntriesPlugin } from './discover-entries-esbuild-plugin.j
 import { createNodeModuleErrorPlugin } from './node-module-esbuild-plugin.js';
 import { createSwcPlugin } from './swc-esbuild-plugin.js';
 import type { WorkflowConfig } from './types.js';
+import { extractWorkflowGraphs } from './workflows-extractor.js';
 
 const enhancedResolve = promisify(enhancedResolveOriginal);
 
@@ -280,6 +281,7 @@ export abstract class BaseBuilder {
    * Steps have full Node.js runtime access and handle side effects, API calls, etc.
    *
    * @param externalizeNonSteps - If true, only bundles step entry points and externalizes other code
+   * @returns Build context (for watch mode) and the collected workflow manifest
    */
   protected async createStepsBundle({
     inputFiles,
@@ -295,16 +297,17 @@ export abstract class BaseBuilder {
     outfile: string;
     format?: 'cjs' | 'esm';
     externalizeNonSteps?: boolean;
-  }): Promise<esbuild.BuildContext | undefined> {
+  }): Promise<{
+    context: esbuild.BuildContext | undefined;
+    manifest: WorkflowManifest;
+  }> {
     // These need to handle watching for dev to scan for
     // new entries and changes to existing ones
-    const { discoveredSteps: stepFiles } = await this.discoverEntries(
-      inputFiles,
-      dirname(outfile)
-    );
+    const { discoveredSteps: stepFiles, discoveredWorkflows: workflowFiles } =
+      await this.discoverEntries(inputFiles, dirname(outfile));
 
     // log the step files for debugging
-    await this.writeDebugFile(outfile, { stepFiles });
+    await this.writeDebugFile(outfile, { stepFiles, workflowFiles });
 
     const stepsBundleStart = Date.now();
     const workflowManifest: WorkflowManifest = {};
@@ -326,6 +329,7 @@ export abstract class BaseBuilder {
 
     const combinedStepFiles: string[] = [
       ...stepFiles,
+      ...workflowFiles,
       ...(resolvedBuiltInSteps
         ? [
             resolvedBuiltInSteps,
@@ -337,6 +341,8 @@ export abstract class BaseBuilder {
 
     // Create a virtual entry that imports all files. All step definitions
     // will get registered thanks to the swc transform.
+    // We also import workflow files so their metadata is collected by the SWC plugin,
+    // even though they'll be externalized from the final bundle.
     const imports = combinedStepFiles
       .map((file) => {
         // Normalize both paths to forward slashes before calling relative()
@@ -420,23 +426,14 @@ export abstract class BaseBuilder {
     this.logEsbuildMessages(stepsResult, 'steps bundle creation');
     console.log('Created steps bundle', `${Date.now() - stepsBundleStart}ms`);
 
-    const partialWorkflowManifest = {
-      steps: workflowManifest.steps,
-    };
-    // always write to debug file
-    await this.writeDebugFile(
-      join(dirname(outfile), 'manifest'),
-      partialWorkflowManifest,
-      true
-    );
-
     // Create .gitignore in .swc directory
     await this.createSwcGitignore();
 
     if (this.config.watch) {
-      return esbuildCtx;
+      return { context: esbuildCtx, manifest: workflowManifest };
     }
     await esbuildCtx.dispose();
+    return { context: undefined, manifest: workflowManifest };
   }
 
   /**
@@ -556,16 +553,6 @@ export abstract class BaseBuilder {
       `${Date.now() - bundleStartTime}ms`
     );
 
-    const partialWorkflowManifest = {
-      workflows: workflowManifest.workflows,
-    };
-
-    await this.writeDebugFile(
-      join(dirname(outfile), 'manifest'),
-      partialWorkflowManifest,
-      true
-    );
-
     if (this.config.workflowManifestPath) {
       const resolvedPath = resolve(
         process.cwd(),
@@ -917,4 +904,107 @@ export const OPTIONS = handler;`;
       // We're intentionally silently ignoring this error - creating .gitignore isn't critical
     }
   }
+
+  /**
+   * Creates a manifest JSON file containing step/workflow metadata
+   * and graph data for visualization.
+   */
+  protected async createManifest({
+    workflowBundlePath,
+    manifestDir,
+    manifest,
+  }: {
+    workflowBundlePath: string;
+    manifestDir: string;
+    manifest: WorkflowManifest;
+  }): Promise<void> {
+    const buildStart = Date.now();
+    console.log('Creating manifest...');
+
+    try {
+      const workflowGraphs = await extractWorkflowGraphs(workflowBundlePath);
+
+      const steps = this.convertStepsManifest(manifest.steps);
+      const workflows = this.convertWorkflowsManifest(
+        manifest.workflows,
+        workflowGraphs
+      );
+
+      const output = { version: '1.0.0', steps, workflows };
+
+      await mkdir(manifestDir, { recursive: true });
+      await writeFile(
+        join(manifestDir, 'manifest.json'),
+        JSON.stringify(output, null, 2)
+      );
+
+      const stepCount = Object.values(steps).reduce(
+        (acc, s) => acc + Object.keys(s).length,
+        0
+      );
+      const workflowCount = Object.values(workflows).reduce(
+        (acc, w) => acc + Object.keys(w).length,
+        0
+      );
+
+      console.log(
+        `Created manifest with ${stepCount} step(s) and ${workflowCount} workflow(s)`,
+        `${Date.now() - buildStart}ms`
+      );
+    } catch (error) {
+      console.warn(
+        'Failed to create manifest:',
+        error instanceof Error ? error.message : String(error)
+      );
+    }
+  }
+
+  private convertStepsManifest(
+    steps: WorkflowManifest['steps']
+  ): Record<string, Record<string, { stepId: string }>> {
+    const result: Record<string, Record<string, { stepId: string }>> = {};
+    if (!steps) return result;
+
+    for (const [filePath, entries] of Object.entries(steps)) {
+      result[filePath] = {};
+      for (const [name, data] of Object.entries(entries)) {
+        result[filePath][name] = { stepId: data.stepId };
+      }
+    }
+    return result;
+  }
+
+  private convertWorkflowsManifest(
+    workflows: WorkflowManifest['workflows'],
+    graphs: Record<
+      string,
+      Record<string, { graph: { nodes: any[]; edges: any[] } }>
+    >
+  ): Record<
+    string,
+    Record<
+      string,
+      { workflowId: string; graph: { nodes: any[]; edges: any[] } }
+    >
+  > {
+    const result: Record<
+      string,
+      Record<
+        string,
+        { workflowId: string; graph: { nodes: any[]; edges: any[] } }
+      >
+    > = {};
+    if (!workflows) return result;
+
+    for (const [filePath, entries] of Object.entries(workflows)) {
+      result[filePath] = {};
+      for (const [name, data] of Object.entries(entries)) {
+        result[filePath][name] = {
+          workflowId: data.workflowId,
+          graph: graphs[filePath]?.[name]?.graph || { nodes: [], edges: [] },
+        };
+      }
+    }
+    return result;
+  }
 }

packages/builders/src/standalone.ts

@@ -18,10 +18,21 @@ export class StandaloneBuilder extends BaseBuilder {
       tsBaseUrl: tsConfig.baseUrl,
       tsPaths: tsConfig.paths,
     };
-    await this.buildStepsBundle(options);
+    const manifest = await this.buildStepsBundle(options);
     await this.buildWorkflowsBundle(options);
     await this.buildWebhookFunction();
 
+    // Build unified manifest from workflow bundle
+    const workflowBundlePath = this.resolvePath(
+      this.config.workflowsBundlePath
+    );
+    const manifestDir = this.resolvePath('.well-known/workflow/v1');
+    await this.createManifest({
+      workflowBundlePath,
+      manifestDir,
+      manifest,
+    });
+
     await this.createClientLibrary();
   }
 
@@ -33,18 +44,20 @@ export class StandaloneBuilder extends BaseBuilder {
     inputFiles: string[];
     tsBaseUrl?: string;
     tsPaths?: Record<string, string[]>;
-  }): Promise<void> {
+  }) {
     console.log('Creating steps bundle at', this.config.stepsBundlePath);
 
     const stepsBundlePath = this.resolvePath(this.config.stepsBundlePath);
     await this.ensureDirectory(stepsBundlePath);
 
-    await this.createStepsBundle({
+    const { manifest } = await this.createStepsBundle({
       outfile: stepsBundlePath,
       inputFiles,
       tsBaseUrl,
       tsPaths,
     });
+
+    return manifest;
   }
 
   private async buildWorkflowsBundle({

packages/builders/src/vercel-build-output-api.ts

@@ -20,11 +20,19 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder {
       tsBaseUrl: tsConfig.baseUrl,
       tsPaths: tsConfig.paths,
     };
-    await this.buildStepsFunction(options);
+    const manifest = await this.buildStepsFunction(options);
     await this.buildWorkflowsFunction(options);
     await this.buildWebhookFunction(options);
     await this.createBuildOutputConfig(outputDir);
 
+    // Generate unified manifest
+    const workflowBundlePath = join(workflowGeneratedDir, 'flow.func/index.js');
+    await this.createManifest({
+      workflowBundlePath,
+      manifestDir: workflowGeneratedDir,
+      manifest,
+    });
+
     await this.createClientLibrary();
   }
 
@@ -38,13 +46,13 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder {
     workflowGeneratedDir: string;
     tsBaseUrl?: string;
     tsPaths?: Record<string, string[]>;
-  }): Promise<void> {
+  }) {
     console.log('Creating Vercel Build Output API steps function');
     const stepsFuncDir = join(workflowGeneratedDir, 'step.func');
     await mkdir(stepsFuncDir, { recursive: true });
 
     // Create steps bundle
-    await this.createStepsBundle({
+    const { manifest } = await this.createStepsBundle({
       inputFiles,
       outfile: join(stepsFuncDir, 'index.js'),
       tsBaseUrl,
@@ -57,6 +65,8 @@ export class VercelBuildOutputAPIBuilder extends BaseBuilder {
       shouldAddSourcemapSupport: true,
       experimentalTriggers: [STEP_QUEUE_TRIGGER],
     });
+
+    return manifest;
   }
 
   private async buildWorkflowsFunction({