JSON schema conversion benchmarks

.github/workflows/bench-pr.yml

@@ -148,8 +148,31 @@ jobs:
           name: bench-results
           path: bench/bench.json
 
+  bench-json-schema:
+    needs: build-schemas
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.head_ref }}
+      - name: Download schemas
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: schemas-build
+          path: schemas/dist/
+      - name: Setup environment
+        uses: ./.github/actions/environments
+      - name: Run JSON schema benchmark
+        run: pnpm bench:json-schema
+      - name: Upload JSON schema results
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: json-schema-results
+          path: bench/json-schema.json
+
   collect-results:
-    needs: [gate, bench-stack, bench-download, bench-bench]
+    needs: [gate, bench-stack, bench-download, bench-bench, bench-json-schema]
     if: needs.gate.outputs.should_run == 'true' && github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
@@ -173,6 +196,11 @@ jobs:
             with:
               name: download-results
               path: .
+          - name: Download JSON schema results
+            uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+            with:
+              name: json-schema-results
+              path: bench
 
       - name: Derive short commit SHA
         env:

.github/workflows/bench.yml

@@ -111,8 +111,28 @@ jobs:
           name: bench-results
           path: bench/bench.json
 
+  bench-json-schema:
+    needs: build-schemas
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Download schemas
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: schemas-build
+          path: schemas/dist/
+      - name: Setup environment
+        uses: ./.github/actions/environments
+      - name: Run JSON schema benchmark
+        run: pnpm bench:json-schema
+      - name: Upload JSON schema results
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: json-schema-results
+          path: bench/json-schema.json
+
   collect-results:
-    needs: [bench-stack, bench-download, bench-bench]
+    needs: [bench-stack, bench-download, bench-bench, bench-json-schema]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -132,6 +152,12 @@ jobs:
             with:
               name: download-results
               path: .
+          - name: Download JSON schema results
+            uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+            with:
+              name: json-schema-results
+              path: bench
+
       - name: Derive short commit SHA
         env:
           SOURCE_SHA: ${{ github.sha }}

.oxfmtrc.jsonc

@@ -4,6 +4,7 @@
     "bench/bench.json",
     "bench/download.json",
     "bench/stack.json",
+    "bench/json-schema.json",
     "schemas/libraries/**/download_compiled/**",
     "schemas/libraries/**/*.gen.ts",
     "website/**/routeTree.gen.ts",

bench/json-schema.d.json.ts

@@ -0,0 +1,5 @@
+import type { JsonSchemaBenchResults } from ".";
+
+declare const results: JsonSchemaBenchResults;
+
+export default results;

bench/oxlint.config.ts

@@ -4,7 +4,17 @@ import { baseConfig } from "../oxlint.config.ts";
 
 export default defineConfig({
   extends: [baseConfig],
+  jsPlugins: [
+    ...baseConfig.jsPlugins,
+    { name: "no-relative", specifier: "eslint-plugin-no-relative-import-paths" },
+  ],
   env: {
     node: true,
   },
+  rules: {
+    "no-relative/no-relative-import-paths": [
+      "error",
+      { allowSameFolder: true, rootDir: "bench/src", prefix: "#src" },
+    ],
+  },
 });

bench/package.json

@@ -4,14 +4,19 @@
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
+  "imports": {
+    "#src/*": "./src/*"
+  },
   "exports": {
     ".": "./src/index.ts",
     "./bench.json": "./bench.json",
     "./download.json": "./download.json",
+    "./json-schema.json": "./json-schema.json",
     "./stack.json": "./stack.json"
   },
   "scripts": {
     "bench": "node ./src/scripts/bench/orchestrate.ts",
+    "bench:json-schema": "node ./src/scripts/json-schema/orchestrate.ts",
     "download": "node ./src/scripts/download.ts",
     "stack": "node ./src/scripts/stack.ts",
     "typecheck": "tsc"

bench/src/bench/registry.ts

@@ -2,29 +2,39 @@ import type {
   BaseBenchmarkConfig,
   BenchmarksConfig,
   ErrorType,
+  JsonSchemaDirection,
+  JsonSchemaTarget,
   OptimizeType,
   StringFormat,
 } from "@schema-benchmarks/schemas";
 import type { Compute, DistributiveOmit, OneOf, Satisfies } from "@schema-benchmarks/utils";
 
-import type { DataType } from "../results/types.ts";
+import type { DataType } from "#src/results/types.ts";
 
-export type BenchmarkType = Exclude<keyof BenchmarksConfig, "library" | "stack">;
+export type BenchmarkType = Exclude<keyof BenchmarksConfig, "library" | "stack" | "jsonSchema">;
 
 interface BaseBenchInfo extends DistributiveOmit<BaseBenchmarkConfig, "optimizeType"> {
   libraryName: string;
   version: string;
+}
+
+/** Every case but the JSON schema one, where how a library validates says nothing. */
+interface RuntimeBenchInfo {
   optimizeType: OptimizeType;
 }
 
 type BenchInfoByType = Satisfies<
   {
-    initialization: unknown;
-    validation: { dataType: DataType };
-    parsing: { dataType: DataType; errorType: ErrorType };
-    standard: { dataType: DataType; errorType: ErrorType };
-    string: { stringFormat: StringFormat; dataType: DataType };
-    codec: { codecType: "encode" | "decode"; codecId: string; acceptsUnknown?: boolean };
+    initialization: RuntimeBenchInfo;
+    validation: RuntimeBenchInfo & { dataType: DataType };
+    parsing: RuntimeBenchInfo & { dataType: DataType; errorType: ErrorType };
+    standard: RuntimeBenchInfo & { dataType: DataType; errorType: ErrorType };
+    string: RuntimeBenchInfo & { stringFormat: StringFormat; dataType: DataType };
+    codec: RuntimeBenchInfo & {
+      codecType: "encode" | "decode";
+      codecId: string;
+      acceptsUnknown?: boolean;
+    };
   },
   Record<BenchmarkType, unknown>
 >;
@@ -35,8 +45,34 @@ export type BenchmarkConfigEntry = OneOf<
   }[BenchmarkType]
 >;
 
-export class CaseRegistry extends Map<string, BenchmarkConfigEntry> {
-  add(entry: BenchmarkConfigEntry) {
+export type JsonSchemaBenchmarkType = keyof NonNullable<BenchmarksConfig["jsonSchema"]>;
+
+export type JsonSchemaInfoByType = Satisfies<
+  {
+    conversion:
+      | {
+          conversionType: "toJson";
+          target: JsonSchemaTarget;
+          direction: JsonSchemaDirection;
+          jsonSchema: string;
+        }
+      | {
+          conversionType: "fromJson";
+        };
+  },
+  Record<JsonSchemaBenchmarkType, unknown>
+>;
+
+export type JsonSchemaBenchmarkConfigEntry = OneOf<
+  {
+    [Type in JsonSchemaBenchmarkType]: Compute<
+      { type: Type } & BaseBenchInfo & JsonSchemaInfoByType[Type]
+    >;
+  }[JsonSchemaBenchmarkType]
+>;
+
+export class Registry<Entry> extends Map<string, Entry> {
+  add(entry: Entry) {
     const id = crypto.randomUUID();
     this.set(id, entry);
     return id;

bench/src/results/types.ts

@@ -1,5 +1,7 @@
 import {
   errorTypeSchema,
+  jsonSchemaDirectionSchema,
+  jsonSchemaTargetSchema,
   optimizeTypeSchema,
   stringFormatSchema,
 } from "@schema-benchmarks/schemas";
@@ -18,44 +20,111 @@ export const baseBenchResultSchema = v.object({
   snippet: v.string(),
   throws: v.optional(v.boolean()),
   mean: v.number(),
-  optimizeType: optimizeTypeSchema,
 });
 export type BaseBenchResult = v.InferOutput<typeof baseBenchResultSchema>;
 
-export const initializationResultSchema = v.object({
+export const runtimeBenchResultSchema = v.object({
   ...baseBenchResultSchema.entries,
+  optimizeType: optimizeTypeSchema,
+});
+export type RuntimeBenchResult = v.InferOutput<typeof runtimeBenchResultSchema>;
+
+export const initializationResultSchema = v.object({
+  ...runtimeBenchResultSchema.entries,
   type: v.literal("initialization"),
 });
 export type InitializationResult = v.InferOutput<typeof initializationResultSchema>;
 
 export const validationResultSchema = v.object({
-  ...baseBenchResultSchema.entries,
+  ...runtimeBenchResultSchema.entries,
   type: v.literal("validation"),
 });
 export type ValidationResult = v.InferOutput<typeof validationResultSchema>;
 
 export const parsingResultSchema = v.object({
-  ...baseBenchResultSchema.entries,
+  ...runtimeBenchResultSchema.entries,
   type: v.literal("parsing"),
   errorType: errorTypeSchema,
 });
 export type ParsingResult = v.InferOutput<typeof parsingResultSchema>;
 
 const standardResultSchema = v.object({
-  ...baseBenchResultSchema.entries,
+  ...runtimeBenchResultSchema.entries,
   errorType: errorTypeSchema,
   type: v.literal("standard"),
 });
 export type StandardResult = v.InferOutput<typeof standardResultSchema>;
 
+const jsonSchemaSourceResultSchema = v.union([
+  v.picklist(["native", "opt-in"]),
+  v.object({
+    type: v.literal("package"),
+    package: v.string(),
+  }),
+]);
+export type JsonSchemaSourceResult = v.InferOutput<typeof jsonSchemaSourceResultSchema>;
+
+const schemaConversionToJsonResultSchema = v.object({
+  ...v.omit(baseBenchResultSchema, ["throws"]).entries,
+  target: jsonSchemaTargetSchema,
+  direction: jsonSchemaDirectionSchema,
+  /** The JSON schema the library generated, so it can be compared with the others. */
+  jsonSchema: v.string(),
+});
+export type SchemaToJsonResult = v.InferOutput<typeof schemaConversionToJsonResultSchema>;
+
+const schemaConversionFromJsonResultSchema = v.object({
+  ...v.omit(baseBenchResultSchema, ["throws"]).entries,
+});
+export type SchemaFromJsonResult = v.InferOutput<typeof schemaConversionFromJsonResultSchema>;
+
+export type JsonSchemaConversionResult = OneOf<SchemaToJsonResult | SchemaFromJsonResult>;
+
+export const jsonSchemaSupportMatrixSchema = v.object(
+  v.entriesFromList(
+    jsonSchemaTargetSchema.options,
+    v.optional(
+      v.object(v.entriesFromList(jsonSchemaDirectionSchema.options, v.optional(v.string()))),
+    ),
+  ),
+);
+export type JsonSchemaSupportMatrix = v.InferOutput<typeof jsonSchemaSupportMatrixSchema>;
+
+export const jsonSchemaSupportMatricesSchema = v.record(
+  v.string(),
+  v.object({
+    version: v.string(),
+    source: jsonSchemaSourceResultSchema,
+    standardJsonSchema: v.optional(jsonSchemaSourceResultSchema),
+    matrix: jsonSchemaSupportMatrixSchema,
+  }),
+);
+export type JsonSchemaSupportMatrices = v.InferOutput<typeof jsonSchemaSupportMatricesSchema>;
+
+export const jsonSchemaBenchResultsSchema = v.object({
+  conversion: v.object({
+    toJson: v.array(schemaConversionToJsonResultSchema),
+    fromJson: v.array(schemaConversionFromJsonResultSchema),
+    toJsonSupport: jsonSchemaSupportMatricesSchema,
+  }),
+});
+export type JsonSchemaBenchResults = v.InferOutput<typeof jsonSchemaBenchResultsSchema>;
+export const getEmptyJsonSchemaResults = (): JsonSchemaBenchResults => ({
+  conversion: {
+    toJson: [],
+    fromJson: [],
+    toJsonSupport: {},
+  },
+});
+
 const stringResultSchema = v.object({
-  ...baseBenchResultSchema.entries,
+  ...runtimeBenchResultSchema.entries,
   type: v.literal("string"),
 });
 export type StringResult = v.InferOutput<typeof stringResultSchema>;
 
 export const codecResultSchema = v.object({
-  ...v.omit(baseBenchResultSchema, ["snippet", "mean"]).entries,
+  ...v.omit(runtimeBenchResultSchema, ["snippet", "mean"]).entries,
   encode: v.object({
     snippet: v.string(),
     mean: v.number(),
@@ -70,7 +139,7 @@ export const codecResultSchema = v.object({
 
 export type CodecResult = v.InferOutput<typeof codecResultSchema>;
 
-export type BenchResult = OneOf<
+export type RuntimeResult = OneOf<
   InitializationResult | ValidationResult | ParsingResult | StandardResult | StringResult
 >;