fix(dxf): recode TEXT from $DWGCODEPAGE after ST_Read

apps/geolibre-desktop/src/lib/cad-encoding.ts

@@ -0,0 +1,250 @@
+/**
+ * Repair DXF string fields after DuckDB-WASM `ST_Read`.
+ *
+ * AutoCAD TEXT in R2004 and earlier is stored in `$DWGCODEPAGE`; R2007
+ * (`AC1021`) and later is UTF-8. Desktop GDAL recodes to UTF-8; WASM GDAL has
+ * no iconv, so each file byte becomes a Latin-1 character. Recode those
+ * strings — recover the original bytes, then decode with the drawing
+ * codepage — at the GeoJSON boundary. Rewriting the file as UTF-8 does not
+ * help: WASM GDAL still copies bytes as Latin-1. Binary DXF is left unchanged.
+ *
+ * Kept free of the DuckDB-WASM import so `node --test` can cover it without
+ * pulling the engine into the coverage denominator.
+ */
+
+import type { Feature, FeatureCollection, GeoJsonProperties } from "geojson";
+
+const HEADER_PROBE_BYTES = 64 * 1024;
+const BINARY_DXF_MAGIC = "AutoCAD Binary DXF";
+/** `$ACADVER` numeric suffix at which DXF switched from codepage to UTF-8. */
+const UTF8_DXF_VERSION = 1021;
+
+/**
+ * AutoCAD `$DWGCODEPAGE` → WHATWG `TextDecoder` label.
+ *
+ * `ISO8859-1` / `ISO-8859-1` / `ASCII` / `US-ASCII` are deliberately absent:
+ * WASM GDAL already turns each file byte into the matching Latin-1 code point,
+ * which *is* the correct Unicode for those codepages, so recoding has nothing
+ * to repair. Mapping them through `TextDecoder` would instead corrupt bytes
+ * 0x80–0x9F, because the WHATWG Encoding Standard aliases every `latin1` /
+ * `iso-8859-1` label to windows-1252, where that range holds printable
+ * characters rather than ISO-8859-1's C1 controls. `ANSI_1252` is a different
+ * matter and stays mapped: there the file really is windows-1252, so those
+ * bytes do need recoding.
+ */
+const CODEPAGE_LABELS: Record<string, string> = {
+  "UTF-8": "utf-8",
+  UTF8: "utf-8",
+  ANSI_936: "gb18030",
+  GBK: "gb18030",
+  GB2312: "gb18030",
+  GB18030: "gb18030",
+  ANSI_950: "big5",
+  BIG5: "big5",
+  ANSI_932: "shift_jis",
+  SHIFT_JIS: "shift_jis",
+  ANSI_949: "euc-kr",
+  ANSI_1252: "windows-1252",
+  ANSI_1250: "windows-1250",
+  ANSI_1251: "windows-1251",
+  ANSI_1253: "windows-1253",
+  ANSI_1254: "windows-1254",
+  ANSI_1255: "windows-1255",
+  ANSI_1256: "windows-1256",
+  ANSI_1257: "windows-1257",
+  ANSI_1258: "windows-1258",
+};
+
+/**
+ * Decode a Latin-1 prefix so ASCII DXF headers can be scanned.
+ *
+ * The `latin1` label is windows-1252 in disguise (see {@link CODEPAGE_LABELS}),
+ * which is harmless here: the header variables and values this scans are ASCII.
+ *
+ * @param bytes The DXF file bytes.
+ * @returns The first {@link HEADER_PROBE_BYTES} decoded as Latin-1.
+ */
+function headerLatin1(bytes: Uint8Array): string {
+  return new TextDecoder("latin1").decode(
+    bytes.subarray(0, Math.min(bytes.length, HEADER_PROBE_BYTES)),
+  );
+}
+
+/**
+ * True when the buffer is a binary DXF (not the ASCII/ANSI text form).
+ *
+ * @param bytes The file bytes.
+ * @returns True when the AutoCAD binary-DXF magic is present.
+ */
+function isBinaryDxf(bytes: Uint8Array): boolean {
+  if (bytes.length < BINARY_DXF_MAGIC.length) return false;
+  for (let i = 0; i < BINARY_DXF_MAGIC.length; i += 1) {
+    if (bytes[i] !== BINARY_DXF_MAGIC.charCodeAt(i)) return false;
+  }
+  return true;
+}
+
+/**
+ * Read one HEADER variable from an ASCII DXF prefix.
+ *
+ * @param header Latin-1 text of the file prefix.
+ * @param name The variable without `$`, such as `ACADVER`.
+ * @param groupCode The DXF group code of the value line (`1` or `3`).
+ * @returns The trimmed value, or null when the variable is absent.
+ */
+function readDxfTaggedValue(header: string, name: string, groupCode: number): string | null {
+  const pattern = new RegExp(
+    `\\$${name}[ \\t]*\\r?\\n[ \\t]*0*${groupCode}[ \\t]*\\r?\\n([^\\r\\n]+)`,
+    "i",
+  );
+  const value = pattern.exec(header)?.[1]?.trim();
+  return value || null;
+}
+
+/**
+ * True when `$ACADVER` is R2007 or later (UTF-8 DXF).
+ *
+ * @param acadver A value such as `AC1021`.
+ * @returns True when the numeric suffix is >= {@link UTF8_DXF_VERSION}.
+ */
+function isUtf8DxfVersion(acadver: string): boolean {
+  const match = /^AC(\d+)$/i.exec(acadver.trim());
+  return match !== null && Number(match[1]) >= UTF8_DXF_VERSION;
+}
+
+/**
+ * Map an AutoCAD codepage name to a `TextDecoder` label, if supported.
+ *
+ * @param codepage An uppercased `$DWGCODEPAGE` value.
+ * @returns A WHATWG encoding label, or null when unknown/unsupported.
+ */
+function decoderLabelForCodepage(codepage: string): string | null {
+  const mapped = CODEPAGE_LABELS[codepage];
+  if (!mapped) return null;
+  try {
+    new TextDecoder(mapped);
+    return mapped;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Normalize a `$DWGCODEPAGE` value to a CODEPAGE_LABELS key.
+ *
+ * @param raw A token such as `ansi_936` or `utf8`.
+ * @returns The uppercased key, with `UTF8` folded to `UTF-8`.
+ */
+function normalizeCodepageToken(raw: string): string {
+  const upper = raw.trim().toUpperCase();
+  return upper === "UTF8" ? "UTF-8" : upper;
+}
+
+/**
+ * Read the drawing codepage from an ASCII DXF header.
+ *
+ * R2007+ (`$ACADVER` >= AC1021) is UTF-8 even when `$DWGCODEPAGE` still names
+ * a legacy ANSI_* page. Earlier versions use `$DWGCODEPAGE`. Binary DXF and
+ * files with neither variable are left unlabelled.
+ *
+ * @param bytes The file bytes (must be read before DuckDB detaches the buffer).
+ * @returns An AutoCAD codepage name, or null when recoding should not run.
+ */
+export function readDxfCodepage(bytes: Uint8Array): string | null {
+  if (bytes.length === 0 || isBinaryDxf(bytes)) return null;
+  const header = headerLatin1(bytes);
+  const acadver = readDxfTaggedValue(header, "ACADVER", 1);
+  if (acadver && isUtf8DxfVersion(acadver)) return "UTF-8";
+  const codepage = readDxfTaggedValue(header, "DWGCODEPAGE", 3);
+  return codepage ? normalizeCodepageToken(codepage) : null;
+}
+
+/**
+ * Rebuild the DXF bytes WASM GDAL copied into a JS string as Latin-1.
+ *
+ * @param value A DuckDB string field.
+ * @returns The original bytes, or null when `value` is already real Unicode.
+ */
+function duckDbStringBytes(value: string): Uint8Array | null {
+  const bytes = new Uint8Array(value.length);
+  for (let i = 0; i < value.length; i += 1) {
+    const code = value.charCodeAt(i);
+    if (code > 255) return null;
+    bytes[i] = code;
+  }
+  return bytes;
+}
+
+/**
+ * Recode one DXF string with an already-constructed decoder.
+ *
+ * @param value The raw DuckDB/OGR string (or already-correct Unicode).
+ * @param decoder A `TextDecoder` for the drawing codepage.
+ * @returns Unicode text, or `value` when recoding does not apply.
+ */
+function recodeWithDecoder(value: string, decoder: TextDecoder): string {
+  if (!value) return value;
+  const bytes = duckDbStringBytes(value);
+  if (!bytes) return value;
+  try {
+    return decoder.decode(bytes);
+  } catch {
+    return value;
+  }
+}
+
+/**
+ * Recode one DXF attribute that WASM GDAL exposed as Latin-1 mojibake.
+ *
+ * @param value The raw DuckDB/OGR string (or already-correct Unicode).
+ * @param codepage From {@link readDxfCodepage}, or null.
+ * @returns Unicode text, or `value` when recoding does not apply.
+ */
+export function recodeCadString(value: string, codepage: string | null): string {
+  if (!value || !codepage) return value;
+  const label = decoderLabelForCodepage(codepage);
+  if (!label) return value;
+  return recodeWithDecoder(value, new TextDecoder(label, { fatal: true }));
+}
+
+/**
+ * Recode string properties on one feature.
+ *
+ * @param properties The feature properties object, or null.
+ * @param decoder A `TextDecoder` for the drawing codepage.
+ * @returns Properties with DXF strings recoded; non-strings left unchanged.
+ */
+function recodeCadProperties(
+  properties: GeoJsonProperties,
+  decoder: TextDecoder,
+): GeoJsonProperties {
+  if (!properties) return properties;
+  const next: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(properties)) {
+    next[key] = typeof value === "string" ? recodeWithDecoder(value, decoder) : value;
+  }
+  return next;
+}
+
+/**
+ * Recode string properties on every feature in a DXF-derived collection.
+ *
+ * @param collection The FeatureCollection `ST_Read` materialized.
+ * @param codepage From {@link readDxfCodepage}, or null.
+ * @returns A new collection when recoding ran; `collection` when it did not.
+ */
+export function recodeCadFeatureCollection(
+  collection: FeatureCollection,
+  codepage: string | null,
+): FeatureCollection {
+  const label = codepage ? decoderLabelForCodepage(codepage) : null;
+  if (!label) return collection;
+  const decoder = new TextDecoder(label, { fatal: true });
+  return {
+    ...collection,
+    features: collection.features.map((feature: Feature) => ({
+      ...feature,
+      properties: recodeCadProperties(feature.properties, decoder),
+    })),
+  };
+}

apps/geolibre-desktop/src/lib/duckdb-vector-loader.ts

@@ -15,6 +15,7 @@ import {
   wkbRowsToFeatureCollection,
 } from "./duckdb-geometry";
 import { confirmLargeDataset, type DuckDbVectorLoadOptions } from "./duckdb-vector-guard";
+import { readDxfCodepage, recodeCadFeatureCollection } from "./cad-encoding";
 import { ensureGpkgFeatureCount } from "./gpkg-ogr-contents";
 import { isLikelyGeoPackage, loadGeoPackageVectorFile } from "./gpkg-reader";
 import { prjSidecarCrs } from "./prj-sidecar";
@@ -606,6 +607,10 @@ export async function loadDuckDbVectorFile(
     // Inside the try so the finally still closes the connection if it throws.
     // `prjSidecarCrs` is `.shp`-scoped, so a non-shapefile's siblings are safe.
     const prjCrs = prjSidecarCrs(file);
+    // Read $DWGCODEPAGE / $ACADVER before registerFileBuffer transfers (and
+    // detaches) the bytes. WASM GDAL has no iconv, so DXF TEXT is recoded
+    // after ST_Read. Other formats skip this (null → no-op).
+    const dxfCodepage = file.extension === "dxf" ? readDxfCodepage(file.data) : null;
 
     await registerVectorFileBuffers(db, file);
     await ensureSpatialExtension(
@@ -655,7 +660,10 @@ export async function loadDuckDbVectorFile(
       );
       // Features may carry a null geometry; the app's layer model treats them
       // as a regular FeatureCollection and the map ignores null geometries.
-      return toFeatureCollection(rowsFromResult(result), detected.column) as FeatureCollection;
+      return recodeCadFeatureCollection(
+        toFeatureCollection(rowsFromResult(result), detected.column) as FeatureCollection,
+        dxfCodepage,
+      );
     } catch (error) {
       // DuckDB Spatial's WKB reader rejects surface geometries (TIN /
       // PolyhedralSurface), which its bundled GDAL emits for ESRI MultiPatch
@@ -677,7 +685,15 @@ export async function loadDuckDbVectorFile(
       if (isParquetExtension(file.extension) || !isSurfaceError) {
         throw error;
       }
-      return loadViaKeepWkbFallback(db, file, options, sourceCrs, error, guardConfirmed);
+      return loadViaKeepWkbFallback(
+        db,
+        file,
+        options,
+        sourceCrs,
+        error,
+        guardConfirmed,
+        dxfCodepage,
+      );
     }
   } finally {
     await connection.close();
@@ -700,6 +716,10 @@ export async function loadDuckDbVectorFile(
  * @param guardConfirmed Whether the normal path already confirmed the
  *   large-dataset guard; when false (the error fired on the count guard) it is
  *   re-run here so a huge file is not loaded without confirmation.
+ * @param dxfCodepage The drawing codepage the normal path read from the DXF
+ *   header, or null. A DXF with a 3DFACE/PolyfaceMesh entity can reach this
+ *   fallback too, so its TEXT is recoded here as well; without it the
+ *   attributes would keep the Latin-1 mojibake the normal path repairs.
  */
 async function loadViaKeepWkbFallback(
   db: duckdb.AsyncDuckDB,
@@ -708,6 +728,7 @@ async function loadViaKeepWkbFallback(
   sourceCrs: string | null,
   originalError: unknown,
   guardConfirmed: boolean,
+  dxfCodepage: string | null,
 ): Promise<FeatureCollection> {
   // Read on a fresh connection: re-running ST_Read on the connection that
   // already scanned the file trips a "Missing DB manager" GDAL assertion in the
@@ -754,8 +775,14 @@ async function loadViaKeepWkbFallback(
     }
     // The decoded geometry is in the file's own CRS; reproject to WGS84 with the
     // same source CRS the normal path resolved. Reuses the shared ST_Transform
-    // path, which handles the MultiPolygon the TIN decoded to.
-    return reprojectFeatureCollectionToWgs84(collection, sourceCrs);
+    // path, which handles the MultiPolygon the TIN decoded to. Recode first, at
+    // this `ST_Read` boundary: reprojection re-reads the collection as GeoJSON,
+    // which OGR already treats as UTF-8, so it is not the layer that mangled
+    // the strings and must not be handed mojibake to round-trip.
+    return reprojectFeatureCollectionToWgs84(
+      recodeCadFeatureCollection(collection, dxfCodepage),
+      sourceCrs,
+    );
   } finally {
     await connection.close();
   }