fixes #16 - handle byte range that starts in the middle of a grapheme

Author: severo

src/types.d.ts

@@ -1,14 +1,15 @@
 /** Error structure */
 export interface ParseError {
   /** A generalization of the error */
-  type: 'Quotes' | 'Delimiter' | 'FieldMismatch'
+  type: 'Quotes' | 'Delimiter' | 'FieldMismatch' | 'Decoding'
   /** Standardized error code */
   code:
     | 'MissingQuotes'
     | 'UndetectableDelimiter'
     | 'TooFewFields'
     | 'TooManyFields'
     | 'InvalidQuotes'
+    | 'InvalidData'
   /** Human-readable details */
   message: string
 }

src/url.ts

@@ -4,6 +4,7 @@ import { defaultChunkSize } from './options/constants'
 import { type DelimiterError, type ParseOptions, validateAndGuessParseOptions } from './options/parseOptions'
 import { parse } from './parser'
 import type { ParseResult } from './types'
+import { decode } from './utils'
 
 interface FetchOptions {
   chunkSize?: number
@@ -51,10 +52,19 @@ export async function* parseURL(
   let parseOptions: ParseOptions | undefined = undefined
   let delimiterError: DelimiterError | undefined = undefined
 
-  const decoder = new TextDecoder('utf-8', {
-    // don't strip the BOM, we handle it in the parse function
-    ignoreBOM: true,
-  })
+  /*
+   * Number of bytes skipped due to decoding errors
+   * They are only accepted at the start of the range, otherwise an exception is thrown.
+   * Note that, in the extreme case where chunkSize = 1, and multiple bytes are invalid at the start,
+   * multiple chunk iterations may be needed until a valid byte is found.
+   * If there are decoding errors, the first valid character will be considered the start of the text,
+   * and the byte offsets will be adjusted accordingly. An error will be reported in the first parsed row.
+   * It also means that the first row's byteOffset may be greater than options.firstByte.
+   */
+  const invalidData = {
+    byteCount: 0,
+    status: 'pending' as 'pending' | 'done',
+  }
   let chunkByteOffset = firstByte
   let bytes: Uint8Array<ArrayBufferLike> = new Uint8Array(0)
   while (true) {
@@ -84,20 +94,24 @@ export async function* parseURL(
       bytes = combinedBytes
     }
 
-    let consumedBytes = 0
-    const text = decoder.decode(bytes)
+    const { text, invalidByteCount } = decode(bytes, { stripInvalidBytesAtStart: invalidData.status === 'pending' })
+    // Skip the invalid bytes at the start (should only happen if pending, else invalidByteCount should be 0)
+    invalidData.byteCount += invalidByteCount
+    bytes = bytes.slice(invalidByteCount)
+    chunkByteOffset += invalidByteCount
 
     if (!parseOptions) {
       const result = validateAndGuessParseOptions(options, { text, delimitersToGuess })
       parseOptions = result.parseOptions
       delimiterError = result.error
     }
 
+    let consumedBytes = 0
     for (const result of (options.parse ?? parse)(text, {
       ...parseOptions,
       // the remaining bytes may not contain a full last row
       ignoreLastRow: true,
-      stripBOM: isFirstChunk ? stripBOM : false,
+      stripBOM: isFirstChunk ? stripBOM : false, // TODO(SL): only if firstByte + invalidByteCount === 0 ?
     })) {
       isFirstChunk = false
       consumedBytes += result.meta.byteCount
@@ -109,6 +123,17 @@ export async function* parseURL(
         result.errors.push(delimiterError)
         delimiterError = undefined
       }
+      // Add invalid byte count to the first reported row only
+      if (invalidData.status === 'pending') {
+        invalidData.status = 'done'
+        if (invalidData.byteCount > 0) {
+          result.errors.push({
+            type: 'Decoding',
+            code: 'InvalidData',
+            message: `Skipped ${invalidData.byteCount} invalid byte(s) at the start of the range`,
+          })
+        }
+      }
       // Yield the result with updated byte offset
       yield {
         ...result,
@@ -138,7 +163,12 @@ export async function* parseURL(
   }
 
   // Parse the last row (even if the remaining bytes are empty)
-  const text = decoder.decode(bytes)
+  const { text, invalidByteCount } = decode(bytes, { stripInvalidBytesAtStart: invalidData.status === 'pending' })
+  // Skip the invalid bytes at the start (should only happen if pending, else invalidByteCount should be 0)
+  invalidData.byteCount += invalidByteCount
+  bytes = bytes.slice(invalidByteCount)
+  chunkByteOffset += invalidByteCount
+
   for (const result of (options.parse ?? parse)(text, {
     ...parseOptions,
     // parse until the last byte
@@ -150,6 +180,17 @@ export async function* parseURL(
       result.errors.push(delimiterError)
       delimiterError = undefined
     }
+    // Add invalid byte count to the first reported row only
+    if (invalidData.status === 'pending') {
+      invalidData.status = 'done'
+      if (invalidData.byteCount > 0) {
+        result.errors.push({
+          type: 'Decoding',
+          code: 'InvalidData',
+          message: `Skipped ${invalidData.byteCount} invalid byte(s) at the start of the range`,
+        })
+      }
+    }
     yield {
       ...result,
       meta: {

src/utils.ts

@@ -40,3 +40,44 @@ export function escapeRegExp(string: string) {
 export function testEmptyLine(s: string[], skipEmptyLines?: 'greedy' | boolean) {
   return skipEmptyLines === 'greedy' ? s.join('').trim() === '' : 0 in s && s.length === 1 && s[0].length === 0
 }
+
+/**
+ * Decodes the given bytes using the provided decoder.
+ * @param bytes The bytes to decode.
+ * @param options The options for decoding.
+ * @param options.stripInvalidBytesAtStart Whether to strip invalid bytes at the start of the byte array.
+ * @returns The decoded text and the number of invalid bytes skipped at the start
+ */
+export function decode(bytes: Uint8Array<ArrayBufferLike>, { stripInvalidBytesAtStart }: { stripInvalidBytesAtStart?: boolean } = {}): {
+  text: string
+  invalidByteCount: number
+} {
+  const decoder = new TextDecoder('utf-8', {
+    // don't strip the BOM, we handle it in the parse function
+    ignoreBOM: true,
+    // throw on decoding errors, see https://github.com/severo/csv-range/issues/16
+    fatal: true,
+  })
+
+  if (!stripInvalidBytesAtStart) {
+    // Let the decoder throw on errors, since they should not occur anymore
+    return { text: decoder.decode(bytes), invalidByteCount: 0 }
+  }
+
+  for (let i = 0; i < bytes.length; i++) {
+    try {
+      const text = decoder.decode(bytes.subarray(i))
+      // found the first valid byte
+      return { text, invalidByteCount: i }
+    }
+    catch {
+      // still invalid, try the next byte
+      continue
+    }
+  }
+  // the byte array is empty, or all bytes are invalid
+  return {
+    text: '',
+    invalidByteCount: bytes.length,
+  }
+}

tests/url.test.ts

@@ -196,4 +196,117 @@ describe('parseURL', () => {
     // includes BOM bytes, independently of stripBOM
     expect(result[0]?.meta.byteCount).toBe(fileSize)
   })
+
+  it.for([
+    { firstByte: 0, expected: { row: ['👉🏿', '1'], charCount: 6 } },
+    // There is no way to know that '👉' and '🏿' were part of a combined emoji.
+    { firstByte: 0, lastByte: 3, expected: { row: ['👉'], charCount: 2 } },
+    { firstByte: 1, expected: { row: ['🏿', '1'], charCount: 4, invalidByteCount: 3 } },
+    { firstByte: 2, expected: { row: ['🏿', '1'], charCount: 4, invalidByteCount: 2 } },
+    { firstByte: 3, expected: { row: ['🏿', '1'], charCount: 4, invalidByteCount: 1 } },
+    { firstByte: 4, expected: { row: ['🏿', '1'], charCount: 4 } },
+    { firstByte: 4, lastByte: 7, expected: { row: ['🏿'], charCount: 2 } },
+    { firstByte: 5, expected: { row: ['', '1'], charCount: 2, invalidByteCount: 3 } },
+    { firstByte: 6, expected: { row: ['', '1'], charCount: 2, invalidByteCount: 2 } },
+    { firstByte: 7, expected: { row: ['', '1'], charCount: 2, invalidByteCount: 1 } },
+    { firstByte: 8, expected: { row: ['', '1'], charCount: 2 } },
+  ])('should support cutting 👉🏿 emoji: firstByte=$firstByte, lastByte=$lastByte', async ({ firstByte, lastByte, expected: { row, charCount, invalidByteCount } }) => {
+    // 👉🏿 uses 8 bytes in UTF-8.
+    const text = '👉🏿,1'
+    const { url, fileSize, revoke } = toUrl(text)
+    lastByte ??= fileSize - 1
+    const result = []
+    for await (const r of parseURL(url, { firstByte, lastByte, delimiter: ',', newline: '\n' })) {
+      result.push(r)
+    }
+    revoke()
+
+    const expectedByteOffset = firstByte + (invalidByteCount ?? 0)
+    expect(result.length).toBe(1)
+    expect(result).toEqual([{
+      errors: invalidByteCount === undefined
+        ? []
+        : [{
+            type: 'Decoding',
+            code: 'InvalidData',
+            message: `Skipped ${invalidByteCount} invalid byte(s) at the start of the range`,
+          }],
+      row,
+      meta: {
+        byteOffset: expectedByteOffset,
+        byteCount: lastByte - expectedByteOffset + 1,
+        charCount, // TODO(SL): define what is a "character" (UTF-16 code point? grapheme?)
+        delimiter: ',',
+        newline: '\n',
+      },
+    }])
+    expect((result[0]?.meta.byteCount ?? -Infinity) + (result[0]?.meta.byteOffset ?? -Infinity)).toBe(lastByte + 1)
+  })
+
+  it('should search invalid bytes over multiple chunks if needed', async () => {
+    const text = '👉,1'
+    const { url, fileSize, revoke } = toUrl(text)
+    const result = []
+    // There are 3 invalid bytes at start, when starting at byte 1. Using chunkSize=1 to force multiple iterations.
+    for await (const r of parseURL(url, { chunkSize: 1, firstByte: 1, lastByte: fileSize - 1, delimiter: ',', newline: '\n' })) {
+      result.push(r)
+    }
+    revoke()
+
+    expect(result.length).toBe(1)
+    expect(result).toEqual([{
+      errors: [{
+        type: 'Decoding',
+        code: 'InvalidData',
+        message: 'Skipped 3 invalid byte(s) at the start of the range',
+      }],
+      row: ['', '1'],
+      meta: {
+        byteOffset: 4,
+        byteCount: fileSize - 4,
+        charCount: 2,
+        delimiter: ',',
+        newline: '\n',
+      },
+    }])
+  })
+
+  it('should report invalid data in multiple rows', async () => {
+    const text = '👉a,b\n1,2'
+    const { url, fileSize, revoke } = toUrl(text)
+    const result = []
+    for await (const r of parseURL(url, { firstByte: 3, lastByte: fileSize - 1, delimiter: ',', newline: '\n' })) {
+      result.push(r)
+    }
+    revoke()
+    expect(result.length).toBe(2)
+    expect(result).toEqual([
+      {
+        errors: [{
+          type: 'Decoding',
+          code: 'InvalidData',
+          message: 'Skipped 1 invalid byte(s) at the start of the range',
+        }],
+        row: ['a', 'b'],
+        meta: {
+          byteOffset: 4,
+          byteCount: 4,
+          charCount: 4,
+          delimiter: ',',
+          newline: '\n',
+        },
+      },
+      {
+        errors: [],
+        row: ['1', '2'],
+        meta: {
+          byteOffset: 8,
+          byteCount: 3,
+          charCount: 3,
+          delimiter: ',',
+          newline: '\n',
+        },
+      },
+    ])
+  })
 })

tests/utils.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 
-import { escapeRegExp, testEmptyLine, toUrl } from '../src/utils'
+import { decode, escapeRegExp, testEmptyLine, toUrl } from '../src/utils'
 
 describe('toUrl', () => {
   it('creates a valid blob URL and revokes it', async () => {
@@ -75,3 +75,55 @@ describe('testEmptyLine', () => {
     expect(testEmptyLine(['data'], 'greedy')).toBe(false)
   })
 })
+
+describe('decode', () => {
+  it.for([
+    '\ufeffhello, csvremote!!!',
+    'hello, \ufeffcsvremote!!!',
+    'hello, csvremote!!!\ufeff',
+  ])('should not strip the BOM', (text) => {
+    expect(decode(new TextEncoder().encode(text))).toEqual({
+      text,
+      invalidByteCount: 0,
+    })
+  })
+
+  it('should strip invalid bytes at the start if specified', () => {
+    // Invalid UTF-8 bytes: 0xFF, 0xFE
+    const invalidBytes = new Uint8Array([0xFF, 0xFE, 0x68, 0x65, 0x6C, 0x6C, 0x6F]) // "hello"
+    expect(decode(invalidBytes, { stripInvalidBytesAtStart: true })).toEqual({
+      text: 'hello',
+      invalidByteCount: 2,
+    })
+  })
+
+  it('should throw on invalid bytes if not stripping', () => {
+    // Invalid UTF-8 bytes: 0xFF, 0xFE
+    const invalidBytes = new Uint8Array([0xFF, 0xFE, 0x68, 0x65, 0x6C, 0x6C, 0x6F]) // "hello"
+    expect(() => decode(invalidBytes)).toThrow()
+  })
+
+  it('should return all bytes as invalid if all are invalid', () => {
+    const invalidBytes = new Uint8Array([0xFF, 0xFE, 0xFF, 0xFE])
+    expect(decode(invalidBytes, { stripInvalidBytesAtStart: true })).toEqual({
+      text: '',
+      invalidByteCount: 4,
+    })
+  })
+
+  it('should handle valid UTF-8 bytes correctly', () => {
+    const validBytes = new TextEncoder().encode('Valid UTF-8 text')
+    expect(decode(validBytes, { stripInvalidBytesAtStart: true })).toEqual({
+      text: 'Valid UTF-8 text',
+      invalidByteCount: 0,
+    })
+  })
+
+  it('should handle empty byte array', () => {
+    const emptyBytes = new Uint8Array([])
+    expect(decode(emptyBytes, { stripInvalidBytesAtStart: true })).toEqual({
+      text: '',
+      invalidByteCount: 0,
+    })
+  })
+})