copy the core of Papaparse as an iterator on rows

Author: severo

src/chunk.ts

@@ -1,56 +1,85 @@
-import { getDelimiter } from './delimiter'
-import { getNewline } from './newline'
-import { getQuote } from './quote'
-
-export interface ChunkResult {
-  data: string[]
-  metadata: {
-    byteCount: number
-    offset: number
-    delimiter: string
-    newline: string
-    quote: string
-  }
-}
+import { DefaultDelimiter } from './constants'
+import { guessDelimiter, validateDelimiter } from './delimiter'
+import { guessLineEndings, validateNewline } from './newline'
+import { parse } from './parser'
+import { validateQuoteChar } from './quoteChar'
+import type { ParseResult } from './types'
+import { testEmptyLine } from './utils'
 
 /**
  * Parses a chunk of bytes into CSV data.
  * @param options Options for parsing the chunk.
  * @param options.bytes The chunk of bytes to parse.
  * @param options.delimiter The delimiter used in the CSV data. Defaults to ','.
  * @param options.newline The newline used in the CSV data. Defaults to '\n'.
- * @param options.quote The quote character used in the CSV data. Defaults to '"'.
+ * @param options.quoteChar The quote character used in the CSV data. Defaults to '"'.
+ * @param options.comments The comment character or boolean to indicate comments
+ * @param options.delimitersToGuess The list of delimiters to guess from
+ * @param options.skipEmptyLines Whether to skip empty lines, if so, whether 'greedy' or not. Defaults to false.
+ * @param options.ignoreLastRow Whether to ignore the last row. Defaults to false.
  * @yields Parsed data and metadata.
  * @returns A generator yielding parsed data and metadata.
  */
 export function* parseChunk({
   bytes,
   delimiter,
   newline,
-  quote,
+  quoteChar,
+  comments,
+  delimitersToGuess,
+  skipEmptyLines,
+  ignoreLastRow,
 }: {
   bytes: Uint8Array
   delimiter?: string
   newline?: string
-  quote?: string
-}): Generator<ChunkResult, void, unknown> {
-  delimiter ??= getDelimiter()
-  newline ??= getNewline()
-  quote ??= getQuote()
-
-  // TODO(SL): reuse decoder?
+  quoteChar?: string
+  comments?: boolean | string
+  delimitersToGuess?: string[]
+  skipEmptyLines?: boolean | 'greedy'
+  ignoreLastRow?: boolean
+}): Generator<ParseResult, void, unknown> {
   const decoder = new TextDecoder('utf-8')
+  const input = decoder.decode(bytes)
+
+  skipEmptyLines ??= false
+  quoteChar = validateQuoteChar(quoteChar)
+  newline = validateNewline(newline) ?? guessLineEndings(input, quoteChar)
+
+  let delimiterError = false
+  delimiter = validateDelimiter(delimiter)
+  if (!delimiter) {
+    const delimGuess = guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess)
+    if (delimGuess.successful)
+      delimiter = delimGuess.bestDelimiter
+    else {
+      delimiterError = true // add error after first row parsing
+      delimiter = DefaultDelimiter
+    }
+  }
+
+  for (const result of parse(input, {
+    delimiter,
+    newline,
+    quoteChar,
+    ignoreLastRow,
+    comments,
+    // TODO(SL): add escapeChar?
+  })) {
+    if (delimiterError) {
+      result.errors.push({
+        type: 'Delimiter',
+        code: 'UndetectableDelimiter',
+        message: 'Unable to auto-detect delimiting character; defaulted to \'' + DefaultDelimiter + '\'',
+      })
+      delimiterError = false
+    }
 
-  const text = decoder.decode(bytes)
+    if (skipEmptyLines && testEmptyLine(result.row, skipEmptyLines)) {
+      // TODO(SL) accumulate the byte count of removed lines
+      continue
+    }
 
-  yield {
-    data: [text],
-    metadata: {
-      byteCount: bytes.length,
-      offset: 0,
-      delimiter,
-      newline,
-      quote,
-    },
+    yield result
   }
 }

src/comments.ts

@@ -0,0 +1,23 @@
+import { BAD_DELIMITERS } from './constants'
+
+/**
+ * Validates the comment character
+ * @param comments The comment character or boolean to indicate comments
+ * @param delimiter The delimiter character
+ * @returns The comment character.
+ */
+export function validateComments(comments?: string | boolean, delimiter?: string): undefined | string | false {
+  if (comments === undefined || comments === false) {
+    return comments
+  }
+  if (comments === true) {
+    return '#'
+  }
+  if (delimiter !== undefined && comments === delimiter) {
+    throw new Error('Comment character same as delimiter')
+  }
+  if (BAD_DELIMITERS.includes(comments)) {
+    throw new Error(`Invalid comment character: ${comments}`)
+  }
+  return comments
+}

src/constants.ts

@@ -1 +1,8 @@
 export const defaultChunkSize = 1024 * 1024 // 1MB
+
+export const RECORD_SEP = String.fromCharCode(30)
+export const UNIT_SEP = String.fromCharCode(31)
+export const BYTE_ORDER_MARK = '\ufeff'
+export const BAD_DELIMITERS = ['\r', '\n', '"', BYTE_ORDER_MARK]
+
+export const DefaultDelimiter = ',' // Used if not specified and detection fails

src/delimiter.ts

@@ -1,9 +1,89 @@
+import { BAD_DELIMITERS, RECORD_SEP, UNIT_SEP } from './constants'
+import { parse } from './parser'
+import { testEmptyLine } from './utils'
+
 /**
- * Returns the delimiter used in the CSV file.
+ * Validates the delimiter
+ * @param delimiter The delimiter string
  * @returns The delimiter string.
  */
-export function getDelimiter() {
-  // TODO(SL): make the delimiter configurable
-  // TODO(SL): guess the delimiter from the first chunk
-  return ','
+export function validateDelimiter(delimiter?: string): undefined | string {
+  if (delimiter === undefined) {
+    return undefined
+  }
+  if (BAD_DELIMITERS.includes(delimiter)) {
+    throw new Error(`Invalid delimiter: ${delimiter}`)
+  }
+  return delimiter
+}
+
+/**
+ * Guess the delimiter
+ * @param input The input string
+ * @param newline The newline character
+ * @param skipEmptyLines Whether to skip empty lines, if so, whether 'greedy' or not
+ * @param comments The comment character or boolean to indicate comments
+ * @param delimitersToGuess The list of delimiters to guess from
+ * @returns An object indicating whether guessing was successful and the best delimiter found
+ */
+export function guessDelimiter(input: string, newline?: string, skipEmptyLines?: boolean | 'greedy', comments?: boolean | string, delimitersToGuess?: string[]) {
+  let bestDelimiter, bestDelta, maxFieldCount
+
+  delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', RECORD_SEP, UNIT_SEP]
+
+  for (let i = 0; i < delimitersToGuess.length; i++) {
+    const delimiter = delimitersToGuess[i]
+    let delta = 0
+    let nonEmptyLinesCount = 0
+    let avgFieldCount = 0
+    let fieldCountPrevRow: number | undefined
+    let j = 0
+    const previewLines = 10
+
+    for (const { row } of parse(input, {
+      delimiter,
+      newline,
+      comments,
+      ignoreLastRow: false,
+    })) {
+      if (j >= previewLines) {
+        break
+      }
+      if (skipEmptyLines && testEmptyLine(row, skipEmptyLines)) {
+        continue
+      }
+      nonEmptyLinesCount++
+
+      const fieldCount = row.length
+      avgFieldCount += fieldCount
+
+      if (fieldCountPrevRow === undefined) {
+        fieldCountPrevRow = fieldCount
+        continue
+      }
+      else if (fieldCount > 0) {
+        delta += Math.abs(fieldCount - fieldCountPrevRow)
+        fieldCountPrevRow = fieldCount
+      }
+      j++
+    }
+
+    if (nonEmptyLinesCount > 0)
+      avgFieldCount /= (nonEmptyLinesCount)
+
+    if (
+      (bestDelta === undefined || delta <= bestDelta)
+      && (maxFieldCount === undefined || avgFieldCount > maxFieldCount)
+      && avgFieldCount > 1.99
+    ) {
+      bestDelta = delta
+      bestDelimiter = delimiter
+      maxFieldCount = avgFieldCount
+    }
+  }
+
+  return {
+    successful: !!bestDelimiter,
+    bestDelimiter: bestDelimiter,
+  }
 }

src/newline.ts

@@ -1,9 +1,53 @@
+import { escapeRegExp } from './utils'
+
+type Newline = '\n' | '\r' | '\r\n'
+
 /**
  * Returns the newline string to be used in parsing.
+ * @param newline The newline string to validate.
  * @returns The newline string.
  */
-export function getNewline() {
+export function validateNewline(newline?: string): Newline | undefined {
   // TODO(SL): make the newline configurable
   // TODO(SL): guess the newline from the first chunk
-  return '\n'
+  if (newline === undefined) {
+    return undefined
+  }
+  if (newline !== '\n' && newline !== '\r' && newline !== '\r\n') {
+    throw new Error(`Invalid newline: ${newline}`)
+  }
+  return newline
+}
+
+/**
+ * Guess the line endings
+ * @param input The input string
+ * @param quoteChar The quote character
+ * @returns The line ending character
+ */
+export function guessLineEndings(input: string, quoteChar: string): Newline {
+  input = input.substring(0, 1024 * 1024) // max length 1 MB
+  // Replace all the text inside quotes
+  const re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm')
+  input = input.replace(re, '')
+
+  const r = input.split('\r')
+
+  const n = input.split('\n')
+
+  if (!(0 in r && 0 in n)) {
+    throw new Error('r or n should have at least one element')
+  }
+  const nAppearsFirst = (n.length > 1 && n[0].length < r[0].length)
+
+  if (r.length === 1 || nAppearsFirst)
+    return '\n'
+
+  let numWithN = 0
+  for (const match of r) {
+    if (match[0] === '\n')
+      numWithN++
+  }
+
+  return numWithN >= r.length / 2 ? '\r\n' : '\r'
 }