use parse directly in url, without using parseString

Author: severo

src/parser.ts

@@ -20,7 +20,6 @@ export function* parse(input: string, options: ParseOptions & {
     quoteChar,
     escapeChar,
   } = validateAndSetDefaultParseOptions(options)
-  // TODO(SL): allow passing no options?
   const ignoreLastRow = options.ignoreLastRow ?? false
 
   // We don't need to compute some of these every time parse() is called,

src/url.ts

@@ -2,7 +2,7 @@ import { fetchChunk } from './fetch'
 import { checkIntegerGreaterOrEqualThan } from './options/check'
 import { defaultChunkSize } from './options/constants'
 import { type DelimiterError, type ParseOptions, validateAndGuessParseOptions } from './options/parseOptions'
-import { parseString } from './string'
+import { parse } from './parser'
 import type { ParseResult } from './types'
 import { testEmptyLine } from './utils'
 
@@ -12,7 +12,7 @@ interface FetchOptions {
   lastByte?: number
   requestInit?: RequestInit
   fetchChunk?: typeof fetchChunk
-  parseString?: typeof parseString
+  parse?: typeof parse
 }
 interface ParseUrlOptions extends ParseOptions, FetchOptions {
   skipEmptyLines?: boolean | 'greedy'
@@ -28,7 +28,7 @@ interface ParseUrlOptions extends ParseOptions, FetchOptions {
  * @param options.lastByte The last byte parsed (inclusive). It must be a non-negative integer. Default is the end of the file.
  * @param options.requestInit Optional fetch request initialization parameters.
  * @param options.fetchChunk Optional custom fetchChunk function for fetching chunks.
- * @param options.parseString Optional custom parseString function for parsing a string.
+ * @param options.parse Optional custom parse function for parsing a string.
  * @param options.delimiter The delimiter used in the CSV data. Defaults to ','.
  * @param options.newline The newline used in the CSV data. Defaults to '\n'.
  * @param options.quoteChar The quote character used in the CSV data. Defaults to '"'.
@@ -96,13 +96,11 @@ export async function* parseUrl(
       delimiterError = result.error
     }
 
-    for (const result of (options.parseString ?? parseString)(input, {
+    for (const result of (options.parse ?? parse)(input, {
       // the remaining bytes may not contain a full last row
       ignoreLastRow: true,
       // pass other options
       ...parseOptions,
-      // handle the empty lines here, to avoid issues when guessing delimiter/newline + to get the correct offsets
-      skipEmptyLines: false,
     })) {
       consumedBytes += result.meta.byteCount
       if (consumedBytes > bytes.length) {
@@ -148,14 +146,16 @@ export async function* parseUrl(
   // Parse remaining bytes, if any
   if (bytes.length > 0) {
     const input = decoder.decode(bytes)
-    for (const result of (options.parseString ?? parseString)(input, {
+    for (const result of (options.parse ?? parse)(input, {
       // parse until the last byte
       ignoreLastRow: false,
       // pass other options
       ...parseOptions,
-      // TODO(SL): should we skip the empty lines here as well?
-      skipEmptyLines,
     })) {
+      if (skipEmptyLines && testEmptyLine(result.row, skipEmptyLines)) {
+        // TODO(SL) how to report the skipped lines in the metadata?
+        continue
+      }
       yield {
         ...result,
         meta: {

tests/url.test.ts

@@ -5,7 +5,7 @@ import { parseUrl } from '../src/url'
 import { toUrl } from '../src/utils'
 import { PARSE_TESTS } from './cases'
 
-function* parseStringMock(input: string): Generator<ParseResult, void, unknown> {
+function* parseMock(input: string): Generator<ParseResult, void, unknown> {
   const encoder = new TextEncoder()
   const bytes = encoder.encode(input)
   yield {
@@ -29,7 +29,7 @@ describe('parseUrl, while mocking parseString, ', () => {
     let result = ''
     let bytes = 0
     // passing 'to: fileSize - 1' for Node.js bug: https://github.com/nodejs/node/issues/60382
-    for await (const { row, meta: { offset, byteCount } } of parseUrl(url, { chunkSize, lastByte: fileSize - 1, parseString: parseStringMock })) {
+    for await (const { row, meta: { offset, byteCount } } of parseUrl(url, { chunkSize, lastByte: fileSize - 1, parse: parseMock })) {
       result += row
       expect(offset).toBe(bytes)
       bytes += byteCount
@@ -58,7 +58,7 @@ describe('parseUrl, while mocking parseString, ', () => {
     const { url, revoke } = toUrl(text)
     let result = ''
     // implicit assertation in the loop: no exceptions thrown
-    for await (const { row } of parseUrl(url, { firstByte, lastByte, parseString: parseStringMock })) {
+    for await (const { row } of parseUrl(url, { firstByte, lastByte, parse: parseMock })) {
       result += row[0]
     }
     revoke()
@@ -96,7 +96,7 @@ describe('parseUrl, while mocking parseString, ', () => {
   it('keeps bytes between iterations, and might not consume all the bytes', async () => {
     const text = 'hello, csvremote!!!'
     const { url, fileSize, revoke } = toUrl(text)
-    function* parseStringMock(input: string) {
+    function* parseMock(input: string) {
       // only yield the first two bytes, decoded as text
       const encoder = new TextEncoder()
       const bytes = encoder.encode(input)
@@ -122,7 +122,7 @@ describe('parseUrl, while mocking parseString, ', () => {
     for await (const { row, meta: { offset, byteCount } } of parseUrl(url, {
       chunkSize: 5,
       lastByte: fileSize - 1,
-      parseString: parseStringMock,
+      parse: parseMock,
     })) {
       i++
       result += row[0]
@@ -137,7 +137,7 @@ describe('parseUrl, while mocking parseString, ', () => {
   it('keeps bytes between iterations and might consume all the bytes', async () => {
     const text = 'hello, csvremote!!!'
     const { url, fileSize, revoke } = toUrl(text)
-    function* parseStringMock(text: string) {
+    function* parseMock(text: string) {
       // only process up to the first comma
       const splits = text.split(',')
       const firstPart = splits[0] + (splits.length > 1 ? ',' : '')
@@ -163,7 +163,7 @@ describe('parseUrl, while mocking parseString, ', () => {
     for await (const { row, meta: { offset, byteCount } } of parseUrl(url, {
       chunkSize: 10,
       lastByte: fileSize - 1,
-      parseString: parseStringMock,
+      parse: parseMock,
     })) {
       expect(offset).toBe(bytes)
       if (i === 0) {
@@ -184,7 +184,7 @@ describe('parseUrl, while mocking parseString, ', () => {
   it('throws if parseString yields more bytes than provided', async () => {
     const text = 'hello, csvremote!!!'
     const { url, revoke } = toUrl(text)
-    function* parseStringMock(text: string) {
+    function* parseMock(text: string) {
       // yield more bytes than provided
       yield {
         row: [],
@@ -201,7 +201,7 @@ describe('parseUrl, while mocking parseString, ', () => {
     }
     const iterator = parseUrl(url, {
       chunkSize: 5,
-      parseString: parseStringMock,
+      parse: parseMock,
     })
     await expect(iterator.next()).rejects.toThrow()
     revoke()