Objects only, no arrays no rowFormat

Author: platypii

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## [2.0.0]
  - Rename `parquetMetadataAsync` to `parquetMetadata`
  - Previous `parquetMetadata` is now `import { parquetMetadataSync } from 'hyparquet/src/metadata.js'`
+ - Rename `onPage.columnName: string` to `onPage.pathInSchema: string[]`
+ - Remove `rowFormat` and always return rows as objects
 
 ## [1.23.0]
  - Replace `columnName: string` with `pathInSchema: string[]` in `onPage` callback (#144)

README.md

@@ -171,22 +171,6 @@ await parquetRead({
 })
 ```
 
-### Returned row format
-
-By default, the `onComplete` function returns an **array** of values for each row: `[value]`. If you would prefer each row to be an **object**:  `{ columnName: value }`, set the option `rowFormat` to `'object'`.
-
-```javascript
-import { parquetRead } from 'hyparquet'
-
-await parquetRead({
-  file,
-  rowFormat: 'object',
-  onComplete: data => console.log(data),
-})
-```
-
-The `parquetReadObjects` function defaults to `rowFormat: 'object'`.
-
 ### Binary columns
 
 Hyparquet defaults to decoding binary columns as utf8 text strings. A parquet `BYTE_ARRAY` column may contain arbitrary binary data or utf8 encoded text data. In theory, a column should be annotated as [LogicalType](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) STRING if it contains utf8 text. But in practice, many parquet files omit this annotation. Hyparquet's default decoding behavior can be disabled by setting the `utf8` option to `false`. The `utf8` option only affects `BYTE_ARRAY` columns _without_ an annotation.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyparquet",
-  "version": "1.23.0",
+  "version": "2.0.0",
   "description": "Parquet file parser for JavaScript",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

src/read.js

@@ -25,7 +25,7 @@ export async function parquetRead(options) {
   // read row groups
   const asyncGroups = parquetReadAsync(options)
 
-  const { rowStart = 0, rowEnd, columns, onChunk, onComplete, rowFormat } = options
+  const { rowStart = 0, rowEnd, columns, onChunk, onComplete } = options
 
   // skip assembly if no onComplete or onChunk, but wait for reading to finish
   if (!onComplete && !onChunk) {
@@ -61,17 +61,14 @@ export async function parquetRead(options) {
 
   // onComplete transpose column chunks to rows
   if (onComplete) {
-    // loosen the types to avoid duplicate code
-    /** @type {any[]} */
+    /** @type {Record<string, any>[]} */
     const rows = []
     for (const asyncGroup of assembled) {
       // filter to rows in range
       const selectStart = Math.max(rowStart - asyncGroup.groupStart, 0)
       const selectEnd = Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows)
       // transpose column chunks to rows in output
-      const groupData = rowFormat === 'object' ?
-        await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, 'object') :
-        await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, 'array')
+      const groupData = await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns)
       concat(rows, groupData)
     }
     onComplete(rows)
@@ -135,7 +132,6 @@ export function parquetReadObjects(options) {
   return new Promise((onComplete, reject) => {
     parquetRead({
       ...options,
-      rowFormat: 'object', // force object output
       onComplete,
     }).catch(reject)
   })

src/rowgroup.js

@@ -73,71 +73,33 @@ export function readRowGroup(options, { metadata, columns }, groupPlan) {
 }
 
 /**
- * @overload
  * @param {AsyncRowGroup} asyncGroup
  * @param {number} selectStart
  * @param {number} selectEnd
  * @param {string[] | undefined} columns
- * @param {'object'} rowFormat
  * @returns {Promise<Record<string, any>[]>} resolves to row data
  */
-/**
- * @overload
- * @param {AsyncRowGroup} asyncGroup
- * @param {number} selectStart
- * @param {number} selectEnd
- * @param {string[] | undefined} columns
- * @param {'array'} [rowFormat]
- * @returns {Promise<any[][]>} resolves to row data
- */
-/**
- * @param {AsyncRowGroup} asyncGroup
- * @param {number} selectStart
- * @param {number} selectEnd
- * @param {string[] | undefined} columns
- * @param {'object' | 'array'} [rowFormat]
- * @returns {Promise<Record<string, any>[] | any[][]>} resolves to row data
- */
-export async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, columns, rowFormat) {
+export async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, columns) {
   // columnData[i] for asyncColumns[i]
   // TODO: do it without flatten
   const columnDatas = await Promise.all(asyncColumns.map(({ data }) => data.then(flatten)))
 
-  // careful mapping of column order for rowFormat: array
-  const includedColumnNames = asyncColumns
-    .map(child => child.pathInSchema[0])
-    .filter(name => !columns || columns.includes(name))
-  const columnOrder = columns ?? includedColumnNames
-  const columnIndexes = columnOrder.map(name => asyncColumns.findIndex(column => column.pathInSchema[0] === name))
+  // filter columns
+  const filteredColumns = columns
+    ? asyncColumns.filter(column => columns.includes(column.pathInSchema[0]))
+    : asyncColumns
 
   // transpose columns into rows
   const selectCount = selectEnd - selectStart
-  if (rowFormat === 'object') {
-    /** @type {Record<string, any>[]} */
-    const groupData = new Array(selectCount)
-    for (let selectRow = 0; selectRow < selectCount; selectRow++) {
-      const row = selectStart + selectRow
-      // return each row as an object
-      /** @type {Record<string, any>} */
-      const rowData = {}
-      for (let i = 0; i < asyncColumns.length; i++) {
-        rowData[asyncColumns[i].pathInSchema[0]] = columnDatas[i][row]
-      }
-      groupData[selectRow] = rowData
-    }
-    return groupData
-  }
-
-  /** @type {any[][]} */
+  /** @type {Record<string, any>[]} */
   const groupData = new Array(selectCount)
   for (let selectRow = 0; selectRow < selectCount; selectRow++) {
     const row = selectStart + selectRow
-    // return each row as an array
-    const rowData = new Array(asyncColumns.length)
-    for (let i = 0; i < columnOrder.length; i++) {
-      if (columnIndexes[i] >= 0) {
-        rowData[i] = columnDatas[columnIndexes[i]][row]
-      }
+    /** @type {Record<string, any>} */
+    const rowData = {}
+    for (let i = 0; i < filteredColumns.length; i++) {
+      const columnIndex = asyncColumns.indexOf(filteredColumns[i])
+      rowData[filteredColumns[i].pathInSchema[0]] = columnDatas[columnIndex][row]
     }
     groupData[selectRow] = rowData
   }

src/types.d.ts

@@ -23,7 +23,7 @@ export interface MetadataOptions {
 /**
  * Parquet query options for reading data
  */
-export interface BaseParquetReadOptions {
+export interface ParquetReadOptions {
   file: AsyncBuffer // file-like object containing parquet data
   metadata?: FileMetaData // parquet metadata, will be parsed if not provided
   columns?: string[] // columns to read, all columns if undefined
@@ -32,21 +32,14 @@ export interface BaseParquetReadOptions {
   rowEnd?: number // last requested row index (exclusive)
   onChunk?: (chunk: ColumnData) => void // called when a column chunk is parsed. chunks may contain data outside the requested range.
   onPage?: (chunk: SubColumnData) => void // called when a data page is parsed. pages may contain data outside the requested range.
+  onComplete?: (rows: Record<string, any>[]) => void // called when all requested rows and columns are parsed
   compressors?: Compressors // custom decompressors
   utf8?: boolean // decode byte arrays as utf8 strings (default true)
   parsers?: ParquetParsers // custom parsers to decode advanced types
   geoparquet?: boolean // parse geoparquet metadata and set logical type to geometry/geography for geospatial columns (default true)
 }
 
-interface ArrayRowFormat {
-  rowFormat?: 'array' // format of each row passed to the onComplete function. Can be omitted, as it's the default.
-  onComplete?: (rows: any[][]) => void // called when all requested rows and columns are parsed
-}
-interface ObjectRowFormat {
-  rowFormat: 'object' // format of each row passed to the onComplete function
-  onComplete?: (rows: Record<string, any>[]) => void // called when all requested rows and columns are parsed
-}
-export type ParquetReadOptions = BaseParquetReadOptions & (ArrayRowFormat | ObjectRowFormat)
+export type BaseParquetReadOptions = ParquetReadOptions
 
 /**
  * Parquet query options for filtering data

test/read.test.js

@@ -27,7 +27,7 @@ describe('parquetRead', () => {
       rowStart: 2,
       rowEnd: 4,
       onComplete(rows) {
-        expect(rows).toEqual([[3n], [4n]])
+        expect(rows).toEqual([{ numbers: 3n }, { numbers: 4n }])
       },
     })
   })
@@ -39,7 +39,9 @@ describe('parquetRead', () => {
       rowEnd: 100,
       onComplete(rows) {
         expect(rows).toEqual([
-          [1n], [2n], [3n], [4n], [5n], [6n], [7n], [8n], [9n], [10n], [11n], [12n], [13n], [14n], [15n],
+          { numbers: 1n }, { numbers: 2n }, { numbers: 3n }, { numbers: 4n }, { numbers: 5n },
+          { numbers: 6n }, { numbers: 7n }, { numbers: 8n }, { numbers: 9n }, { numbers: 10n },
+          { numbers: 11n }, { numbers: 12n }, { numbers: 13n }, { numbers: 14n }, { numbers: 15n },
         ])
       },
     })
@@ -77,11 +79,11 @@ describe('parquetRead', () => {
       },
       onComplete(rows) {
         expect(rows).toEqual([
-          [[1, 2, 3]],
-          [undefined],
-          [undefined],
-          [[1, 2, 3]],
-          [[1, 2]],
+          { e: [1, 2, 3] },
+          { e: undefined },
+          { e: undefined },
+          { e: [1, 2, 3] },
+          { e: [1, 2] },
         ])
       },
     })
@@ -110,24 +112,23 @@ describe('parquetRead', () => {
       },
       onComplete(rows) {
         expect(rows).toEqual([
-          [{ k1: 1, k2: 100 }],
-          [{ k1: 2, k2: null }],
-          [{ }],
-          [{ }],
-          [{ }],
-          [undefined],
-          [{ k1: null, k3: null }],
+          { int_map: { k1: 1, k2: 100 } },
+          { int_map: { k1: 2, k2: null } },
+          { int_map: { } },
+          { int_map: { } },
+          { int_map: { } },
+          { int_map: undefined },
+          { int_map: { k1: null, k3: null } },
         ])
       },
     })
   })
 
-  it('format row as object', async () => {
+  it('read single column as objects', async () => {
     const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
     await parquetRead({
       file,
       columns: ['c'],
-      rowFormat: 'object',
       onComplete(rows) {
         expect(rows).toEqual([
           { c: 2 },
@@ -140,18 +141,18 @@ describe('parquetRead', () => {
     })
   })
 
-  it('read columns out of order', async () => {
+  it('read selected columns', async () => {
     const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
     await parquetRead({
       file,
-      columns: ['c', 'missing', 'b', 'c'],
+      columns: ['c', 'b'],
       onComplete(rows) {
         expect(rows).toEqual([
-          [2, undefined, 1, 2],
-          [3, undefined, 2, 3],
-          [4, undefined, 3, 4],
-          [5, undefined, 4, 5],
-          [2, undefined, 5, 2],
+          { b: 1, c: 2 },
+          { b: 2, c: 3 },
+          { b: 3, c: 4 },
+          { b: 4, c: 5 },
+          { b: 5, c: 2 },
         ])
       },
     })

test/readFiles.test.js

@@ -5,6 +5,15 @@ import { parquetMetadata, parquetRead, toJson } from '../src/index.js'
 import { asyncBufferFromFile } from '../src/node.js'
 import { fileToJson } from './helpers.js'
 
+/**
+ * Convert object rows to array rows for comparison with legacy JSON test files.
+ * @param {Record<string, any>[]} rows
+ * @returns {any[][]}
+ */
+function toArrays(rows) {
+  return rows.map(row => Object.values(row))
+}
+
 describe('parquetRead test files', () => {
   const files = fs.readdirSync('test/files').filter(f => f.endsWith('.parquet'))
 
@@ -18,7 +27,7 @@ describe('parquetRead test files', () => {
           const base = filename.replace('.parquet', '')
           const expected = fileToJson(`test/files/${base}.json`)
           // stringify and parse to make legal json (NaN, -0, etc)
-          expect(JSON.parse(JSON.stringify(toJson(rows)))).toEqual(expected)
+          expect(JSON.parse(JSON.stringify(toJson(toArrays(rows))))).toEqual(expected)
         },
       })
     })
@@ -39,7 +48,7 @@ describe('parquetRead test files', () => {
           const base = filename.replace('.parquet', '')
           if (rows.length) {
             const expected = [fileToJson(`test/files/${base}.json`).at(-1)]
-            expect(toJson(rows)).toEqual(expected)
+            expect(toJson(toArrays(rows))).toEqual(expected)
           }
         },
       })