Timestamp and cast literal pushdown

Author: platypii

package.json

@@ -53,7 +53,7 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "hyparquet": "1.27.1",
+    "hyparquet": "1.28.1",
     "hyparquet-compressors": "1.1.1",
     "hyparquet-writer": "0.16.5",
     "squirreling": "0.15.0"

src/prune.js

@@ -150,6 +150,12 @@ function opMightMatch(op, value, v, transform, sourceType) {
   }
 
   const kind = transformKind(transform)
+  // A day-transform partition value has iceberg type `date`, which the Avro
+  // reader decodes to a `Date`, while `applyTransform('day', ...)` projects
+  // literals to day ordinals; normalize to days so both sides compare.
+  if (transform === 'day' && v instanceof Date) {
+    v = Math.floor(v.getTime() / 86400000)
+  }
   if (kind === 'identity') return identityMightMatch(op, v, value)
   if (kind === 'monotonic') return monotonicMightMatch(op, v, value, transform, sourceType)
   if (kind === 'bucket') return bucketMightMatch(op, v, value, transform, sourceType)

src/sql/whereFilter.js

@@ -8,7 +8,7 @@
  * downstream in `readDataFile`.
  *
  * @import {ExprNode} from 'squirreling'
- * @import {BinaryNode, InValuesNode} from 'squirreling/src/ast.js'
+ * @import {BinaryNode, CastType, InValuesNode} from 'squirreling/src/ast.js'
  * @import {ParquetQueryFilter} from 'hyparquet'
  * @param {ExprNode | undefined} where
  * @returns {ParquetQueryFilter | undefined}
@@ -38,12 +38,20 @@ function convertExpr(node, negate) {
   if (node.type === 'in valuelist') {
     return convertInValues(node, negate)
   }
-  if (node.type === 'cast') {
+  if (node.type === 'cast' && TRUTHINESS_PRESERVING_CASTS.has(node.toType)) {
+    // A cast at boolean position (WHERE CAST(a = 1 AS INT)) keeps the operand's
+    // truthiness only for boolean/numeric targets; TEXT ('false' is truthy) and
+    // TIMESTAMP (any Date is truthy) do not, so those fall back to the engine.
     return convertExpr(node.expr, negate)
   }
   return undefined
 }
 
+/** @type {Set<CastType>} */
+const TRUTHINESS_PRESERVING_CASTS = new Set(
+  ['BOOLEAN', 'BOOL', 'INTEGER', 'INT', 'BIGINT', 'FLOAT', 'REAL', 'DOUBLE']
+)
+
 /**
  * @param {BinaryNode} node
  * @param {boolean} negate
@@ -78,15 +86,97 @@ function convertBinary({ op, left, right }, negate) {
  * @returns {{column: string | undefined, value: any, flipped: boolean}}
  */
 function extractColumnAndValue(left, right) {
-  if (left.type === 'identifier' && right.type === 'literal') {
-    return { column: left.name, value: right.value, flipped: false }
-  }
-  if (left.type === 'literal' && right.type === 'identifier') {
-    return { column: right.name, value: left.value, flipped: true }
+  if (left.type === 'identifier') {
+    const lit = staticLiteral(right)
+    if (lit) return { column: left.name, value: lit.value, flipped: false }
+  } else if (right.type === 'identifier') {
+    const lit = staticLiteral(left)
+    if (lit) return { column: right.name, value: lit.value, flipped: true }
   }
   return { column: undefined, value: undefined, flipped: false }
 }
 
+/**
+ * Statically evaluate an expression to a constant. Handles plain literals and
+ * casts of literals, including CAST(string AS TIMESTAMP), which is how
+ * squirreling parses typed literals like TIMESTAMP '2026-08-06T00:00:00Z'. The
+ * result is wrapped in {value} so an undefined-valued literal isn't confused
+ * with "not constant".
+ *
+ * @param {ExprNode} node
+ * @returns {{value: any} | undefined}
+ */
+function staticLiteral(node) {
+  if (node.type === 'literal') return { value: node.value }
+  if (node.type === 'cast') {
+    const inner = staticLiteral(node.expr)
+    if (!inner) return undefined
+    return foldCast(node.toType, inner.value)
+  }
+  return undefined
+}
+
+/**
+ * Mirror of squirreling's CAST evaluation over primitive literals. Must stay
+ * in lockstep with the engine: a pushed-down filter replaces engine-side
+ * WHERE, so a folded value that compares differently would change results.
+ * A cast the engine would evaluate to null (unparseable date, NaN) returns
+ * undefined: null comparisons match no rows, and falling back to the engine
+ * preserves that without needing a filter for it.
+ *
+ * @param {CastType} toType
+ * @param {any} val
+ * @returns {{value: any} | undefined}
+ */
+function foldCast(toType, val) {
+  if (val === null || val === undefined) return undefined
+  if (toType === 'TEXT' || toType === 'STRING' || toType === 'VARCHAR') {
+    return { value: String(val) }
+  }
+  if (toType === 'INTEGER' || toType === 'INT') {
+    const num = Number(val)
+    return isNaN(num) ? undefined : { value: Math.trunc(num) }
+  }
+  if (toType === 'BIGINT') {
+    if (typeof val === 'bigint') return { value: val }
+    const num = Number(val)
+    return isNaN(num) ? undefined : { value: BigInt(Math.trunc(num)) }
+  }
+  if (toType === 'FLOAT' || toType === 'REAL' || toType === 'DOUBLE') {
+    const num = Number(val)
+    return isNaN(num) ? undefined : { value: num }
+  }
+  if (toType === 'BOOLEAN' || toType === 'BOOL') {
+    return { value: Boolean(val) }
+  }
+  if (toType === 'TIMESTAMP') {
+    const date = castTimestamp(val)
+    return date ? { value: date } : undefined
+  }
+  return undefined
+}
+
+/**
+ * Mirror of squirreling's TIMESTAMP cast: numbers as epoch millis, strings via
+ * its `toDate` parse, which requires a YYYY-MM-DD prefix. (`toDate` is not in
+ * squirreling's public exports, hence the copy.)
+ *
+ * @param {any} val
+ * @returns {Date | undefined}
+ */
+function castTimestamp(val) {
+  if (val instanceof Date) return val
+  if (typeof val === 'number' || typeof val === 'bigint') {
+    const date = new Date(Number(val))
+    return isNaN(date.getTime()) ? undefined : date
+  }
+  if (typeof val === 'string' && /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2})?/.test(val)) {
+    const date = new Date(val)
+    if (!isNaN(date.getTime())) return date
+  }
+  return undefined
+}
+
 const COMP_OPS = new Set(['=', '==', '!=', '<>', '<', '>', '<=', '>='])
 
 /**
@@ -152,8 +242,9 @@ function convertInValues(node, negate) {
   if (node.expr.type !== 'identifier') return undefined
   const values = []
   for (const val of node.values) {
-    if (val.type !== 'literal') return undefined
-    values.push(val.value)
+    const lit = staticLiteral(val)
+    if (!lit) return undefined
+    values.push(lit.value)
   }
   return { [node.expr.name]: { [negate ? '$nin' : '$in']: values } }
 }

test/prune.test.js

@@ -114,6 +114,16 @@ describe('partitionMightMatch — monotonic (day)', () => {
     const ts = new Date('2022-01-01T00:00:00Z')
     expect(partitionMightMatch({ ts: { $ne: ts } }, entry({ ts_day: day('2022-01-01') }), schema, m)).toBe(true)
   })
+
+  it('prunes when the partition value decodes as a Date (avro logical date)', () => {
+    // Spec-compliant manifests (Spark, Java) type day partitions as
+    // {int, logicalType: date}, which the avro reader decodes to a Date.
+    const ts = new Date('2022-06-15T12:00:00Z')
+    expect(partitionMightMatch({ ts: { $gt: ts } }, entry({ ts_day: new Date('2022-01-01') }), schema, m)).toBe(false)
+    expect(partitionMightMatch({ ts: { $gt: ts } }, entry({ ts_day: new Date('2022-12-01') }), schema, m)).toBe(true)
+    expect(partitionMightMatch({ ts: { $eq: ts } }, entry({ ts_day: new Date('2022-06-15') }), schema, m)).toBe(true)
+    expect(partitionMightMatch({ ts: { $eq: ts } }, entry({ ts_day: new Date('2022-06-14') }), schema, m)).toBe(false)
+  })
 })
 
 describe('partitionMightMatch — combinators and conservative defaults', () => {

test/sql/scanPruning.test.js

@@ -1,10 +1,12 @@
+import { collect } from 'squirreling'
 import { describe, expect, it, vi } from 'vitest'
 import { ByteWriter, parquetWrite } from 'hyparquet-writer'
 import { fileCatalogCommit } from '../../src/write/commit.js'
 import { icebergCreate } from '../../src/create.js'
 import { readDataFile } from '../../src/read.js'
 import { icebergStageAppend } from '../../src/write/stage.js'
 import { icebergDataSource } from '../../src/sql/icebergDataSource.js'
+import { icebergQuery } from '../../src/sql/icebergQuery.js'
 import { memResolver } from '../helpers.js'
 
 /**
@@ -293,3 +295,102 @@ describe('#21 row-group pruning (readDataFile)', () => {
     expect(filtered.length).toBe(25)
   })
 })
+
+describe('timestamp predicate pushdown on a day-partitioned table', () => {
+  /** @type {Schema} */
+  const schema = {
+    type: 'struct',
+    'schema-id': 0,
+    fields: [
+      { id: 1, name: 'id', required: true, type: 'long' },
+      { id: 2, name: 'message_created_at', required: false, type: 'timestamptz' },
+    ],
+  }
+  /** @type {import('../../src/types.js').PartitionSpec} */
+  const partitionSpec = {
+    'spec-id': 0,
+    fields: [{ 'source-id': 2, 'field-id': 1000, name: 'created_day', transform: 'day' }],
+  }
+  const days = ['2026-08-04', '2026-08-05', '2026-08-06']
+
+  /**
+   * Build a table partitioned by day(message_created_at) with one data file
+   * per day (a single append; the writer groups records by partition tuple).
+   *
+   * @returns {Promise<{ tableUrl: string, resolver: Resolver, dataFilesRead: () => number }>}
+   */
+  async function buildDayPartitionedTable() {
+    vi.spyOn(Date, 'now').mockReturnValue(1700000000000)
+    const tableUrl = 'mem://events'
+    const { resolver: memR } = memResolver()
+    let metadata = await icebergCreate({ tableUrl, resolver: memR, schema, partitionSpec })
+    const records = []
+    let id = 0n
+    for (const day of days) {
+      for (const hour of ['01', '09', '17']) {
+        records.push({ id: id++, message_created_at: new Date(`${day}T${hour}:30:00Z`) })
+      }
+    }
+    const staged = await icebergStageAppend({ tableUrl, metadata, records, resolver: memR })
+    metadata = await fileCatalogCommit({ tableUrl, metadata, staged, resolver: memR })
+    const { resolver, dataFilesRead } = countingResolver({ reader: memR.reader })
+    return { tableUrl, resolver, dataFilesRead }
+  }
+
+  /**
+   * @param {string} where
+   * @returns {Promise<{ rows: Record<string, any>[], dataFilesRead: number }>}
+   */
+  async function query(where) {
+    const { tableUrl, resolver, dataFilesRead } = await buildDayPartitionedTable()
+    const source = await icebergDataSource({ tableUrl, resolver })
+    const result = await icebergQuery({
+      query: `SELECT id, message_created_at FROM events WHERE ${where} ORDER BY id`,
+      tables: { events: source },
+    })
+    const rows = await collect(result)
+    return { rows, dataFilesRead: dataFilesRead() }
+  }
+
+  it('skips files from days that cannot match a >= TIMESTAMP predicate', async () => {
+    const { rows, dataFilesRead } = await query('message_created_at >= TIMESTAMP \'2026-08-06T00:00:00Z\'')
+    expect(rows.map(r => r.id)).toEqual([6n, 7n, 8n])
+    expect(dataFilesRead).toBe(1)
+  })
+
+  it('returns the same rows as engine-side filtering', async () => {
+    // The added `id + 0 >= 0` conjunct is a tautology, but arithmetic is not
+    // pushable, so the whole WHERE falls back to the engine and reads all files.
+    const pushed = await query('message_created_at >= TIMESTAMP \'2026-08-06T00:00:00Z\'')
+    const engine = await query('message_created_at >= TIMESTAMP \'2026-08-06T00:00:00Z\' AND id + 0 >= 0')
+    expect(pushed.rows).toEqual(engine.rows)
+    expect(engine.dataFilesRead).toBe(3)
+    expect(pushed.dataFilesRead).toBe(1)
+  })
+
+  it('prunes to two files for a < TIMESTAMP mid-range boundary', async () => {
+    const { rows, dataFilesRead } = await query('message_created_at < TIMESTAMP \'2026-08-06T00:00:00Z\'')
+    expect(rows.map(r => r.id)).toEqual([0n, 1n, 2n, 3n, 4n, 5n])
+    expect(dataFilesRead).toBe(2)
+  })
+
+  it('prunes to one file for an equality on a single instant', async () => {
+    const { rows, dataFilesRead } = await query('message_created_at = TIMESTAMP \'2026-08-05T09:30:00Z\'')
+    expect(rows.map(r => r.id)).toEqual([4n])
+    expect(dataFilesRead).toBe(1)
+  })
+
+  it('prunes to one file for an IN list of same-day instants', async () => {
+    const { rows, dataFilesRead } = await query(
+      'message_created_at IN (TIMESTAMP \'2026-08-05T01:30:00Z\', TIMESTAMP \'2026-08-05T17:30:00Z\')')
+    expect(rows.map(r => r.id)).toEqual([3n, 5n])
+    expect(dataFilesRead).toBe(1)
+  })
+
+  it('keeps the boundary day for BETWEEN-style range predicates', async () => {
+    const { rows, dataFilesRead } = await query(
+      'message_created_at >= TIMESTAMP \'2026-08-05T00:00:00Z\' AND message_created_at < TIMESTAMP \'2026-08-06T00:00:00Z\'')
+    expect(rows.map(r => r.id)).toEqual([3n, 4n, 5n])
+    expect(dataFilesRead).toBe(1)
+  })
+})

test/sql/whereFilter.test.js

@@ -49,6 +49,15 @@ function inList(expr, values) {
   return /** @type {ExprNode} */ ({ type: 'in valuelist', expr, values })
 }
 
+/**
+ * @param {string} toType
+ * @param {ExprNode} expr
+ * @returns {ExprNode}
+ */
+function cast(toType, expr) {
+  return /** @type {ExprNode} */ ({ type: 'cast', toType, expr })
+}
+
 describe.concurrent('whereToParquetFilter', () => {
   it('returns undefined for missing where', () => {
     expect(whereToParquetFilter(undefined)).toBeUndefined()
@@ -144,6 +153,74 @@ describe.concurrent('whereToParquetFilter', () => {
     expect(whereToParquetFilter(where)).toEqual({ a: { $eq: 1 } })
   })
 
+  it('falls back for a truthiness-changing cast at boolean position', () => {
+    // CAST(a = 1 AS TEXT) yields 'false', which is truthy, so unwrapping the
+    // cast would filter rows the engine keeps.
+    expect(whereToParquetFilter(cast('TEXT', bin('=', id('a'), lit(1))))).toBeUndefined()
+    expect(whereToParquetFilter(cast('TIMESTAMP', bin('=', id('a'), lit(1))))).toBeUndefined()
+  })
+
+  it('converts a TIMESTAMP typed literal into a Date predicate', () => {
+    const where = bin('>=', id('message_created_at'), cast('TIMESTAMP', lit('2026-08-06T00:00:00Z')))
+    expect(whereToParquetFilter(where)).toEqual({
+      message_created_at: { $gte: new Date('2026-08-06T00:00:00Z') },
+    })
+  })
+
+  it('flips a TIMESTAMP literal on the left of the comparison', () => {
+    const where = bin('>', cast('TIMESTAMP', lit('2026-08-06T00:00:00Z')), id('ts'))
+    expect(whereToParquetFilter(where)).toEqual({ ts: { $lt: new Date('2026-08-06T00:00:00Z') } })
+  })
+
+  it('negates a TIMESTAMP comparison under NOT', () => {
+    const where = un('NOT', bin('<', id('ts'), cast('TIMESTAMP', lit('2026-08-06T00:00:00Z'))))
+    expect(whereToParquetFilter(where)).toEqual({ ts: { $gte: new Date('2026-08-06T00:00:00Z') } })
+  })
+
+  it('casts numeric epoch literals to TIMESTAMP', () => {
+    const where = bin('=', id('ts'), cast('TIMESTAMP', lit(86400000)))
+    expect(whereToParquetFilter(where)).toEqual({ ts: { $eq: new Date('1970-01-02T00:00:00Z') } })
+  })
+
+  it('folds numeric, boolean, and text casts of literals', () => {
+    expect(whereToParquetFilter(bin('=', id('a'), cast('INT', lit('5'))))).toEqual({ a: { $eq: 5 } })
+    expect(whereToParquetFilter(bin('=', id('a'), cast('INT', lit(5.7))))).toEqual({ a: { $eq: 5 } })
+    expect(whereToParquetFilter(bin('=', id('a'), cast('BIGINT', lit(5))))).toEqual({ a: { $eq: 5n } })
+    expect(whereToParquetFilter(bin('=', id('a'), cast('DOUBLE', lit('2.5'))))).toEqual({ a: { $eq: 2.5 } })
+    expect(whereToParquetFilter(bin('=', id('a'), cast('BOOL', lit(1))))).toEqual({ a: { $eq: true } })
+    expect(whereToParquetFilter(bin('=', id('a'), cast('TEXT', lit(5))))).toEqual({ a: { $eq: '5' } })
+  })
+
+  it('folds nested casts', () => {
+    const where = bin('=', id('a'), cast('TEXT', cast('INT', lit('5.7'))))
+    expect(whereToParquetFilter(where)).toEqual({ a: { $eq: '5' } })
+  })
+
+  it('falls back for casts the engine would evaluate to null', () => {
+    expect(whereToParquetFilter(bin('>=', id('ts'), cast('TIMESTAMP', lit('not a date'))))).toBeUndefined()
+    expect(whereToParquetFilter(bin('>=', id('ts'), cast('TIMESTAMP', lit(null))))).toBeUndefined()
+    expect(whereToParquetFilter(bin('=', id('a'), cast('INT', lit('abc'))))).toBeUndefined()
+    expect(whereToParquetFilter(bin('=', id('a'), cast('INT', lit(null))))).toBeUndefined()
+  })
+
+  it('pushes TIMESTAMP literals inside IN lists', () => {
+    // Needs hyparquet >= 1.28.0, whose $in/$nin compare Dates by time.
+    const where = inList(id('ts'), [cast('TIMESTAMP', lit('2026-08-06T00:00:00Z')), lit(1)])
+    expect(whereToParquetFilter(where)).toEqual({
+      ts: { $in: [new Date('2026-08-06T00:00:00Z'), 1] },
+    })
+  })
+
+  it('falls back when an IN value cast is unparseable', () => {
+    const where = inList(id('ts'), [cast('TIMESTAMP', lit('not a date'))])
+    expect(whereToParquetFilter(where)).toBeUndefined()
+  })
+
+  it('folds casts inside IN lists', () => {
+    const where = inList(id('a'), [cast('INT', lit('1')), lit(2)])
+    expect(whereToParquetFilter(where)).toEqual({ a: { $in: [1, 2] } })
+  })
+
   it('returns undefined for LIKE (not pushable)', () => {
     const where = bin('LIKE', id('a'), lit('foo%'))
     expect(whereToParquetFilter(where)).toBeUndefined()