types.d.ts

import type { AsyncBuffer } from 'hyparquet'
import type { Writer } from 'hyparquet-writer/src/types.js'

export interface WriterOptions {
  /**
   * If `'*'`, the writer must create the object only if it does not already
   * exist (S3 conditional write: `If-None-Match: *`). On collision the
   * underlying `finish()` rejects with an error whose `status` is 412 (or 409
   * on some providers). The HTTP/S3 `urlResolver` translates this to the
   * `If-None-Match` header. Other resolvers may honor or ignore it.
   */
  ifNoneMatch?: '*'
}

export interface Resolver {
  reader: (path: string, byteLength?: number) => AsyncBuffer | Promise<AsyncBuffer>
  writer?: (path: string, options?: WriterOptions) => Writer
  deleter?: (path: string) => Promise<void>
}
export type Lister = (path: string) => Promise<string[]>

/** Catalogs */
export interface RestCatalogContext {
  type: 'rest'
  url: string
  prefix: string
  defaults: Record<string, string>
  overrides: Record<string, string>
  requestInit?: RequestInit
}

export interface FileCatalog {
  type: 'file'
  resolver: Resolver
  lister?: Lister
  /**
   * Opt in to S3-safe metadata commits: every `vN.metadata.json` (the
   * initial create and every subsequent commit) is written with
   * `If-None-Match: *` and `version-hint.text` is best-effort. High-level
   * write functions retry on 412/409 by reloading the latest metadata and
   * re-staging. `icebergCreateTable` and `icebergTransaction` do not retry.
   * Default false preserves backwards-compatible (overwrite) behavior.
   */
  conditionalCommits?: boolean
}

export type Catalog = RestCatalogContext | FileCatalog

export interface TableIdentifier {
  namespace: string[]
  name: string
}

export interface LoadTableResponse {
  metadataLocation?: string
  metadata: TableMetadata
  config: Record<string, string>
}

export interface StorageCredential {
  prefix: string
  config: Record<string, string>
}

export interface TableMetadata {
  'format-version': number
  'table-uuid': string
  location: string
  'last-sequence-number': number // missing in V1, required in V2+
  'last-updated-ms': number
  'last-column-id': number
  'current-schema-id': number // optional in V1, required in V2+
  schemas: Schema[] // optional in V1, required in V2+
  'default-spec-id': number // optional in V1, required in V2+
  'partition-specs': PartitionSpec[] // optional in V1, required in V2+
  'last-partition-id': number // optional in V1, required in V2+
  properties?: Record<string, string>
  'current-snapshot-id'?: number | bigint
  snapshots?: Snapshot[]
  'snapshot-log'?: SnapshotLog[]
  'metadata-log'?: MetadataLog[]
  'sort-orders': SortOrder[] // optional in V1, required in V2+
  'default-sort-order-id': number // optional in V1, required in V2+
  refs?: Record<string, SnapshotRef>
  statistics?: TableStatistics[]
  'partition-statistics'?: PartitionStatistics[]
  'next-row-id'?: number | bigint // required in V3
  'encryption-keys'?: EncryptionKey[] // V3
}

export interface Schema {
  type: 'struct'
  'schema-id': number
  'identifier-field-ids'?: number[]
  fields: Field[]
}

export interface Field {
  id: number
  name: string
  required: boolean
  type: IcebergType
  doc?: string
  'initial-default'?: any
  'write-default'?: any
}

export type IcebergType =
  'unknown' |
  'boolean' |
  'int' |
  'long' |
  'float' |
  'double' |
  'date' |
  'time' |
  'timestamp' |
  'timestamptz' |
  'timestamp_ns' |
  'timestamptz_ns' |
  'string' |
  'uuid' |
  `fixed[${number}]` |
  'binary' |
  `decimal(${number},${number})` |
  `decimal(${number}, ${number})` |
  'variant' |
  'geometry' |
  `geometry(${string})` |
  'geography' |
  `geography(${string})` |
  IcebergNestedType

export type IcebergNestedType =
  | Schema
  | {
      type: 'list'
      'element-id': number
      'element-required': boolean
      element: IcebergType
    }
  | {
      type: 'map'
      'key-id': number
      key: IcebergType
      'value-id': number
      'value-required': boolean
      value: IcebergType
    }

export interface PartitionSpec {
  'spec-id': number
  fields: PartitionField[]
}
interface PartitionField {
  'source-id'?: number
  'source-ids'?: number[]
  'field-id': number
  name: string
  transform: PartitionTransform
}
export type PartitionTransform =
  'identity' |
  `bucket[${number}]` |
  `truncate[${number}]` |
  'year' |
  'month' |
  'day' |
  'hour' |
  'void' |
  string
interface PartitionStatistics {
  'snapshot-id': bigint
  'statistics-path': string
  'file-size-in-bytes': bigint
}

interface SortOrder {
  'order-id': number
  'fields': SortField[]
}
interface SortField {
  transform: string
  'source-id'?: number
  'source-ids'?: number[] // V3
  'direction': 'asc' | 'desc'
  'null-order': 'nulls-first' | 'nulls-last'
}

export interface NameMapping {
  names: string[]
  'field-id'?: number
  fields?: NameMapping[]
}

export interface Snapshot {
  'snapshot-id': number | bigint
  'parent-snapshot-id'?: number | bigint
  'sequence-number': number
  'timestamp-ms': number
  'manifest-list': string
  manifests?: Manifest[]
  summary: {
    // spec: "value of these fields should be of string type"
    operation: string
    // 'spark.app.id'?: string
    'added-data-files'?: string
    'added-records'?: string
    'deleted-data-files'?: string
    'deleted-records'?: string
    'removed-files-size'?: string
    'added-delete-files'?: string
    'removed-delete-files'?: string
    'added-position-deletes'?: string
    'removed-position-deletes'?: string
    'added-equality-deletes'?: string
    'removed-equality-deletes'?: string
    'added-dvs'?: string
    'removed-dvs'?: string
    'added-files-size'?: string
    'changed-partition-count'?: string
    'total-records'?: string
    'total-files-size'?: string
    'total-data-files'?: string
    'total-delete-files'?: string
    'total-position-deletes'?: string
    'total-equality-deletes'?: string
  }
  'schema-id'?: number
  'first-row-id'?: number // V3
  'added-rows'?: number // V3
  'key-id'?: string // V3
}

interface TableStatistics {
  'snapshot-id': number | bigint
  'statistics-path': string
  'file-size-in-bytes': bigint
  'file-footer-size-in-bytes': bigint
}

interface SnapshotLog {
  'timestamp-ms': number
  'snapshot-id': number | bigint
}

export interface SnapshotRef {
  'snapshot-id': number | bigint
  type: 'branch' | 'tag'
  'min-snapshots-to-keep'?: number
  'max-snapshot-age-ms'?: number
  'max-ref-age-ms'?: number
}

interface EncryptionKey {
  'key-id': string
  'key-metadata': string
}

/**
 * Iceberg REST `TableRequirement`s recognized by `checkRequirements`.
 */
export type TableRequirement =
  | { type: 'assert-create' }
  | { type: 'assert-table-uuid', uuid: string }
  | { type: 'assert-ref-snapshot-id', ref: string, 'snapshot-id': number | bigint | null }
  | { type: 'assert-next-row-id', 'next-row-id': number }
  | { type: 'assert-current-schema-id', 'current-schema-id': number }
  | { type: 'assert-last-assigned-field-id', 'last-assigned-field-id': number }
  | { type: 'assert-last-assigned-partition-id', 'last-assigned-partition-id': number }
  | { type: 'assert-default-spec-id', 'default-spec-id': number }
  | { type: 'assert-default-sort-order-id', 'default-sort-order-id': number }

/**
 * Subset of Iceberg REST `TableUpdate`s that the staging API emits.
 */
export type TableUpdate =
  | { action: 'add-snapshot', snapshot: Snapshot }
  | {
      action: 'set-snapshot-ref'
      'ref-name': string
      type: 'branch' | 'tag'
      'snapshot-id': number | bigint
      'min-snapshots-to-keep'?: number
      'max-snapshot-age-ms'?: number
      'max-ref-age-ms'?: number
    }
  | { action: 'set-properties', updates: Record<string, string> }
  | { action: 'remove-properties', removals: string[] }
  | { action: 'add-schema', schema: Schema, 'last-column-id'?: number }
  | { action: 'set-current-schema', 'schema-id': number }
  | { action: 'add-sort-order', 'sort-order': SortOrder }
  | { action: 'set-default-sort-order', 'sort-order-id': number }
  | { action: 'add-spec', spec: PartitionSpec }
  | { action: 'set-default-spec', 'spec-id': number }
  | { action: 'remove-snapshots', 'snapshot-ids': (number | bigint)[] }

/**
 * Argument passed to the `icebergTransaction` callback. Methods stage
 * operations against an in-memory working copy of the table metadata; the
 * accumulated updates ship in one commit when the callback resolves.
 */
export interface IcebergTransaction {
  append(options: { records: Record<string, any>[], sortOrderId?: number }): Promise<void>
  delete(options: {
    deletes: { file_path: string, pos: bigint | number }[]
    mode?: 'puffin' | 'parquet'
  }): Promise<void>
  setRef(options: {
    ref: string
    snapshotId: number | bigint
    type?: 'branch' | 'tag'
    minSnapshotsToKeep?: number
    maxSnapshotAgeMs?: number
    maxRefAgeMs?: number
  }): void
  expireSnapshots(options: { snapshotIds: (number | bigint)[] }): void
}

/**
 * Input to a commit function (`fileCatalogCommit`, `restCatalogUpdateTable`):
 * the CAS preconditions and updates a catalog must apply, and the
 * data/manifest files already written to storage (useful for cleanup on
 * commit failure). `snapshot` is absent for metadata-only updates like a
 * schema change.
 */
export interface StagedCommit {
  snapshot?: Snapshot
  requirements: TableRequirement[]
  updates: TableUpdate[]
  writtenFiles: string[]
}

/**
 * Output of an `icebergStage*` call that produces a snapshot (append, delete,
 * rewrite, setRef, expireSnapshots).
 */
export interface StagedUpdate extends StagedCommit {
  snapshot: Snapshot
}

/**
 * Output of `prepareAppend`. Captures everything that does NOT depend on the
 * eventually-committed snapshot's sequence number or parent: the data files,
 * the manifest, and the assigned snapshot id. `stageSnapshotForAppend` consumes
 * this against the freshest metadata to produce a `StagedUpdate` per attempt
 * without re-writing data or manifest files.
 */
export interface PreparedAppend {
  snapshotId: bigint
  manifestUuid: string
  formatVersion: 2 | 3
  manifestPath: string
  manifestLength: bigint
  partitionSpecId: number
  partitions: FieldSummary[]
  addedDataFilesCount: number
  addedRowCount: bigint
  addedFilesSize: bigint
  recordsCount: number
  /** Data files + the manifest. Useful for orphan cleanup if the commit ultimately fails. */
  writtenFiles: string[]
}

interface MetadataLog {
  'timestamp-ms': number
  'metadata-file': string
}

export interface Manifest {
  manifest_path: string
  manifest_length: bigint
  partition_spec_id: number
  content: 0 | 1 // 0=data, 1=deletes
  sequence_number?: bigint
  min_sequence_number?: bigint
  added_snapshot_id: bigint
  added_files_count: number
  existing_files_count: number
  deleted_files_count: number
  added_rows_count: bigint
  existing_rows_count: bigint
  deleted_rows_count: bigint
  partitions?: FieldSummary[]
  // key_metadata?: unknown
  first_row_id?: bigint | number
}

export interface ManifestEntry {
  status: 0 | 1 | 2 // 0=existing, 1=added, 2=deleted
  snapshot_id?: bigint
  sequence_number?: bigint
  file_sequence_number?: bigint
  partition_spec_id?: number
  data_file: DataFile
}

interface FieldSummary {
  contains_null: boolean
  contains_nan?: boolean | null
  lower_bound?: Uint8Array | null
  upper_bound?: Uint8Array | null
}

export interface DataFile {
  content: 0 | 1 | 2 // 0=data, 1=position_delete, 2=equality_delete
  file_path: string
  file_format: 'avro' | 'orc' | 'parquet' | 'puffin'
  partition: Record<string, unknown> // keyed by partition-field name in Avro
  record_count: bigint
  file_size_in_bytes: bigint
  column_sizes?: Record<number, bigint>
  value_counts?: Record<number, bigint>
  null_value_counts?: Record<number, bigint>
  nan_value_counts?: Record<number, bigint>
  lower_bounds?: Record<number, unknown>
  upper_bounds?: Record<number, unknown>
  // key_metadata?: string
  split_offsets?: bigint[]
  equality_ids?: number[]
  sort_order_id?: number
  first_row_id?: bigint | number
  referenced_data_file?: string
  content_offset?: bigint
  content_size_in_bytes?: bigint
}

export interface FilePositionDelete {
  file_path: string
  pos: bigint
}

/* Avro types */
interface AvroField {
  name: string
  type: AvroType
  doc?: string
  default?: any
  'field-id'?: number
}

export type AvroType = AvroPrimitiveType | AvroComplexType | AvroLogicalType

type AvroPrimitiveType = 'null' | 'boolean' | 'int' | 'long' | 'float' | 'double' | 'bytes' | 'string'

interface AvroRecord {
  type: 'record'
  name: string
  namespace?: string
  doc?: string
  aliases?: string[]
  fields: AvroField[]
  'schema-id'?: number
}

interface AvroArray {
  type: 'array'
  items: AvroType
  default?: any[]
  logicalType?: 'map' // Iceberg map-as-array annotation for non-string keys
  'element-id'?: number // Iceberg field id of the array element (e.g. equality_ids[*])
}

type AvroUnion = AvroType[]

type AvroDate = {
  type: 'int'
  logicalType: 'date'
}

type AvroTimeMillis = {
  type: 'int'
  logicalType: 'time-millis'
}

type AvroTimeMicros = {
  type: 'long'
  logicalType: 'time-micros'
}

type AvroDecimal = {
  type: 'bytes'
  logicalType: 'decimal'
  precision: number
  scale?: number
}

type AvroTimestampMillis = {
  type: 'long'
  logicalType: 'timestamp-millis'
  'adjust-to-utc'?: boolean
}

type AvroTimestampMicros = {
  type: 'long'
  logicalType: 'timestamp-micros'
  'adjust-to-utc'?: boolean
}

type AvroTimestampNanos = {
  type: 'long'
  logicalType: 'timestamp-nanos'
  'adjust-to-utc'?: boolean
}

type AvroLogicalTypeType =
  'date' |
  'decimal' |
  'duration' |
  'local-timestamp-millis' |
  'local-timestamp-micros' |
  'time-millis' |
  'time-micros' |
  'timestamp-millis' |
  'timestamp-micros' |
  'timestamp-nanos' |
  'uuid'

// catch-all: "implementations must ignore unknown logical types when reading"
type AvroGenericLogicalType = {
  type: AvroPrimitiveType
  logicalType: AvroLogicalTypeType
}

type AvroLogicalType =
  AvroDate |
  AvroTimeMillis |
  AvroTimeMicros |
  AvroDecimal |
  AvroTimestampMillis |
  AvroTimestampMicros |
  AvroTimestampNanos |
  AvroGenericLogicalType

type AvroFixed = {
  type: 'fixed'
  name: string
  size: number
  logicalType?: 'uuid' | 'decimal'
  precision?: number
  scale?: number
}

// Avro complex types: records, enums, arrays, maps, unions, fixed
type AvroComplexType = AvroRecord | AvroArray | AvroUnion | AvroFixed