|
| 1 | +// Iceberg snapshot ids are 64-bit longs. Icebird's metadata reader parses |
| 2 | +// integers above Number.MAX_SAFE_INTEGER (2^53-1) as BigInt to avoid lossy |
| 3 | +// doubles. The write path must round-trip those BigInts: a plain |
| 4 | +// `JSON.stringify` throws "Do not know how to serialize a BigInt", and a |
| 5 | +// naive replacer that emits a quoted string would corrupt the metadata |
| 6 | +// (Iceberg expects bare JSON numbers). These tests pin the contract that a |
| 7 | +// commit re-serializing loaded metadata with a snapshot id > 2^53 succeeds |
| 8 | +// and preserves the id exactly. |
| 9 | + |
| 10 | +import { describe, expect, it } from 'vitest' |
| 11 | +import { fileCatalog } from '../../src/catalog/file.js' |
| 12 | +import { loadLatestFileCatalogMetadata } from '../../src/metadata.js' |
| 13 | +import { icebergAppend, icebergCreateTable, icebergExpireSnapshots } from '../../src/write/write.js' |
| 14 | +import { memResolver } from '../helpers.js' |
| 15 | + |
| 16 | +/** |
| 17 | + * @import {Schema} from '../../src/types.js' |
| 18 | + */ |
| 19 | + |
| 20 | +// > 2^53 (9007199254740992) and < 2^63, so it is a valid 64-bit long that a |
| 21 | +// double cannot represent. parseIcebergJson keeps it as a BigInt. |
| 22 | +const BIG_ID = 9151314442816847871n |
| 23 | + |
| 24 | +/** @type {Schema} */ |
| 25 | +const schema = { |
| 26 | + type: 'struct', |
| 27 | + 'schema-id': 0, |
| 28 | + fields: [ |
| 29 | + { id: 1, name: 'id', required: true, type: 'long' }, |
| 30 | + { id: 2, name: 'msg', required: false, type: 'string' }, |
| 31 | + ], |
| 32 | +} |
| 33 | + |
| 34 | +/** |
| 35 | + * Rewrite every bare-number occurrence of `from` to `to` in the latest |
| 36 | + * on-disk metadata file, simulating a table whose snapshot id exceeds 2^53. |
| 37 | + * Operates on the raw JSON text so it does not depend on the serializer |
| 38 | + * under test. Word boundaries avoid rewriting a coincidental substring of a |
| 39 | + * larger number (e.g. a timestamp). |
| 40 | + * |
| 41 | + * @param {Map<string, Uint8Array>} files |
| 42 | + * @param {string} tableUrl |
| 43 | + * @param {bigint} from |
| 44 | + * @param {bigint} to |
| 45 | + */ |
| 46 | +function rewriteSnapshotId(files, tableUrl, from, to) { |
| 47 | + const dir = `${tableUrl}/metadata/` |
| 48 | + let latestKey = '' |
| 49 | + let latestVersion = -1 |
| 50 | + for (const key of files.keys()) { |
| 51 | + const m = key.startsWith(dir) && key.match(/\/v(\d+)\.metadata\.json$/) |
| 52 | + if (m && Number(m[1]) > latestVersion) { |
| 53 | + latestVersion = Number(m[1]) |
| 54 | + latestKey = key |
| 55 | + } |
| 56 | + } |
| 57 | + if (!latestKey) throw new Error('no metadata file found') |
| 58 | + const text = new TextDecoder().decode(/** @type {Uint8Array} */ (files.get(latestKey))) |
| 59 | + // Only rewrite bare JSON number values (`"key": <id>`), never digits inside |
| 60 | + // a quoted string such as the `manifest-list` path (`.../snap-<id>-...avro`), |
| 61 | + // whose physical file on disk keeps the original id in its name. |
| 62 | + const re = new RegExp(`(?<=: )${from.toString()}(?=[,\\n}\\]])`, 'g') |
| 63 | + const rewritten = text.replace(re, to.toString()) |
| 64 | + files.set(latestKey, new TextEncoder().encode(rewritten)) |
| 65 | +} |
| 66 | + |
| 67 | +describe('commit round-trips snapshot ids above 2^53', () => { |
| 68 | + it('expireSnapshots preserves a surviving snapshot id > 2^53', async () => { |
| 69 | + const tableUrl = 'http://test/bigint-expire' |
| 70 | + const { resolver, files, lister } = memResolver() |
| 71 | + const catalog = fileCatalog({ resolver, lister }) |
| 72 | + |
| 73 | + await icebergCreateTable({ catalog, tableUrl, schema }) |
| 74 | + await icebergAppend({ catalog, tableUrl, records: [{ id: 1n, msg: 'a' }] }) |
| 75 | + await icebergAppend({ catalog, tableUrl, records: [{ id: 2n, msg: 'b' }] }) |
| 76 | + |
| 77 | + // Promote the current (newest) snapshot's id above 2^53 on disk. |
| 78 | + const before = await loadLatestFileCatalogMetadata({ tableUrl, resolver, lister }) |
| 79 | + const currentId = before.metadata['current-snapshot-id'] |
| 80 | + if (currentId == null) throw new Error('expected a current snapshot') |
| 81 | + const olderId = (before.metadata.snapshots ?? []) |
| 82 | + .map(s => s['snapshot-id']) |
| 83 | + .find(id => BigInt(id) !== BigInt(currentId)) |
| 84 | + if (olderId == null) throw new Error('expected two snapshots') |
| 85 | + rewriteSnapshotId(files, tableUrl, BigInt(currentId), BIG_ID) |
| 86 | + |
| 87 | + // Expiring the older snapshot re-serializes metadata that still holds the |
| 88 | + // BIG_ID snapshot. Before the fix this threw on JSON.stringify(BigInt). |
| 89 | + await icebergExpireSnapshots({ catalog, tableUrl, snapshotIds: [olderId] }) |
| 90 | + |
| 91 | + const after = await loadLatestFileCatalogMetadata({ tableUrl, resolver, lister }) |
| 92 | + const ids = (after.metadata.snapshots ?? []).map(s => s['snapshot-id']) |
| 93 | + expect(ids).toEqual([BIG_ID]) |
| 94 | + expect(after.metadata['current-snapshot-id']).toBe(BIG_ID) |
| 95 | + expect(after.metadata.refs?.main?.['snapshot-id']).toBe(BIG_ID) |
| 96 | + }) |
| 97 | + |
| 98 | + it('append records parent-snapshot-id > 2^53 without precision loss', async () => { |
| 99 | + const tableUrl = 'http://test/bigint-append' |
| 100 | + const { resolver, files, lister } = memResolver() |
| 101 | + const catalog = fileCatalog({ resolver, lister }) |
| 102 | + |
| 103 | + await icebergCreateTable({ catalog, tableUrl, schema }) |
| 104 | + await icebergAppend({ catalog, tableUrl, records: [{ id: 1n, msg: 'a' }] }) |
| 105 | + |
| 106 | + // Promote the only snapshot's id above 2^53 on disk. A subsequent append |
| 107 | + // will reference it as parent-snapshot-id (a BigInt), which the commit |
| 108 | + // must serialize as a bare JSON number. |
| 109 | + const before = await loadLatestFileCatalogMetadata({ tableUrl, resolver, lister }) |
| 110 | + const parentId = before.metadata['current-snapshot-id'] |
| 111 | + if (parentId == null) throw new Error('expected a current snapshot') |
| 112 | + rewriteSnapshotId(files, tableUrl, BigInt(parentId), BIG_ID) |
| 113 | + |
| 114 | + await icebergAppend({ catalog, tableUrl, records: [{ id: 2n, msg: 'b' }] }) |
| 115 | + |
| 116 | + const after = await loadLatestFileCatalogMetadata({ tableUrl, resolver, lister }) |
| 117 | + const child = (after.metadata.snapshots ?? []) |
| 118 | + .find(s => BigInt(s['snapshot-id']) !== BIG_ID) |
| 119 | + if (!child) throw new Error('expected the appended snapshot') |
| 120 | + // Exact BigInt equality proves the parent id was not coerced to a double. |
| 121 | + expect(child['parent-snapshot-id']).toBe(BIG_ID) |
| 122 | + }) |
| 123 | +}) |
0 commit comments