Skip to content

Commit 8466d2b

Browse files
zxch3ncodex
andauthored
feat(moon): bootstrap encoding codec primitives (#903)
* docs(moon): add detailed plan for encoding format * docs(moon): design Change/Op IR for testing * docs(moon): rename Change/Op structs without IR suffix * feat(moon): bootstrap codec primitives * feat(moon): add lz4 frame decoder * feat(moon): implement sstable codec * feat(moon): parse fastsnapshot/fastupdates bodies * feat(moon): add container id/wrapper and change-block id * feat(moon): add serde_columnar codecs * feat(moon): add custom value codec * fix(moon): match serde_columnar delta-of-delta empty encoding * feat(moon): add ChangeBlock encoded-block codec * feat(moon): add incremental rle/delta-of-delta decoding helpers * feat(moon): decode change block header/meta and arenas * feat(moon): add PositionArena v2 codec * feat(moon): extend serde_columnar and value codecs * feat(moon): decode ChangeBlock ops into Change/Op * feat(moon): encode and transcode document blobs * feat(moon): add JS transcode CLI with Node fs * chore: sync Cargo.lock * fix(moon): handle serde_columnar wrappers * fix(moon): decode ContainerArena as columnar vec * fix(moon): decode ContainerArena as vec of rows * feat(moon): add postcard LoroValue and vv/frontiers codecs * feat(moon): validate fastsnapshot oplog/state kv stores * feat(moon): transcode container state snapshots * test(loro): add optional MoonBit transcode e2e * docs(moon): add ChangeBlock encoding notes * feat(moon): re-encode FastUpdates ChangeBlocks * test(loro): broaden MoonBit transcode e2e coverage * docs(moon): note current e2e harness * feat(moon): add decode-updates JSON output * test(loro): verify Moon decodes text insert op * feat(moon): export JsonSchema updates * test(loro): verify Moon JsonSchema export * test(loro): expand Moon JsonSchema export coverage * docs(moon): document JsonSchema export * feat(moon): encode FastUpdates from JsonSchema * test(loro): e2e Moon encodes JsonSchema to updates * docs(moon): document JsonSchema encode * chore(skills): vendor moonbit agent guide Upstream: moonbitlang/moonbit-agent-guide@e2b5be3 * refactor(moon): split JsonSchema export/import modules * refactor(moon): split serde_columnar strategies * refactor(moon): split ChangeBlock codec into files * refactor(moon): derive Show/Eq for codec errors * refactor(moon): make JsonSchema import field access safe * style(moon): format with moon fmt * refactor(moon): split state snapshot codecs * refactor(moon): split postcard codecs * refactor(moon): split JsonSchema import * refactor(moon): split JsonSchema export * refactor(moon): split custom value codec * refactor(moon): split sstable codec * refactor(moon): split change block encoder * refactor(moon): split change block encoded ops * refactor(moon): split delta-of-delta strategy * test(moon): split change block tests * refactor(moon): split changes_json renderer * refactor(moon): split container id codecs * refactor(moon): split JsonSchema import ops * refactor(moon): extract serde_columnar rle core * refactor(moon): dedupe delete-seq decode * refactor(moon): split postcard common value codec * refactor(moon): move delta-of-delta helpers * chore: update Cargo.lock * docs(moon): define ultimate e2e golden tests * feat(loro): add moon golden generator CLI * feat(moon): export deep JSON from snapshots * fix(moon): align JsonSchema start_version for FastUpdates * test(loro): add Moon golden JsonSchema/deep-json checks * docs(moon): link ultimate golden test implementations * test(loro): enable serde_json float_roundtrip * test(loro): expand moon golden generator coverage * fix(moon): sort JsonSchema changes by lamport * test(loro): expand moon golden coverage matrix * docs(moon): update e2e coverage status * test(loro): add counter snapshot e2e (optional) * fix(moon): export Counter ops in JsonSchema * test(loro): add Counter JsonSchema e2e check * feat(moon): support Counter JsonSchema import and LZ4 SSTable encode * fix(moon): make lz4 encoder compatible with moon 0.1 * chore(loro): enable counter feature by default * test(loro): add moon snapshot fuzz driver * Fix dead container cache invalidation * docs(encoding): align spec with impl * fix(moon): preserve integer values in jsonschema import - Decode JsonSchema number tokens into i64 when integral, even if the JSON parser doesn't retain the lexical repr. - Add an edge e2e case covering varint/length boundaries and encode-jsonschema roundtrip. * moon: harden encode-jsonschema * docs/tests: strengthen Moon codec fuzzing --------- Co-authored-by: Codex CLI <codex@openai.com>
1 parent c22de68 commit 8466d2b

File tree

158 files changed

+17278
-9
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

158 files changed

+17278
-9
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
*.ipynb
44
flamegraph.svg
55
target
6+
moon/_build/
7+
moon_*_fuzz_artifacts*/
68
dhat-heap.json
79
.DS_Store
810
node_modules/

Cargo.lock

Lines changed: 91 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
use bytes::Bytes;
2+
use loro_kv_store::sstable::SsTable;
3+
4+
#[test]
5+
fn import_moon_encoded_sstable() {
6+
let bytes = Bytes::from_static(include_bytes!("testdata/moon_sstable_simple.bin"));
7+
let table = SsTable::import_all(bytes, true).unwrap();
8+
let kvs: Vec<(Bytes, Bytes)> = table.iter().collect();
9+
10+
assert_eq!(
11+
kvs,
12+
vec![
13+
(Bytes::from_static(b"a"), Bytes::from_static(b"1")),
14+
(Bytes::from_static(b"ab"), Bytes::from_static(b"2")),
15+
(Bytes::from_static(b"z"), Bytes::from_static(b"")),
16+
]
17+
);
18+
}
19+
50 Bytes
Binary file not shown.

crates/loro-internal/src/state/dead_containers_cache.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ impl DocState {
2121
pub(crate) fn is_deleted(&mut self, idx: ContainerIdx) -> bool {
2222
#[cfg(not(debug_assertions))]
2323
{
24-
if let Some(is_deleted) = self.dead_containers_cache.cache.get(&idx) {
25-
return *is_deleted;
24+
// Cache stores only deleted containers.
25+
if self.dead_containers_cache.cache.contains_key(&idx) {
26+
return true;
2627
}
2728
}
2829

@@ -52,8 +53,14 @@ impl DocState {
5253
}
5354
}
5455

55-
for idx in visited {
56-
self.dead_containers_cache.cache.insert(idx, is_deleted);
56+
if is_deleted {
57+
for idx in visited {
58+
self.dead_containers_cache.cache.insert(idx, true);
59+
}
60+
} else {
61+
for idx in visited {
62+
self.dead_containers_cache.cache.remove(&idx);
63+
}
5764
}
5865

5966
is_deleted

crates/loro/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ tracing = { workspace = true }
2626
rustc-hash = { workspace = true }
2727

2828
[dev-dependencies]
29-
serde_json = "1.0.87"
29+
serde_json = { version = "1.0.87", features = ["float_roundtrip"] }
3030
anyhow = "1.0.83"
3131
ctor = "0.2"
3232
dev-utils = { path = "../dev-utils" }
@@ -37,6 +37,7 @@ base64 = "0.22.1"
3737
serial_test = "3"
3838

3939
[features]
40+
default = ["counter"]
4041
counter = ["loro-internal/counter"]
4142
jsonpath = ["loro-internal/jsonpath"]
4243
logging = ["loro-internal/logging"]

0 commit comments

Comments
 (0)