Skip to content

Commit a3a8aa1

Browse files
committed
internal: Implement a new storage layer for better caching and local/remote interop. (#2582)
* Use bytes. * Polish. * Move blob to crate. * Add manifest. * Rename compat. * Flatten manifest. * Start on storage. * Update starbase. * Start on backend. * Impl store flow. * Start on hydrate. * Move store to backend. * Move retrieve to backend. * Polish. * Rework CAS. * Update tests. * Polish. * Add chunking. * Add tests.
1 parent 34d8e42 commit a3a8aa1

47 files changed

Lines changed: 3273 additions & 365 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.vscode/settings.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"editor.fontSize": 15,
33
"editor.fontFamily": "Source Code Pro",
4-
"editor.defaultFormatter": "oxc.oxc-vscode",
4+
// "editor.defaultFormatter": "oxc.oxc-vscode",
55
"editor.formatOnSave": true,
66
"editor.formatOnSaveMode": "file",
77
"editor.codeActionsOnSave": {

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44

55
#### 🚀 Updates
66

7+
- **CAS**
8+
- Further improvements to the content-addressable storage (CAS) cache, including better error handling and performance improvements, and a new API for local/remote storage interoperability.
9+
- When copying files to/from the CAS, we now use OS reflink's when available, which can improve performance and reduce disk space usage.
710
- **Processes**
811
- Improved our "stream and capture output" child process handling to operate on bytes instead of
912
lines, which should resolve some edge cases with output not being written to the console, or

Cargo.lock

Lines changed: 57 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ async-recursion = "1.1.1"
1010
async-stream = "0.3.6"
1111
blake3 = { version = "1.8.5", features = ["mmap"] }
1212
bazel-remote-apis = { version = "0.27.0", features = ["serde"] }
13+
bytes = "1.12.0"
1314
async-trait = "0.1.89"
1415
chrono = { version = "0.4.44", features = ["serde"] }
1516
cd_env = "0.4.1"
@@ -28,7 +29,6 @@ criterion = { package = "codspeed-criterion-compat", version = "4.7.0", default-
2829
daggy = { version = "0.9.0", features = ["serde-1"] }
2930
dirs = "6.0.0"
3031
futures = "0.3.31"
31-
hex = "0.4.3"
3232
indexmap = "2.13.0"
3333
iocraft = "0.8.3"
3434
libc = "0.2.186"
@@ -54,18 +54,18 @@ serde = { version = "1.0.228", features = ["derive"] }
5454
serde_json = "1.0.150"
5555
sha2 = "0.11.0"
5656
starbase = { version = "0.12.2", features = ["clap", "miette"] }
57-
starbase_archive = { version = "0.13.1", default-features = false, features = [
57+
starbase_archive = { version = "0.13.2", default-features = false, features = [
5858
"miette",
5959
"tar-all",
6060
"zip-all",
6161
] }
62-
starbase_args = { version = "0.1.10" }
63-
starbase_console = { version = "0.6.27", features = ["miette"] }
64-
starbase_id = { version = "0.3.2", features = ["miette", "schematic"] }
65-
starbase_sandbox = "0.11.0"
66-
starbase_shell = "0.12.4"
62+
starbase_args = { version = "0.1.11" }
63+
starbase_console = { version = "0.6.29", features = ["miette"] }
64+
starbase_id = { version = "0.3.3", features = ["miette", "schematic"] }
65+
starbase_sandbox = "0.11.1"
66+
starbase_shell = "0.12.5"
6767
starbase_styles = { version = "0.6.1", features = ["relative-path"] }
68-
starbase_utils = { version = "0.13.6", default-features = false, features = [
68+
starbase_utils = { version = "0.13.8", default-features = false, features = [
6969
"editor-config",
7070
"glob-cache",
7171
"miette",

crates/app/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ moon_workspace = { path = "../workspace" }
4848
moon_workspace_graph = { path = "../workspace-graph" }
4949
async-recursion = { workspace = true }
5050
async-trait = { workspace = true }
51-
bytes = "1.12.0"
51+
bytes = { workspace = true }
5252
ci_env = { workspace = true }
5353
clap = { workspace = true, features = [
5454
"color",

crates/blob/Cargo.toml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[package]
2+
name = "moon_blob"
3+
version = "0.0.1"
4+
edition = "2024"
5+
license = "MIT"
6+
description = "Binary large objects."
7+
homepage = "https://moonrepo.dev/moon"
8+
repository = "https://github.com/moonrepo/moon"
9+
publish = false
10+
11+
[dependencies]
12+
moon_common = { path = "../common" }
13+
moon_hash = { path = "../hash" }
14+
bytes = { workspace = true }
15+
miette = { workspace = true }
16+
serde = { workspace = true }
17+
starbase_utils = { workspace = true }
18+
19+
[dev-dependencies]
20+
starbase_sandbox = { workspace = true }
21+
starbase_utils = { workspace = true }
22+
23+
[lints]
24+
workspace = true

crates/blob/src/blob.rs

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
use bytes::Bytes;
2+
use miette::IntoDiagnostic;
3+
use moon_common::path::WorkspaceRelativePathBuf;
4+
use moon_hash::Digest;
5+
use serde::Serialize;
6+
use starbase_utils::{fs, json};
7+
use std::fmt::Debug;
8+
use std::path::Path;
9+
10+
pub struct BlobSource {
11+
pub content: BlobContent,
12+
pub digest: Digest,
13+
}
14+
15+
pub enum BlobContent {
16+
Inline(Bytes),
17+
File(WorkspaceRelativePathBuf),
18+
}
19+
20+
#[derive(Clone)]
21+
pub struct Blob {
22+
pub bytes: Bytes,
23+
pub digest: Digest,
24+
}
25+
26+
impl Blob {
27+
pub fn new(digest: Digest, bytes: Vec<u8>) -> Self {
28+
Blob {
29+
digest,
30+
bytes: Bytes::from(bytes),
31+
}
32+
}
33+
34+
pub fn from_bytes(bytes: Vec<u8>) -> miette::Result<Self> {
35+
Ok(Blob::new(Digest::from_bytes(&bytes)?, bytes))
36+
}
37+
38+
pub fn from_data<T: Serialize>(data: T) -> miette::Result<Self> {
39+
Self::from_bytes(json::serde_json::to_vec(&data).into_diagnostic()?)
40+
}
41+
42+
pub fn from_file<T: AsRef<Path>>(path: T) -> miette::Result<Self> {
43+
Self::from_bytes(fs::read_file_bytes(path.as_ref())?)
44+
}
45+
}
46+
47+
impl Debug for Blob {
48+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
49+
f.debug_struct("Blob")
50+
.field("digest", &self.digest)
51+
.finish()
52+
}
53+
}

crates/blob/src/lib.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
mod blob;
2+
3+
pub use blob::*;
4+
pub use bytes::Bytes;

crates/blob/tests/blob_test.rs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
use bytes::Bytes;
2+
use moon_blob::Blob;
3+
use moon_hash::ContentHash;
4+
use serde::Serialize;
5+
use starbase_sandbox::create_empty_sandbox;
6+
use starbase_utils::json::serde_json;
7+
8+
// Sanity-pin: SHA-256 of "abc" — used to assert that the digest path matches
9+
// the known-good algorithm rather than just "some" hash.
10+
const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
11+
12+
#[derive(Serialize)]
13+
struct Sample {
14+
name: &'static str,
15+
count: u32,
16+
}
17+
18+
mod blob {
19+
use super::*;
20+
21+
#[test]
22+
fn from_bytes_retains_bytes_and_digest() {
23+
let blob = Blob::from_bytes(b"abc".to_vec()).unwrap();
24+
25+
assert_eq!(blob.bytes, Bytes::from("abc"));
26+
assert_eq!(blob.digest.hash.as_hex(), ABC_SHA256);
27+
assert_eq!(blob.digest.size, 3);
28+
}
29+
30+
#[test]
31+
fn from_data_round_trips_through_json() {
32+
let sample = Sample {
33+
name: "y",
34+
count: 7,
35+
};
36+
let blob = Blob::from_data(&sample).unwrap();
37+
38+
// The blob's bytes should be the canonical serde_json output. We then
39+
// confirm that re-hashing those bytes reproduces the blob's digest.
40+
let canonical = serde_json::to_vec(&sample).unwrap();
41+
assert_eq!(blob.bytes, canonical);
42+
assert_eq!(
43+
blob.digest.hash,
44+
ContentHash::hash_bytes(&canonical).unwrap()
45+
);
46+
}
47+
48+
#[test]
49+
fn from_file_round_trips() {
50+
let sandbox = create_empty_sandbox();
51+
sandbox.create_file("payload.bin", "hello bytes");
52+
53+
let blob = Blob::from_file(sandbox.path().join("payload.bin")).unwrap();
54+
55+
assert_eq!(blob.bytes, Bytes::from("hello bytes"));
56+
assert_eq!(blob.digest.size, "hello bytes".len() as i64);
57+
assert_eq!(
58+
blob.digest.hash,
59+
ContentHash::hash_bytes(b"hello bytes").unwrap()
60+
);
61+
}
62+
63+
#[test]
64+
fn debug_does_not_dump_bytes() {
65+
// Bytes may be large (or sensitive). The Debug impl deliberately omits
66+
// them — this regression guard catches an accidental `derive(Debug)`
67+
// that would expose the full payload.
68+
let blob = Blob::from_bytes(vec![0xAB, 0xCD, 0xEF]).unwrap();
69+
let dbg = format!("{:?}", blob);
70+
assert!(dbg.contains("digest"));
71+
assert!(!dbg.contains("bytes"));
72+
}
73+
}

crates/cache-local/Cargo.toml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[package]
2+
name = "moon_cache_local"
3+
version = "0.0.1"
4+
edition = "2024"
5+
license = "MIT"
6+
description = "Local cache storage."
7+
homepage = "https://moonrepo.dev/moon"
8+
repository = "https://github.com/moonrepo/moon"
9+
publish = false
10+
11+
[dependencies]
12+
moon_blob = { path = "../blob" }
13+
moon_cache_storage = { path = "../cache-storage" }
14+
moon_cas = { path = "../cas" }
15+
moon_common = { path = "../common" }
16+
moon_config = { path = "../config" }
17+
moon_hash = { path = "../hash" }
18+
async-trait = { workspace = true }
19+
miette = { workspace = true }
20+
rustc-hash = { workspace = true }
21+
serde_json = { workspace = true }
22+
tokio = { workspace = true }
23+
tracing = { workspace = true }
24+
25+
[dev-dependencies]
26+
starbase_sandbox = { workspace = true }
27+
28+
[lints]
29+
workspace = true

0 commit comments

Comments
 (0)