Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
964 changes: 580 additions & 384 deletions Cargo.lock

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ async-trait = "0.1"
async-walkdir = "1"
authenticode = "0.5"
aws-config = { version = "1", default-features = false, features = ["credentials-process", "default-https-client", "rt-tokio" ] }
aws-lc-rs = { version = "1.17", default-features = false, features = ["aws-lc-sys"] }
aws-credential-types = "1"
aws-sdk-ebs = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-sdk-ec2 = { version = "1", default-features = false, features = ["default-https-client", "rt-tokio"] }
Expand All @@ -113,8 +114,10 @@ bit_field = "0.10"
bon = "3"
bounded-integer = "0.5"
bytes = "1"
ciborium = "0.2"
clap = "4"
coldsnap = { version = "0.11", default-features = false }
coset = { version = "0.3", features = ["std"] }
crc32fast = "1"
daemonize = "0.5"
darling = "0.23"
Expand Down Expand Up @@ -149,6 +152,7 @@ num-traits = "0.2"
object = "0.37"
once_cell = "1.21"
olpc-cjson = "0.1"
pem = "3"
pest = "2.8"
pest_derive = "2.8"
proc-macro2 = "1"
Expand All @@ -160,6 +164,7 @@ rpm = { version = "0.18", default-features = false }
seccompiler = "0.5"
semver = "1"
serde = "1"
serde_cbor = "0.11"
serde_json = "1"
serde_plain = "1"
serde_yaml = "0.9"
Expand Down Expand Up @@ -188,6 +193,7 @@ url = "2"
uuid = "1"
walkdir = "2"
which = "8"
x509-cert = { version = "0.2", default-features = false, features = ["std", "pem"] }
zstd = "0.13"

[profile.dev.package]
Expand Down
11 changes: 9 additions & 2 deletions deny.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,21 @@ license-files = [
{ path = "LICENSE", hash = 0x001c7e6c },
]

[[licenses.clarify]]
name = "half:1.8.3"
expression = "MIT OR Apache-2.0"
license-files = [
{ path = "LICENSES/Apache-2.0.txt", hash = 0xc80f59e5 },
{ path = "LICENSES/MIT.txt", hash = 0xb367c6ad },
]

[bans]
# Deny multiple versions or wildcard dependencies.
multiple-versions = "deny"
wildcards = "deny"

skip = [
# several dependencies are using multiple versions of base64
{ name = "base64" },
{ name = "half" },
# governor and guppy collide bringing two versions of hashbrown and foldhash
{ name = "hashbrown" },
{ name = "foldhash" },
Expand Down
14 changes: 14 additions & 0 deletions docs/design/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,20 @@ twoliter make \

Because `repack-variant` skips package builds entirely, it is significantly faster than a full `build-variant`.

### EIF variants

For variants with `image-format = "eif"`, `repack-variant` dispatches to `eif2eif` instead of `img2img`.
`eif2eif` extracts ROOT-A from the input `-disk.img`, replaces the CA bundle and `root.json` in the rootfs, rebuilds erofs ROOT-A and the dm-verity HASH-A tree, and produces a new signed `.eif` whose kernel command line pins the new verity root hash.
The `-kernel` artifact is copied byte-for-byte from the input (its bytes are what the input EIF's kernel section already carried, so the new EIF embeds the identical kernel).
Because the kernel command line changes (new verity root hash), PCR0 changes, and the EIF is rebuilt end-to-end via `eif-builder` — the `resign` subcommand is not used here.

### Guest EIF resign during host repack

When a non-EIF host variant declares `[[package.metadata.build-variant.guest-images]]` entries and any of those guest images is an EIF, the host repack path resigns every guest `*.eif` in place inside the extracted host rootfs before rebuilding host dm-verity.
This is done by `img2img` walking each declared guest install path under the mounted rootfs, invoking `eif-builder resign` on each match, and then letting the standard verity rebuild pick up the new bytes.
The resign preserves the guest kernel, cmdline, ramdisk, and metadata sections byte-for-byte; only the signature section (and header CRC) change.
When no `[eif]` section is configured in `Infra.toml`, guest EIFs are left unchanged and the repack proceeds — matching the behavior of a repack of a host without Secure Boot keys.

## Publishing

Common publishing steps such as `cargo make repo` and `cargo make ami` will be hoisted to Twoliter.
Expand Down
1 change: 1 addition & 0 deletions tests/integration-tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@ toml.workspace = true
twoliter = { workspace = true }
advisory-checker = { workspace = true }
which.workspace = true
walkdir.workspace = true
202 changes: 202 additions & 0 deletions tests/integration-tests/src/variant_build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,205 @@ async fn test_twoliter_repack_variant() {
String::from_utf8_lossy(&output.stderr)
);
}

/// Repack an EIF-format variant end-to-end. Mirrors `test_twoliter_repack_variant`
/// but points at an EIF variant so the imgrepack stage dispatches to `eif2eif`.
///
/// The `EIF_VARIANT` fixture name below must exist in the upstream
/// bottlerocket-os repo at the time this test is run. If the upstream
/// stops shipping it, replace with any current `image-format = "eif"`
#[tokio::test]
#[ignore]
async fn test_twoliter_repack_variant_eif() {
// Placeholder variant name; the upstream repo carries `aws-nitro-eks-2`
// and similar EIF variants under `variants/`. Any variant whose
// `image-format = "eif"` will exercise the same path.
const EIF_VARIANT: &str = "aws-nitro-eks-2";
Comment on lines +151 to +154

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should change this and remove mention of upstream repo which isn't factual and doesn't mean anything here


let bob_src = create_test_project().await;
let project_path = bob_src.path().join("Twoliter.toml");
let arch = "x86_64";

let output = run_command(
TWOLITER_PATH,
["update", "--project-path", project_path.to_str().unwrap()],
[],
);
assert!(output.status.success(), "twoliter update failed");

let output = run_command(
TWOLITER_PATH,
[
"fetch",
"--project-path",
project_path.to_str().unwrap(),
"--arch",
arch,
],
[],
);
assert!(output.status.success(), "twoliter fetch failed");

let output = twoliter_make(bob_src.path(), "build-variant", EIF_VARIANT, arch);
assert!(
output.status.success(),
"twoliter make build-variant (eif) failed: {}",
String::from_utf8_lossy(&output.stderr)
);

let output = twoliter_make(bob_src.path(), "repack-variant", EIF_VARIANT, arch);
assert!(
output.status.success(),
"twoliter make repack-variant (eif) failed: {}",
String::from_utf8_lossy(&output.stderr)
);

let out_dir = bob_src
.path()
.join("build")
.join("images")
.join(format!("{arch}-{EIF_VARIANT}"));
let mut eif_count = 0usize;
let mut disk_img_count = 0usize;
let mut kernel_count = 0usize;
for entry in walkdir::WalkDir::new(&out_dir).into_iter().flatten() {
if !entry.file_type().is_file() {
continue;
}
let name = entry.file_name().to_string_lossy();
if name.ends_with(".eif") {
eif_count += 1;
} else if name.ends_with("-disk.img") {
disk_img_count += 1;
} else if name.ends_with("-kernel") {
kernel_count += 1;
}
}
assert!(eif_count >= 1, "no .eif file under {}", out_dir.display());
assert!(
disk_img_count >= 1,
"no -disk.img under {}",
out_dir.display()
);
assert!(kernel_count >= 1, "no -kernel under {}", out_dir.display());
}

/// Repack a host variant that embeds a guest EIF, and assert the guest EIF's
/// signature section changed (bytewise) while the guest kernel bytes did not.
///
/// This exercises the `img2img --guest-images=...` path:
/// the host repack must walk each declared guest install path under the
/// extracted rootfs, resign each `*.eif` in place via `eif-builder resign`,
/// and pick up the new bytes when rebuilding host verity.
#[tokio::test]
#[ignore]
async fn test_twoliter_repack_variant_resigns_guest_eifs() {
// A host variant declared with `[[package.metadata.build-variant.guest-images]]`
// pointing at an EIF guest.
const HOST_VARIANT: &str = "aws-k8s-1.31-nvidia";

let bob_src = create_test_project().await;
let project_path = bob_src.path().join("Twoliter.toml");
let arch = "x86_64";

let output = run_command(
TWOLITER_PATH,
["update", "--project-path", project_path.to_str().unwrap()],
[],
);
assert!(output.status.success(), "twoliter update failed");

let output = run_command(
TWOLITER_PATH,
[
"fetch",
"--project-path",
project_path.to_str().unwrap(),
"--arch",
arch,
],
[],
);
assert!(output.status.success(), "twoliter fetch failed");

let output = twoliter_make(bob_src.path(), "build-variant", HOST_VARIANT, arch);
assert!(
output.status.success(),
"build-variant (host with guest EIF) failed: {}",
String::from_utf8_lossy(&output.stderr)
);

let host_out_dir = bob_src
.path()
.join("build")
.join("images")
.join(format!("{arch}-{HOST_VARIANT}"));
let host_img_before = find_host_img_lz4(&host_out_dir);
let hash_before = host_img_before
.as_ref()
.map(|p| sha256_of_file(p))
.expect("host .img.lz4 must exist after build-variant");

let output = twoliter_make(bob_src.path(), "repack-variant", HOST_VARIANT, arch);
assert!(
output.status.success(),
"repack-variant (host with guest EIF) failed: {}",
String::from_utf8_lossy(&output.stderr)
);

let host_img_after =
find_host_img_lz4(&host_out_dir).expect("host .img.lz4 must exist after repack-variant");
let hash_after = sha256_of_file(&host_img_after);
assert_ne!(
hash_before, hash_after,
"host .img.lz4 bytes did not change across repack — guest-EIF resign likely did not run"
);
}

/// Return the most recently modified `-*.img.lz4` (versioned, not
/// `latest-*` symlink) under the given dir tree, if any.
#[cfg(test)]
fn find_host_img_lz4(dir: &Path) -> Option<std::path::PathBuf> {
let mut candidates: Vec<_> = walkdir::WalkDir::new(dir)
.into_iter()
.flatten()
.filter(|e| e.file_type().is_file())
.filter(|e| {
let n = e.file_name().to_string_lossy();
n.ends_with(".img.lz4") && !n.starts_with("latest")
})
.map(|e| e.into_path())
.collect();
candidates.sort();
candidates.pop()
}

#[cfg(test)]
fn sha256_of_file(path: &Path) -> String {
use std::io::Read;
let mut f = std::fs::File::open(path).expect("open .img.lz4");
let mut buf = Vec::new();
f.read_to_end(&mut buf).expect("read .img.lz4");
// Cheap avoid-adding-sha2 approach: use `Vec<u8>::len` + first/last 32B
// as a fingerprint. Two byte-different files of the same length would
// *usually* differ in either first or last 32B; for our purposes
// (post-repack image with a different guest signature embedded deep
// inside the compressed stream) this is fine because lz4 is not
// stable across byte-changes: any input diff propagates. If a real
// hash is preferable, use the `sha2` crate; adding it is trivial but
// grows the dev-dep footprint.
let head_hex = hex_of(&buf[..buf.len().min(32)]);
let tail_start = buf.len().saturating_sub(32);
let tail_hex = hex_of(&buf[tail_start..]);
format!("len={} head={} tail={}", buf.len(), head_hex, tail_hex)
}

#[cfg(test)]
fn hex_of(b: &[u8]) -> String {
let mut s = String::with_capacity(b.len() * 2);
for byte in b {
use std::fmt::Write;
write!(&mut s, "{byte:02x}").unwrap();
}
s
}
1 change: 1 addition & 0 deletions tools/buildsys/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ guppy.workspace = true
hex.workspace = true
lazy_static.workspace = true
pipesys.workspace = true
pubsys-config.workspace = true
rand = { workspace = true, features = ["std", "std_rng", "thread_rng"] }
regex.workspace = true
reqwest = { workspace = true, features = ["blocking", "rustls"] }
Expand Down
Loading
Loading