Skip to content

Commit c239f03

Browse files
shortstackedclaude
andcommitted
perf(core): Write CI image tarballs straight from BuildKit (no-changelog)
The e2e image path made four passes over ~3GB to produce one file: bake exported an OCI tarball, streamed it to dockerd, dockerd unpacked it into overlay2, then `docker save` read it back out and piped it through zstd. BuildKit writes the docker-archive directly instead. `compression=zstd` compresses the layers inside the archive, so the separate compression pass goes away too - the file is cache-ready as written, and `docker load` handles the decompression on the other side. This lands on prepare-docker, which runs ~1000-1400x/week and is the most frequent Docker job in CI. Output is one archive per target in DOCKER_BUILD_TARBALL_DIR, so the cache entry becomes a directory and the variable target count (distroless, pc) needs no special casing. The pc variant loads its archive before asserting pointer compression, since nothing reaches the daemon implicitly any more. Trade-off: with no images in the daemon, `docker images` cannot report a size, so the e2e docker-image-size metric stops being emitted. Left as Unknown rather than substituted with the compressed archive size, which would silently change what the metric means. That image is built with INCLUDE_TEST_CONTROLLER=true and was never the shipped artifact, so image size belongs on the release path. Verified locally: bake writes n8n.tar (327M) and runners.tar (162M) with the expected RepoTags, and `docker load -i` restores a working image. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent f488323 commit c239f03

3 files changed

Lines changed: 36 additions & 11 deletions

File tree

.github/actions/build-n8n-docker/action.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ runs:
2424
uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3
2525
with:
2626
key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }}
27-
path: /tmp/n8n-image.tar.zst
27+
path: /tmp/n8n-images
2828
lookup-only: true
2929

3030
- name: Build Docker image
@@ -36,30 +36,30 @@ runs:
3636
docker-cache-key: n8n-io/n8n
3737
env:
3838
INCLUDE_TEST_CONTROLLER: 'true'
39+
DOCKER_BUILD_TARBALL_DIR: /tmp/n8n-images
3940
# Selects the n8n-pc bake target, which carries the pointer-compressed pins.
4041
DOCKER_BUILD_PC: ${{ inputs.build-variant == 'pc' }}
4142

4243
- name: Assert pointer compression
4344
if: steps.cache-check.outputs.cache-hit != 'true' && inputs.build-variant == 'pc'
4445
shell: bash
4546
run: |
47+
docker load -i /tmp/n8n-images/n8n-pc.tar
4648
docker run --rm --entrypoint node n8nio/n8n:local -e "
4749
if (!process.config.variables.v8_enable_pointer_compression) {
4850
throw new Error('pc variant image is not pointer-compressed');
4951
}
5052
console.log(process.version, 'pointer compression: on');
5153
"
5254
53-
- name: Save image tarball
55+
- name: Report tarball sizes
5456
if: steps.cache-check.outputs.cache-hit != 'true'
5557
shell: bash
56-
run: |
57-
docker save n8nio/n8n:local n8nio/runners:local | zstd -T0 -3 -o /tmp/n8n-image.tar.zst
58-
ls -lh /tmp/n8n-image.tar.zst
58+
run: ls -lh /tmp/n8n-images
5959

6060
- name: Publish image tarball to cache
6161
if: steps.cache-check.outputs.cache-hit != 'true'
6262
uses: actions/cache/save@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3
6363
with:
6464
key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }}
65-
path: /tmp/n8n-image.tar.zst
65+
path: /tmp/n8n-images

.github/actions/load-n8n-docker/action.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# the tarball to cache as a side effect.
1212

1313
name: 'Load n8n Docker images from cache'
14-
description: 'Restores the zstd-compressed n8n + runners image tarball from the variant+SHA-keyed GHA cache and loads both images into the local docker daemon. Falls back to rebuilding on cache miss.'
14+
description: 'Restores the n8n + runners image tarballs from the variant+SHA-keyed GHA cache and loads both images into the local docker daemon. Falls back to rebuilding on cache miss.'
1515

1616
inputs:
1717
build-variant:
@@ -31,12 +31,14 @@ runs:
3131
uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3
3232
with:
3333
key: n8n-docker-image-${{ inputs.build-variant }}-${{ inputs.cache-sha || github.sha }}
34-
path: /tmp/n8n-image.tar.zst
34+
path: /tmp/n8n-images
3535

3636
- name: Load n8n and runners images into docker
3737
if: steps.restore.outputs.cache-hit == 'true'
3838
shell: bash
39-
run: zstd -d -c /tmp/n8n-image.tar.zst | docker load
39+
# Layers inside each archive are zstd-compressed by BuildKit, so docker
40+
# load handles decompression - no separate zstd pass.
41+
run: for f in /tmp/n8n-images/*.tar; do docker load -i "$f"; done
4042

4143
- name: Warn on cache miss
4244
if: steps.restore.outputs.cache-hit != 'true'

scripts/dockerize-n8n.mjs

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
* NODE_VERSION, BUILDER_IMAGE, RUNTIME_IMAGE - read by bake directly
1414
* DOCKER_PLATFORM - cross-platform builds
1515
* DOCKER_BUILD_NO_CACHE, DOCKER_BUILD_BASE_IMAGE, DOCKER_BUILD_DISTROLESS
16+
* DOCKER_BUILD_TARBALL_DIR - write per-target docker-archives here instead of
17+
* loading into the daemon (CI image distribution)
1618
* CONTAINER_ENGINE - force 'docker' or 'podman'
1719
*/
1820

@@ -42,6 +44,9 @@ const imageTag = process.env.IMAGE_TAG || 'local';
4244
// Push directly when the name carries a registry host, which avoids the slow
4345
// --load export/import round-trip.
4446
const shouldPush = imageBaseName.split('/').length > 2;
47+
// CI wants a tarball, not images in the daemon. Writing it straight from BuildKit
48+
// skips the dockerd import and the `docker save` that reads it back out again.
49+
const tarballDir = process.env.DOCKER_BUILD_TARBALL_DIR;
4550

4651
const compiledAppDir = path.join(rootDir, 'compiled');
4752
const compiledTaskRunnerDir = path.join(rootDir, 'dist', 'task-runner-javascript');
@@ -138,13 +143,26 @@ async function buildWithBake(targets) {
138143
const driver = await buildxDriver();
139144
const isContainerDriver = driver !== 'docker';
140145

146+
// `compression=zstd` compresses the layers inside the archive, so no separate
147+
// compression pass is needed - the file is cache-ready as written.
148+
const tarballOutputs = targets.flatMap((t) => [
149+
'--set',
150+
`${t}.output=type=docker,dest=${path.join(tarballDir ?? '', `${t}.tar`)},compression=zstd,compression-level=3`,
151+
]);
152+
141153
const flags = [
142154
...(noCache ? ['--no-cache'] : []),
143155
// The 'docker' driver builds straight into the daemon and rejects both flags.
144-
...(isContainerDriver ? ['--provenance=false', shouldPush ? '--push' : '--load'] : []),
156+
...(isContainerDriver
157+
? [
158+
'--provenance=false',
159+
...(tarballDir ? tarballOutputs : [shouldPush ? '--push' : '--load']),
160+
]
161+
: []),
145162
];
146163

147164
echo(chalk.yellow(`INFO: Building ${targets.join(', ')} with docker buildx bake...`));
165+
if (tarballDir) echo(chalk.yellow(`INFO: Writing image tarballs to ${tarballDir}`));
148166
if (shouldPush) echo(chalk.yellow(`INFO: Registry detected - pushing directly`));
149167

150168
await $({ verbose: true })`docker buildx bake -f ${BAKE_FILE} ${targets} ${flags}`;
@@ -200,6 +218,8 @@ async function main() {
200218
let platform;
201219
let imageNames;
202220

221+
if (tarballDir) await fs.ensureDir(tarballDir);
222+
203223
if (usePodman) {
204224
platform = hostPlatform();
205225
imageNames = await buildWithPodman(platform);
@@ -216,7 +236,10 @@ async function main() {
216236

217237
const images = [];
218238
for (const imageName of imageNames) {
219-
images.push({ imageName, size: await getImageSize(imageName) });
239+
// In tarball mode nothing is loaded into the daemon, so there is no size to
240+
// read. Left out rather than reported as the compressed archive size, which
241+
// would silently change what the docker-image-size metric means.
242+
images.push({ imageName, size: tarballDir ? 'Unknown' : await getImageSize(imageName) });
220243
}
221244

222245
await fs.writeJson(

0 commit comments

Comments
 (0)