Skip to content

Commit 39e1cbc

Browse files
TechatrixTechatrix
committed
CI: update release procedure to separately upload artifacts
1 parent bb6d647 commit 39e1cbc

4 files changed

Lines changed: 174 additions & 237 deletions

File tree

‎.github/workflows/artifacts.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,31 @@ jobs:
2323
- name: Install APT packages
2424
run: |
2525
sudo apt-get update
26-
sudo apt-get install tar 7zip
26+
sudo apt-get install tar 7zip s3cmd
2727
2828
- name: Install minisign
2929
run: |
3030
wget https://github.com/jedisct1/minisign/releases/download/0.11/minisign-0.11-linux.tar.gz
3131
tar -xf minisign-0.11-linux.tar.gz --directory ${HOME}
3232
echo "${HOME}/minisign-linux/x86_64/" >> $GITHUB_PATH
3333
34-
- name: Build and Publish artifacts
34+
- name: Build release artifacts
3535
run: |
3636
echo "${MINISIGN_SECRET_FILE}" > minisign.key
37-
zig build publish -Drelease-minisign -Doptimize=ReleaseSafe --summary all
37+
zig build release -Drelease-minisign -Doptimize=ReleaseSafe --summary all
3838
rm -f minisign.key
3939
env:
40-
ZLS_WORKER_API_TOKEN: ${{ secrets.ZLS_WORKER_API_TOKEN }}
4140
MINISIGN_SECRET_FILE: ${{ secrets.MINISIGN_SECRET_FILE }}
41+
42+
- name: Upload release artifacts
43+
run: s3cmd --host=${R2_ACCOUNT_ID}.r2.cloudflarestorage.com --host-bucket=${R2_ACCOUNT_ID}.r2.cloudflarestorage.com put ./zig-out/artifacts/ --recursive s3://${R2_BUCKET}
44+
env:
45+
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
46+
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
47+
R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
48+
R2_BUCKET: ${{ secrets.R2_BUCKET }}
49+
50+
- name: Publish release
51+
run: zig build publish -Drelease-minisign -Doptimize=ReleaseSafe --summary new
52+
env:
53+
ZLS_WORKER_API_TOKEN: ${{ secrets.ZLS_WORKER_API_TOKEN }}

‎build.zig‎

Lines changed: 68 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -451,56 +451,24 @@ fn release(b: *Build, release_artifacts: []const *Build.Step.Compile) void {
451451
std.debug.assert(release_artifacts.len > 0);
452452
for (release_artifacts) |compile| std.debug.assert(compile.version != null);
453453

454-
const publish_step = b.step("publish", "Publish release artifacts to releases.zigtools.org");
454+
const publish_step = b.step("publish", "Publish release artifacts to releases.zigtools.org (requires curl)");
455455
const release_step = b.step("release", "Build all release artifacts. (requires tar and 7z)");
456456
const release_minisign = b.option(bool, "release-minisign", "Sign release artifacts with Minisign") orelse false;
457457

458-
const FileExtension = enum {
459-
zip,
460-
@"tar.xz",
461-
@"tar.gz",
462-
};
458+
publish_step.dependOn(release_step);
463459

464-
const uri: std.Uri = if (b.graph.env_map.get("ZLS_WORKER_ENDPOINT")) |endpoint| blk: {
465-
var uri = std.Uri.parse(endpoint) catch std.debug.panic("invalid URI: '{s}'", .{endpoint});
466-
if (!uri.path.isEmpty()) std.debug.panic("ZLS_WORKER_ENDPOINT URI must have no path component: '{s}'", .{endpoint});
467-
uri.path = .{ .raw = "/v1/zls/publish" };
468-
break :blk uri;
469-
} else .{
470-
.scheme = "https",
471-
.host = .{ .raw = "releases.zigtools.org" },
472-
.path = .{ .raw = "/v1/zls/publish" },
473-
};
474-
475-
const password = b.graph.env_map.get("ZLS_WORKER_API_TOKEN") orelse "amogus";
476-
477-
const publish_exe = b.addExecutable(.{
478-
.name = "publish",
479-
.target = b.graph.host,
480-
.root_source_file = b.path("src/tools/publish_http_form.zig"),
481-
});
482-
483-
// var publish_artifacts = b.addSystemCommand("curl")
484-
var publish_artifacts = b.addRunArtifact(publish_exe);
485-
publish_step.dependOn(&publish_artifacts.step);
486-
487-
publish_artifacts.addArgs(&.{
488-
b.fmt("{}", .{uri}),
489-
"--user",
490-
b.fmt("admin:{s}", .{password}),
491-
});
492460
// It is possible for the version to be something like `0.12.0-dev` when `git describe` failed.
493461
// Ideally we would want to report a failure about this when running one of the release/publish steps.
494462
// The problem is during the configure phase, it is not possible to know which top level steps gets run.
495463
// So instead we use rely on the release-worker to reject this version string during the make phase.
496464
// One possible alternative would be to use a configuration option (i.e. -Dpublish) to conditionally run an assertion.
497-
publish_artifacts.addArgs(&.{
498-
"--form", b.fmt("zls-version={}", .{release_artifacts[0].version.?}),
499-
"--form", "compatibility=full",
500-
"--form", b.fmt("zig-version={s}", .{builtin.zig_version_string}),
501-
"--form", b.fmt("minimum-build-zig-version={s}", .{minimum_build_zig_version}),
502-
"--form", b.fmt("minimum-runtime-zig-version={s}", .{minimum_runtime_zig_version}),
503-
});
465+
const released_zls_version = release_artifacts[0].version.?;
466+
467+
const FileExtension = enum {
468+
zip,
469+
@"tar.xz",
470+
@"tar.gz",
471+
};
504472

505473
var compressed_artifacts: std.StringArrayHashMapUnmanaged(std.Build.LazyPath) = .empty;
506474

@@ -524,6 +492,7 @@ fn release(b: *Build, release_artifacts: []const *Build.Step.Compile) void {
524492
});
525493

526494
const compress_cmd = std.Build.Step.Run.create(b, "compress artifact");
495+
compress_cmd.clearEnvironment();
527496
compress_cmd.step.max_rss = switch (extension) {
528497
.zip => 160 * 1024 * 1024, // 160 MiB
529498
.@"tar.xz" => 512 * 1024 * 1024, // 512 MiB
@@ -568,24 +537,77 @@ fn release(b: *Build, release_artifacts: []const *Build.Step.Compile) void {
568537

569538
const install_tarball = b.addInstallFileWithDir(file_path, install_dir, file_name);
570539
release_step.dependOn(&install_tarball.step);
571-
publish_artifacts.addArg("--form");
572-
publish_artifacts.addPrefixedFileArg(b.fmt("{s}=@", .{file_name}), file_path);
573540

574541
if (release_minisign) {
575542
const minisign_basename = b.fmt("{s}.minisig", .{file_name});
576543

577544
const minising_cmd = b.addSystemCommand(&.{ "minisign", "-Sm" });
545+
minising_cmd.clearEnvironment();
578546
minising_cmd.addFileArg(file_path);
579547
minising_cmd.addPrefixedFileArg("-s", .{ .cwd_relative = "minisign.key" });
580548
const minising_file_path = minising_cmd.addPrefixedOutputFileArg("-x", minisign_basename);
581549

582550
const install_minising = b.addInstallFileWithDir(minising_file_path, install_dir, minisign_basename);
583551
release_step.dependOn(&install_minising.step);
584-
585-
publish_artifacts.addArg("--form");
586-
publish_artifacts.addPrefixedFileArg(b.fmt("{s}=@", .{minisign_basename}), minising_file_path);
587552
}
588553
}
554+
555+
// Create JSON request data
556+
557+
const prepare_release = b.addRunArtifact(b.addExecutable(.{
558+
.name = "prepare_release",
559+
.root_module = b.createModule(.{
560+
.root_source_file = b.path("src/tools/prepare_release.zig"),
561+
.target = b.graph.host,
562+
.optimize = .Debug,
563+
}),
564+
}));
565+
566+
prepare_release.addArgs(&.{
567+
"--zls-version",
568+
b.fmt("{}", .{released_zls_version}),
569+
"--zig-version",
570+
b.fmt("{s}", .{builtin.zig_version_string}),
571+
"--minimum-build-zig-version",
572+
b.fmt("{s}", .{minimum_build_zig_version}),
573+
"--minimum-runtime-zig-version",
574+
b.fmt("{s}", .{minimum_runtime_zig_version}),
575+
"--compatibility",
576+
"full",
577+
});
578+
579+
for (compressed_artifacts.values()) |path| {
580+
prepare_release.addFileArg(path);
581+
}
582+
583+
// Send Post request to `https://releases.zigtools.org/v1/zls/publish`
584+
585+
const uri: std.Uri = if (b.graph.env_map.get("ZLS_WORKER_ENDPOINT")) |endpoint| blk: {
586+
var uri = std.Uri.parse(endpoint) catch std.debug.panic("invalid URI: '{s}'", .{endpoint});
587+
if (!uri.path.isEmpty()) std.debug.panic("ZLS_WORKER_ENDPOINT URI must have no path component: '{s}'", .{endpoint});
588+
uri.path = .{ .raw = "/v1/zls/publish" };
589+
break :blk uri;
590+
} else .{
591+
.scheme = "https",
592+
.host = .{ .raw = "releases.zigtools.org" },
593+
.path = .{ .raw = "/v1/zls/publish" },
594+
};
595+
596+
const password = b.graph.env_map.get("ZLS_WORKER_API_TOKEN") orelse "amogus";
597+
598+
var publish_artifacts = b.addSystemCommand(&.{"curl"});
599+
publish_artifacts.has_side_effects = true;
600+
publish_step.dependOn(&publish_artifacts.step);
601+
602+
publish_artifacts.addArgs(&.{
603+
b.fmt("{}", .{uri}),
604+
"--fail",
605+
"--show-error",
606+
"--user",
607+
b.fmt("admin:{s}", .{password}),
608+
"--json",
609+
});
610+
publish_artifacts.addPrefixedFileArg("@", prepare_release.captureStdOut());
589611
}
590612

591613
const Build = blk: {

‎src/tools/prepare_release.zig‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
const std = @import("std");
2+
const assert = std.debug.assert;
3+
4+
pub fn main() !void {
5+
var arena_allocator: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
6+
const arena = arena_allocator.allocator();
7+
8+
const args = try std.process.argsAlloc(arena);
9+
var arg_i: usize = 1;
10+
11+
assert(std.mem.eql(u8, "--zls-version", nextArg(args, &arg_i).?));
12+
const zls_version = nextArg(args, &arg_i).?;
13+
assert(std.mem.eql(u8, "--zig-version", nextArg(args, &arg_i).?));
14+
const zig_version = nextArg(args, &arg_i).?;
15+
assert(std.mem.eql(u8, "--minimum-build-zig-version", nextArg(args, &arg_i).?));
16+
const minimum_build_zig_version = nextArg(args, &arg_i).?;
17+
assert(std.mem.eql(u8, "--minimum-runtime-zig-version", nextArg(args, &arg_i).?));
18+
const minimum_runtime_zig_version = nextArg(args, &arg_i).?;
19+
assert(std.mem.eql(u8, "--compatibility", nextArg(args, &arg_i).?));
20+
const compatibility = nextArg(args, &arg_i).?;
21+
22+
var output_buffer: std.ArrayListUnmanaged(u8) = .empty;
23+
24+
var write_stream = std.json.writeStream(output_buffer.writer(arena), .{ .whitespace = .indent_2 });
25+
26+
try write_stream.beginObject();
27+
28+
try write_stream.objectField("zlsVersion");
29+
try write_stream.write(zls_version);
30+
31+
try write_stream.objectField("zigVersion");
32+
try write_stream.write(zig_version);
33+
34+
try write_stream.objectField("minimumBuildZigVersion");
35+
try write_stream.write(minimum_build_zig_version);
36+
37+
try write_stream.objectField("minimumRuntimeZigVersion");
38+
try write_stream.write(minimum_runtime_zig_version);
39+
40+
try write_stream.objectField("compatibility");
41+
try write_stream.write(compatibility);
42+
43+
try write_stream.objectField("artifacts");
44+
45+
try write_stream.beginObject();
46+
47+
while (nextArg(args, &arg_i)) |file_path| {
48+
const file_name = std.fs.path.basename(file_path);
49+
50+
try write_stream.objectField(file_name);
51+
try write_stream.beginObject();
52+
53+
var file = try std.fs.cwd().openFile(file_path, .{});
54+
defer file.close();
55+
56+
const stat = try file.stat();
57+
58+
var sha256sum: std.crypto.hash.sha2.Sha256 = .init(.{});
59+
var read_buffer: [16 * 1024]u8 = undefined;
60+
while (true) {
61+
const amt = try file.read(&read_buffer);
62+
if (amt == 0) break;
63+
sha256sum.update(read_buffer[0..amt]);
64+
}
65+
assert(sha256sum.total_len == stat.size);
66+
67+
const hash = sha256sum.finalResult();
68+
69+
try write_stream.objectField("shasum");
70+
try write_stream.print("\"{}\"", .{std.fmt.fmtSliceHexLower(&hash)});
71+
72+
try write_stream.objectField("size");
73+
try write_stream.write(sha256sum.total_len);
74+
75+
try write_stream.endObject();
76+
try std.fs.cwd().access(file_path, .{});
77+
}
78+
79+
try write_stream.endObject();
80+
81+
try write_stream.endObject();
82+
83+
try std.io.getStdOut().writeAll(output_buffer.items);
84+
}
85+
86+
fn nextArg(args: []const [:0]const u8, i: *usize) ?[:0]const u8 {
87+
if (i.* >= args.len) return null;
88+
defer i.* += 1;
89+
return args[i.*];
90+
}

0 commit comments

Comments
 (0)