Skip to content

Commit 0ef8938

Browse files
jqdsouzajqdsouza
committed
fix
1 parent f597c69 commit 0ef8938

5 files changed

Lines changed: 128 additions & 40 deletions

File tree

‎cli/beacon-hooks/internal/cloudshuttle/shuttle.go‎

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package cloudshuttle
22

33
import (
44
"bytes"
5+
"compress/gzip"
56
"context"
67
"crypto"
78
"crypto/hmac"
@@ -33,12 +34,14 @@ const (
3334
defaultTokenURI = "https://oauth2.googleapis.com/token"
3435
defaultGCSEndpoint = "https://storage.googleapis.com"
3536
defaultS3Region = "us-east-1"
36-
contentTypeJSONL = "text/plain; charset=utf-8"
37+
contentTypeJSONL = "application/x-ndjson"
38+
contentEncoding = "gzip"
3739
uploadGCS = "gcs"
3840
uploadS3 = "s3"
3941
)
4042

4143
var httpClient = http.DefaultClient
44+
var nowUTC = func() time.Time { return time.Now().UTC() }
4245

4346
type Config struct {
4447
Upload string
@@ -167,16 +170,15 @@ func Upload(ctx context.Context, cfg Config, force bool) error {
167170
}
168171

169172
func ObjectName(cfg Config) string {
173+
now := nowUTC()
170174
parts := cleanKeyParts(cfg.Prefix)
171-
parts = append(parts, "provider="+cleanKeyPart(defaultString(cfg.Provider, "unknown")))
172-
if cfg.UserID != "" {
173-
parts = append(parts, "user_id="+cleanKeyPart(cfg.UserID))
174-
}
175-
if cfg.Repository != "" {
176-
parts = append(parts, cleanKeyParts("repo="+cfg.Repository)...)
177-
}
178-
parts = append(parts, "run_id="+cleanKeyPart(defaultString(cfg.RunID, "unknown")))
179-
parts = append(parts, "runtime.jsonl")
175+
parts = append(parts, "runtime", "date="+now.Format("2006-01-02"))
176+
filename := strings.Join([]string{
177+
fmt.Sprintf("%d", now.Unix()),
178+
cleanKeyPart(defaultString(cfg.Provider, "unknown")),
179+
cleanKeyPart(defaultString(cfg.RunID, "unknown")),
180+
}, "-") + ".jsonl.gz"
181+
parts = append(parts, filename)
180182
return path.Join(parts...)
181183
}
182184

@@ -325,16 +327,52 @@ func parseRSAPrivateKey(raw string) (*rsa.PrivateKey, error) {
325327
}
326328

327329
func uploadSnapshot(ctx context.Context, cfg Config, objectName, filePath string) error {
330+
compressedPath, cleanup, err := gzipSnapshot(filePath)
331+
if err != nil {
332+
return err
333+
}
334+
defer cleanup()
328335
switch cfg.Upload {
329336
case uploadS3:
330-
return uploadS3Object(ctx, cfg, objectName, filePath)
337+
return uploadS3Object(ctx, cfg, objectName, compressedPath)
331338
default:
332339
token, err := accessToken(ctx, cfg.CredentialsB64)
333340
if err != nil {
334341
return err
335342
}
336-
return uploadGCSObject(ctx, cfg.GCSEndpoint, cfg.Bucket, objectName, filePath, token)
343+
return uploadGCSObject(ctx, cfg.GCSEndpoint, cfg.Bucket, objectName, compressedPath, token)
344+
}
345+
}
346+
347+
func gzipSnapshot(filePath string) (string, func(), error) {
348+
source, err := os.Open(filePath)
349+
if err != nil {
350+
return "", func() {}, err
351+
}
352+
defer source.Close()
353+
temp, err := os.CreateTemp(filepath.Dir(filePath), "beacon-cloud-*.jsonl.gz")
354+
if err != nil {
355+
return "", func() {}, err
337356
}
357+
tempPath := temp.Name()
358+
cleanup := func() { _ = os.Remove(tempPath) }
359+
gz := gzip.NewWriter(temp)
360+
if _, err := io.Copy(gz, source); err != nil {
361+
_ = gz.Close()
362+
_ = temp.Close()
363+
cleanup()
364+
return "", func() {}, err
365+
}
366+
if err := gz.Close(); err != nil {
367+
_ = temp.Close()
368+
cleanup()
369+
return "", func() {}, err
370+
}
371+
if err := temp.Close(); err != nil {
372+
cleanup()
373+
return "", func() {}, err
374+
}
375+
return tempPath, cleanup, nil
338376
}
339377

340378
func uploadGCSObject(ctx context.Context, endpoint, bucket, objectName, filePath, token string) error {
@@ -353,6 +391,7 @@ func uploadGCSObject(ctx context.Context, endpoint, bucket, objectName, filePath
353391
}
354392
req.Header.Set("Authorization", "Bearer "+token)
355393
req.Header.Set("Content-Type", contentTypeJSONL)
394+
req.Header.Set("Content-Encoding", contentEncoding)
356395
resp, err := httpClient.Do(req)
357396
if err != nil {
358397
return err
@@ -391,6 +430,7 @@ func uploadS3Object(ctx context.Context, cfg Config, objectName, filePath string
391430
req.ContentLength = info.Size()
392431
}
393432
req.Header.Set("Content-Type", contentTypeJSONL)
433+
req.Header.Set("Content-Encoding", contentEncoding)
394434
req.Header.Set("X-Amz-Content-Sha256", payloadHash)
395435
req.Header.Set("X-Amz-Date", time.Now().UTC().Format("20060102T150405Z"))
396436
if cfg.S3SessionToken != "" {
@@ -414,6 +454,7 @@ func signS3Request(req *http.Request, cfg Config, payloadHash string) {
414454
dateStamp := amzDate[:8]
415455
credentialScope := dateStamp + "/" + cfg.S3Region + "/s3/aws4_request"
416456
headers := map[string]string{
457+
"content-encoding": req.Header.Get("Content-Encoding"),
417458
"content-type": req.Header.Get("Content-Type"),
418459
"host": req.URL.Host,
419460
"x-amz-content-sha256": payloadHash,

‎cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package cloudshuttle
22

33
import (
4+
"bytes"
5+
"compress/gzip"
46
"context"
57
"crypto/rand"
68
"crypto/rsa"
@@ -17,30 +19,35 @@ import (
1719
"path/filepath"
1820
"strings"
1921
"testing"
22+
"time"
2023
)
2124

22-
func TestObjectNamePartitionsByProviderUserRepoAndRun(t *testing.T) {
25+
func TestObjectNameUsesDatePartitionedGzipLayout(t *testing.T) {
26+
restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
27+
defer restore()
2328
got := ObjectName(Config{
2429
Prefix: "agent-traces/customer=test",
2530
Provider: "claude_code_web",
2631
UserID: "user-1",
2732
Repository: "asymptote-labs/agent-beacon",
2833
RunID: "cse_123",
2934
})
30-
want := "agent-traces/customer=test/provider=claude_code_web/user_id=user-1/repo=asymptote-labs/agent-beacon/run_id=cse_123/runtime.jsonl"
35+
want := "agent-traces/customer=test/runtime/date=2026-07-12/1783887423-claude_code_web-cse_123.jsonl.gz"
3136
if got != want {
3237
t.Fatalf("ObjectName = %q, want %q", got, want)
3338
}
3439
}
3540

3641
func TestObjectNameUsesCursorCloudProvider(t *testing.T) {
42+
restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
43+
defer restore()
3744
got := ObjectName(Config{
3845
Prefix: "agent-traces",
3946
Provider: "cursor_cloud",
4047
UserID: "user-1",
4148
RunID: "manual-123",
4249
})
43-
want := "agent-traces/provider=cursor_cloud/user_id=user-1/run_id=manual-123/runtime.jsonl"
50+
want := "agent-traces/runtime/date=2026-07-12/1783887423-cursor_cloud-manual-123.jsonl.gz"
4451
if got != want {
4552
t.Fatalf("ObjectName = %q, want %q", got, want)
4653
}
@@ -80,8 +87,11 @@ func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {
8087
}
8188

8289
func TestUploadSendsJSONLToGCS(t *testing.T) {
90+
restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
91+
defer restore()
8392
key := mustRSAKey(t)
84-
var uploadedPath, uploadedAuth, uploadedType, uploadedBody string
93+
var uploadedPath, uploadedAuth, uploadedType, uploadedEncoding string
94+
var uploadedBody []byte
8595
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
8696
switch r.URL.Path {
8797
case "/token":
@@ -96,8 +106,9 @@ func TestUploadSendsJSONLToGCS(t *testing.T) {
96106
uploadedPath = r.URL.EscapedPath()
97107
uploadedAuth = r.Header.Get("Authorization")
98108
uploadedType = r.Header.Get("Content-Type")
109+
uploadedEncoding = r.Header.Get("Content-Encoding")
99110
data, _ := io.ReadAll(r.Body)
100-
uploadedBody = string(data)
111+
uploadedBody = data
101112
w.WriteHeader(http.StatusOK)
102113
}
103114
}))
@@ -136,11 +147,14 @@ func TestUploadSendsJSONLToGCS(t *testing.T) {
136147
if uploadedType != contentTypeJSONL {
137148
t.Fatalf("Content-Type = %q, want %q", uploadedType, contentTypeJSONL)
138149
}
139-
if !strings.Contains(uploadedPath, "/bucket/prefix/provider=claude_code_web/user_id=user/run_id=run/runtime.jsonl") {
150+
if uploadedEncoding != contentEncoding {
151+
t.Fatalf("Content-Encoding = %q, want %q", uploadedEncoding, contentEncoding)
152+
}
153+
if !strings.Contains(uploadedPath, "/bucket/prefix/runtime/date=2026-07-12/1783887423-claude_code_web-run.jsonl.gz") {
140154
t.Fatalf("upload path = %q", uploadedPath)
141155
}
142-
if uploadedBody != "{\"event\":\"ok\"}\n" {
143-
t.Fatalf("uploaded body = %q", uploadedBody)
156+
if got := gunzipString(t, uploadedBody); got != "{\"event\":\"ok\"}\n" {
157+
t.Fatalf("uploaded body = %q", got)
144158
}
145159
}
146160

@@ -183,16 +197,20 @@ func TestConfigFromEnvDefaultsToGCSWhenS3BucketIsAlsoSet(t *testing.T) {
183197
}
184198

185199
func TestUploadSendsJSONLToS3(t *testing.T) {
186-
var uploadedPath, uploadedAuth, uploadedDate, uploadedHash, uploadedToken, uploadedType, uploadedBody string
200+
restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
201+
defer restore()
202+
var uploadedPath, uploadedAuth, uploadedDate, uploadedHash, uploadedToken, uploadedType, uploadedEncoding string
203+
var uploadedBody []byte
187204
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
188205
uploadedPath = r.URL.EscapedPath()
189206
uploadedAuth = r.Header.Get("Authorization")
190207
uploadedDate = r.Header.Get("X-Amz-Date")
191208
uploadedHash = r.Header.Get("X-Amz-Content-Sha256")
192209
uploadedToken = r.Header.Get("X-Amz-Security-Token")
193210
uploadedType = r.Header.Get("Content-Type")
211+
uploadedEncoding = r.Header.Get("Content-Encoding")
194212
data, _ := io.ReadAll(r.Body)
195-
uploadedBody = string(data)
213+
uploadedBody = data
196214
w.WriteHeader(http.StatusOK)
197215
}))
198216
defer server.Close()
@@ -220,21 +238,21 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
220238
if err := Upload(context.Background(), cfg, true); err != nil {
221239
t.Fatalf("Upload returned error: %v", err)
222240
}
223-
if !strings.Contains(uploadedPath, "/bucket/prefix/provider=claude_code_web/user_id=user/run_id=run/runtime.jsonl") {
241+
if !strings.Contains(uploadedPath, "/bucket/prefix/runtime/date=2026-07-12/1783887423-claude_code_web-run.jsonl.gz") {
224242
t.Fatalf("upload path = %q", uploadedPath)
225243
}
226244
if !strings.HasPrefix(uploadedAuth, "AWS4-HMAC-SHA256 Credential=AKIATEST/") {
227245
t.Fatalf("Authorization = %q", uploadedAuth)
228246
}
229-
for _, want := range []string{"us-west-2/s3/aws4_request", "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"} {
247+
for _, want := range []string{"us-west-2/s3/aws4_request", "SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"} {
230248
if !strings.Contains(uploadedAuth, want) {
231249
t.Fatalf("Authorization missing %q: %q", want, uploadedAuth)
232250
}
233251
}
234252
if uploadedDate == "" {
235253
t.Fatal("X-Amz-Date is empty")
236254
}
237-
sum := sha256.Sum256([]byte(body))
255+
sum := sha256.Sum256(uploadedBody)
238256
if uploadedHash != hex.EncodeToString(sum[:]) {
239257
t.Fatalf("X-Amz-Content-Sha256 = %q", uploadedHash)
240258
}
@@ -244,8 +262,11 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
244262
if uploadedType != contentTypeJSONL {
245263
t.Fatalf("Content-Type = %q, want %q", uploadedType, contentTypeJSONL)
246264
}
247-
if uploadedBody != body {
248-
t.Fatalf("uploaded body = %q", uploadedBody)
265+
if uploadedEncoding != contentEncoding {
266+
t.Fatalf("Content-Encoding = %q, want %q", uploadedEncoding, contentEncoding)
267+
}
268+
if got := gunzipString(t, uploadedBody); got != body {
269+
t.Fatalf("uploaded body = %q", got)
249270
}
250271
}
251272

@@ -280,6 +301,27 @@ func mustRSAKey(t *testing.T) *rsa.PrivateKey {
280301
return key
281302
}
282303

304+
func stubNow(t *testing.T, value time.Time) func() {
305+
t.Helper()
306+
old := nowUTC
307+
nowUTC = func() time.Time { return value.UTC() }
308+
return func() { nowUTC = old }
309+
}
310+
311+
func gunzipString(t *testing.T, data []byte) string {
312+
t.Helper()
313+
reader, err := gzip.NewReader(bytes.NewReader(data))
314+
if err != nil {
315+
t.Fatalf("gzip reader: %v", err)
316+
}
317+
defer reader.Close()
318+
out, err := io.ReadAll(reader)
319+
if err != nil {
320+
t.Fatalf("gzip read: %v", err)
321+
}
322+
return string(out)
323+
}
324+
283325
func pemKey(t *testing.T, key *rsa.PrivateKey) string {
284326
t.Helper()
285327
data, err := x509.MarshalPKCS8PrivateKey(key)

‎cli/beacon/README.md‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ configured.
3939

4040
Use `beacon cloud` helpers to configure provider-managed cloud agent sandboxes.
4141
Claude Code on the web and Cursor cloud agents can forward Beacon JSONL to
42-
customer-managed GCS:
42+
customer-managed GCS or S3:
4343

4444
```bash
4545
./beacon cloud gcs setup \
@@ -53,7 +53,12 @@ customer-managed GCS:
5353

5454
Copy the printed `BEACON_CLOUD_GCS_BUCKET`,
5555
`BEACON_CLOUD_GCS_PREFIX`, and `BEACON_CLOUD_GCS_CREDENTIALS_B64`
56-
values into the cloud agent environment. Also set:
56+
values into the cloud agent environment.
57+
58+
For S3, use `beacon cloud s3 setup --apply --print-env` and copy the printed
59+
`BEACON_CLOUD_S3_*` and AWS credential values instead.
60+
61+
For both destinations, also set:
5762

5863
```bash
5964
BEACON_ORIGIN=cloud
@@ -84,14 +89,14 @@ paste it into the provider's cloud setup field:
8489
```
8590

8691
The setup script installs `beacon-hooks` into the cloud sandbox and uploads a
87-
browser-viewable `/tmp/beacon/runtime.jsonl` snapshot to GCS. Claude web writes
92+
gzip-compressed `/tmp/beacon/runtime.jsonl` snapshot to object storage. Claude web writes
8893
`.claude/settings.local.json` inside the sandbox clone. Cursor cloud uses
8994
`.cursor/environment.json` to install Beacon and generate project-level
9095
`.cursor/hooks.json` because Cursor cloud only runs repository project hooks.
9196

9297
```text
93-
<prefix>/provider=claude_code_web/user_id=<id>/run_id=<claude-session-id>/runtime.jsonl
94-
<prefix>/provider=cursor_cloud/user_id=<id>/run_id=<generated-or-explicit-run-id>/runtime.jsonl
98+
<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude-session-id>.jsonl.gz
99+
<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-cursor_cloud-<generated-or-explicit-run-id>.jsonl.gz
95100
```
96101

97102
Cloud network access must allow `oauth2.googleapis.com` and

‎docs/runtimes/claude-code-cloud-agents.mdx‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,13 @@ The setup has three parts:
5353
Each Claude Code cloud agent session writes one readable JSONL object:
5454

5555
```text
56-
gs://<bucket>/<prefix>/provider=claude_code_web/user_id=<user_id>/run_id=<claude_session_id>/runtime.jsonl
56+
gs://<bucket>/<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude_session_id>.jsonl.gz
5757
```
5858

5959
or:
6060

6161
```text
62-
s3://<bucket>/<prefix>/provider=claude_code_web/user_id=<user_id>/run_id=<claude_session_id>/runtime.jsonl
62+
s3://<bucket>/<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude_session_id>.jsonl.gz
6363
```
6464

6565
## Prerequisites
@@ -295,7 +295,7 @@ aws s3 ls --recursive "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/"
295295
You should see a path like:
296296

297297
```text
298-
provider=claude_code_web/user_id=<user_id>/run_id=cse_.../runtime.jsonl
298+
runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-cse_....jsonl.gz
299299
```
300300

301301
<Frame caption="Beacon uploads one readable runtime.jsonl object per Claude Code cloud agent session.">
@@ -305,13 +305,13 @@ provider=claude_code_web/user_id=<user_id>/run_id=cse_.../runtime.jsonl
305305
Inspect the log:
306306

307307
```bash
308-
gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/provider=claude_code_web/user_id=<user_id>/run_id=<run_id>/runtime.jsonl" | head
308+
gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/runtime/date=<date>/<object>.jsonl.gz" | gzip -dc | head
309309
```
310310

311311
For S3:
312312

313313
```bash
314-
aws s3 cp "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/provider=claude_code_web/user_id=<user_id>/run_id=<run_id>/runtime.jsonl" - | head
314+
aws s3 cp "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/runtime/date=<date>/<object>.jsonl.gz" - | gzip -dc | head
315315
```
316316

317317
Expected fields include:

0 commit comments

Comments
 (0)