Claude: test/gcp/gcp-iceberg-e2e.td

Author: ublubu

bin/ci-builder

@@ -274,6 +274,10 @@ case "$cmd" in
                 --env AZURE_SERVICE_ACCOUNT_PASSWORD
                 --env AZURE_SERVICE_ACCOUNT_TENANT
                 --env GCP_SERVICE_ACCOUNT_JSON
+                # For test/gcp Iceberg E2E
+                --env ICEBERG_GCS_BUCKET
+                --env ICEBERG_GCP_PROJECT
+                --env ICEBERG_GCP_SA_JSON_B64
                 --env GITHUB_TOKEN
                 --env GITHUB_GHCR_TOKEN
                 --env GPG_KEY

ci/nightly/pipeline.template.yml

@@ -961,6 +961,20 @@ steps:
           - ./ci/plugins/mzcompose:
               composition: secrets-local-file
 
+  - group: GCP
+    key: gcp
+    steps:
+      - id: gcp-real
+        label: GCP (Real)
+        depends_on: build-aarch64
+        timeout_in_minutes: 30
+        agents:
+          # because of GCP access
+          queue: linux-aarch64-small
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: gcp
+
   - group: "Platform checks"
     key: platform-checks
     steps:

test/gcp/gcp-iceberg-e2e.td

@@ -0,0 +1,36 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+> CREATE SECRET gcpsa AS decode('${arg.gcp-sa-json-b64}', 'base64');
+
+> CREATE CONNECTION gcpconn TO GCP (SERVICE ACCOUNT KEY = SECRET gcpsa);
+
+> CREATE CONNECTION biglake TO ICEBERG CATALOG (
+    CATALOG TYPE = 'rest',
+    URL = 'https://biglake.googleapis.com/iceberg/v1/restcatalog',
+    GCP CONNECTION = gcpconn,
+    WAREHOUSE = 'gs://${arg.gcs-bucket}'
+  );
+
+> create table foo(a int, b text);
+
+> insert into foo values (1, 'one'), (2, 'two'), (3, 'three');
+
+> CREATE SINK demo
+    FROM foo
+    INTO ICEBERG CATALOG CONNECTION biglake (
+        NAMESPACE '${arg.namespace}',
+        TABLE '${arg.table}'
+    )
+    KEY (a) NOT ENFORCED
+    MODE UPSERT
+    WITH (COMMIT INTERVAL '1s');
+
+# Iceberg sinks commit data asynchronously. We leave the sink running and let
+# mzcompose.py poll BigLake until the commit lands or times out.

test/gcp/mzcompose

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# mzcompose — runs Docker Compose with Materialize customizations.
+
+exec "$(dirname "$0")"/../../bin/pyactivate -m materialize.cli.mzcompose "$@"