ci(.github/workflows): move engine install to databricks/databricks-bot-engine@76a6e590 #2178
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2025 ADBC Drivers Contributors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Tests Workflow | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| protocol: | |
| description: 'Protocol to test (thrift, rest, or both)' | |
| required: false | |
| default: 'both' | |
| type: choice | |
| options: [both, thrift, rest] | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - '.github/workflows/e2e-tests.yml' | |
| - 'ci/scripts/**' | |
| - 'csharp/**' | |
| pull_request: | |
| # No paths filter — workflow must always fire so the gate step can post | |
| # the auto-pass check for non-csharp PRs. Path-based gating is handled | |
| # in the gate step below (same pattern as trigger-integration-tests.yml). | |
| types: [opened, synchronize, reopened, labeled] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-e2e-tests: | |
| name: "E2E Tests (${{ matrix.protocol }})" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| protocol: ${{ inputs.protocol == 'thrift' && fromJson('["thrift"]') || inputs.protocol == 'rest' && fromJson('["rest"]') || fromJson('["thrift","rest"]') }} | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_TEST_CLIENT_ID: ${{ secrets.DATABRICKS_TEST_CLIENT_ID }} | |
| DATABRICKS_TEST_CLIENT_SECRET: ${{ secrets.DATABRICKS_TEST_CLIENT_SECRET }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Check if tests should run | |
| id: gate | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| if [[ "${{ github.event.action }}" == "labeled" ]] && [[ "${{ github.event.label.name }}" == "e2e-test" ]]; then | |
| echo "run=true" >> $GITHUB_OUTPUT && echo "✅ e2e-test label added — running E2E tests" | |
| else | |
| # Auto-pass on PR so it can enter the queue; tests run in merge queue | |
| echo "run=false" >> $GITHUB_OUTPUT && echo "⏭️ PR event — E2E tests run in merge queue" | |
| fi | |
| elif [[ "${{ github.event_name }}" == "merge_group" ]]; then | |
| CHANGED=$(git diff --name-only "${{ github.event.merge_group.base_sha }}" "${{ github.event.merge_group.head_sha }}") | |
| if echo "$CHANGED" | grep -qE "^(csharp/|ci/scripts/|\.github/workflows/e2e-tests\.yml)"; then | |
| echo "run=true" >> $GITHUB_OUTPUT && echo "✅ Relevant files changed" | |
| else | |
| echo "run=false" >> $GITHUB_OUTPUT && echo "⏭️ No relevant files changed, skipping E2E tests" | |
| fi | |
| else | |
| echo "run=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set up .NET | |
| if: steps.gate.outputs.run == 'true' | |
| uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 | |
| with: | |
| dotnet-version: '8.0.x' | |
| - name: Generate OAuth access token | |
| if: steps.gate.outputs.run == 'true' | |
| id: oauth | |
| run: | | |
| OAUTH_RESPONSE=$(curl -s -X POST "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}/oidc/v1/token" \ | |
| -H "Content-Type: application/x-www-form-urlencoded" \ | |
| -d "grant_type=client_credentials" \ | |
| -d "client_id=${{ env.DATABRICKS_TEST_CLIENT_ID }}" \ | |
| -d "client_secret=${{ env.DATABRICKS_TEST_CLIENT_SECRET }}" \ | |
| -d "scope=sql") | |
| OAUTH_TOKEN=$(echo "$OAUTH_RESPONSE" | python3 -c "import sys, json; print(json.load(sys.stdin)['access_token'])") | |
| if [ -z "$OAUTH_TOKEN" ]; then | |
| echo "ERROR: Failed to generate OAuth token" | |
| exit 1 | |
| fi | |
| echo "::add-mask::$OAUTH_TOKEN" | |
| echo "OAUTH_TOKEN=$OAUTH_TOKEN" >> $GITHUB_OUTPUT | |
| # Per-run mutable schema isolates this job's CREATE/DROP/INSERT activity | |
| # from every other job that hits the same shared workspace (parallel | |
| # protocol matrix entries, parallel merge-queue PRs, etc.). Read-only | |
| # fixture tables (all_column_types, cross_ref_customers, …) continue to | |
| # live in main.adbc_testing and are not touched here. See | |
| # docs/e2e-test-isolation-guidance.md. | |
| - name: Compute per-run schema name | |
| if: steps.gate.outputs.run == 'true' | |
| id: schema | |
| run: | | |
| # Schema names allow [A-Za-z0-9_]; run_id and run_attempt are | |
| # numeric, protocol is alphabetic, so no escaping needed. | |
| PER_RUN_SCHEMA="adbc_testing_run_${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.protocol }}" | |
| echo "PER_RUN_SCHEMA=$PER_RUN_SCHEMA" >> $GITHUB_OUTPUT | |
| echo "Per-run schema: main.$PER_RUN_SCHEMA" | |
| # Extract warehouse id from the http path (/sql/1.0/warehouses/<id>) | |
| WAREHOUSE_ID=$(echo "${{ env.DATABRICKS_HTTP_PATH }}" | awk -F/ '{print $NF}') | |
| if [ -z "$WAREHOUSE_ID" ]; then | |
| echo "ERROR: Could not extract warehouse id from DATABRICKS_HTTP_PATH" | |
| exit 1 | |
| fi | |
| echo "WAREHOUSE_ID=$WAREHOUSE_ID" >> $GITHUB_OUTPUT | |
| - name: Provision per-run schema | |
| if: steps.gate.outputs.run == 'true' | |
| env: | |
| OAUTH_TOKEN: ${{ steps.oauth.outputs.OAUTH_TOKEN }} | |
| PER_RUN_SCHEMA: ${{ steps.schema.outputs.PER_RUN_SCHEMA }} | |
| WAREHOUSE_ID: ${{ steps.schema.outputs.WAREHOUSE_ID }} | |
| run: | | |
| set -e | |
| STATEMENT="CREATE SCHEMA IF NOT EXISTS main.${PER_RUN_SCHEMA}" | |
| PAYLOAD=$(jq -n --arg w "$WAREHOUSE_ID" --arg s "$STATEMENT" '{warehouse_id: $w, statement: $s, wait_timeout: "30s"}') | |
| RESPONSE=$(curl -sS -X POST "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}/api/2.0/sql/statements" \ | |
| -H "Authorization: Bearer $OAUTH_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| STATUS=$(echo "$RESPONSE" | python3 -c "import sys, json; print(json.load(sys.stdin).get('status', {}).get('state', ''))") | |
| if [ "$STATUS" != "SUCCEEDED" ]; then | |
| echo "ERROR: CREATE SCHEMA failed (status=$STATUS)" | |
| echo "Response: $RESPONSE" | |
| exit 1 | |
| fi | |
| echo "Created schema: main.${PER_RUN_SCHEMA}" | |
| # Pre-seed adbc_testing_table so tests that read it don't depend on | |
| # CanExecuteUpdate (which runs the same SQL) running first. xUnit | |
| # doesn't guarantee inter-class order, so without this seed the per-run | |
| # schema is empty when StatementTests / ClientTests / etc. read the | |
| # table, even though DriverTests.CanExecuteUpdate is supposed to seed | |
| # it. Tests that do mutate the table still work — CREATE OR REPLACE | |
| # rewrites it idempotently. | |
| - name: Seed test data | |
| if: steps.gate.outputs.run == 'true' | |
| env: | |
| OAUTH_TOKEN: ${{ steps.oauth.outputs.OAUTH_TOKEN }} | |
| PER_RUN_SCHEMA: ${{ steps.schema.outputs.PER_RUN_SCHEMA }} | |
| WAREHOUSE_ID: ${{ steps.schema.outputs.WAREHOUSE_ID }} | |
| DATABRICKS_SERVER_HOSTNAME: ${{ env.DATABRICKS_SERVER_HOSTNAME }} | |
| run: | | |
| python3 - <<'PYEOF' | |
| import json, os, re, sys, urllib.request | |
| with open("csharp/test/Resources/Databricks.sql") as f: | |
| content = f.read() | |
| # Strip line comments | |
| content = re.sub(r"^\s*--.*$", "", content, flags=re.MULTILINE) | |
| # Substitute the templated table name | |
| full_table = f"main.{os.environ['PER_RUN_SCHEMA']}.adbc_testing_table" | |
| content = content.replace("{ADBC_CATALOG}.{ADBC_DATASET}.{ADBC_TABLE}", full_table) | |
| # Split on ; — Databricks.sql has no semicolons inside string literals | |
| statements = [s.strip() for s in content.split(";") if s.strip()] | |
| host = os.environ["DATABRICKS_SERVER_HOSTNAME"] | |
| token = os.environ["OAUTH_TOKEN"] | |
| warehouse = os.environ["WAREHOUSE_ID"] | |
| url = f"https://{host}/api/2.0/sql/statements" | |
| for i, stmt in enumerate(statements, 1): | |
| payload = json.dumps({ | |
| "warehouse_id": warehouse, | |
| "statement": stmt, | |
| "wait_timeout": "30s", | |
| }).encode() | |
| req = urllib.request.Request( | |
| url, data=payload, | |
| headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, | |
| method="POST", | |
| ) | |
| with urllib.request.urlopen(req) as resp: | |
| body = json.loads(resp.read()) | |
| state = body.get("status", {}).get("state", "") | |
| if state != "SUCCEEDED": | |
| print(f"ERROR: statement {i} failed (state={state})") | |
| print(f"Statement: {stmt[:200]}") | |
| print(f"Response: {json.dumps(body, indent=2)}") | |
| sys.exit(1) | |
| # Keep log noise low — first 80 chars per statement. | |
| print(f"OK [{i}/{len(statements)}]: {stmt[:80].replace(chr(10), ' ')}") | |
| print(f"Seeded {len(statements)} statements into main.{os.environ['PER_RUN_SCHEMA']}") | |
| PYEOF | |
| - name: Create Databricks config file | |
| if: steps.gate.outputs.run == 'true' | |
| env: | |
| PER_RUN_SCHEMA: ${{ steps.schema.outputs.PER_RUN_SCHEMA }} | |
| run: | | |
| mkdir -p ~/.databricks | |
| cat > ~/.databricks/connection.json << EOF | |
| { | |
| "uri": "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}${{ env.DATABRICKS_HTTP_PATH }}", | |
| "auth_type": "oauth", | |
| "grant_type": "client_credentials", | |
| "client_id": "${{ env.DATABRICKS_TEST_CLIENT_ID }}", | |
| "client_secret": "${{ env.DATABRICKS_TEST_CLIENT_SECRET }}", | |
| "scope": "sql", | |
| "access_token": "${{ steps.oauth.outputs.OAUTH_TOKEN }}", | |
| "type": "databricks", | |
| "protocol": "${{ matrix.protocol }}", | |
| "catalog": "main", | |
| "db_schema": "${PER_RUN_SCHEMA}", | |
| "query": "SELECT * FROM main.${PER_RUN_SCHEMA}.adbc_testing_table", | |
| "expectedResults": 12, | |
| "isCITesting": true, | |
| "tracePropagationEnabled": "true", | |
| "traceParentHeaderName": "traceparent", | |
| "traceStateEnabled": "false", | |
| "metadata": { | |
| "catalog": "main", | |
| "schema": "${PER_RUN_SCHEMA}", | |
| "table": "adbc_testing_table", | |
| "expectedColumnCount": 19 | |
| } | |
| } | |
| EOF | |
| echo "DATABRICKS_TEST_CONFIG_FILE=$HOME/.databricks/connection.json" >> $GITHUB_ENV | |
| - name: Build | |
| if: steps.gate.outputs.run == 'true' | |
| shell: bash | |
| run: | | |
| ./ci/scripts/csharp_build.sh "${{ github.workspace }}" | |
| - name: Run All Tests | |
| if: steps.gate.outputs.run == 'true' | |
| shell: bash | |
| run: | | |
| export DATABRICKS_TEST_CONFIG_FILE="$HOME/.databricks/connection.json" | |
| ./ci/scripts/csharp_test_databricks_e2e.sh "${{ github.workspace }}" | |
| # Always run, even on test failure or job cancellation, so we don't | |
| # leak schemas in the workspace. | |
| - name: Drop per-run schema | |
| if: always() && steps.gate.outputs.run == 'true' && steps.schema.outputs.PER_RUN_SCHEMA != '' | |
| env: | |
| OAUTH_TOKEN: ${{ steps.oauth.outputs.OAUTH_TOKEN }} | |
| PER_RUN_SCHEMA: ${{ steps.schema.outputs.PER_RUN_SCHEMA }} | |
| WAREHOUSE_ID: ${{ steps.schema.outputs.WAREHOUSE_ID }} | |
| run: | | |
| STATEMENT="DROP SCHEMA IF EXISTS main.${PER_RUN_SCHEMA} CASCADE" | |
| PAYLOAD=$(jq -n --arg w "$WAREHOUSE_ID" --arg s "$STATEMENT" '{warehouse_id: $w, statement: $s, wait_timeout: "30s"}') | |
| RESPONSE=$(curl -sS -X POST "https://${{ env.DATABRICKS_SERVER_HOSTNAME }}/api/2.0/sql/statements" \ | |
| -H "Authorization: Bearer $OAUTH_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD") | |
| STATUS=$(echo "$RESPONSE" | python3 -c "import sys, json; print(json.load(sys.stdin).get('status', {}).get('state', ''))") | |
| if [ "$STATUS" != "SUCCEEDED" ]; then | |
| # Don't fail the job on cleanup error — leaked schemas can be | |
| # garbage-collected later. But surface the failure. | |
| echo "WARNING: DROP SCHEMA failed (status=$STATUS)" | |
| echo "Response: $RESPONSE" | |
| else | |
| echo "Dropped schema: main.${PER_RUN_SCHEMA}" | |
| fi |