This repository was archived by the owner on Mar 11, 2026. It is now read-only.
test: schema migration #49
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schema Migration Test (Indexer) | |
| # Cancel running workflows from the same PR | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "packages/indexer/src/configs/schema.graphql" | |
| - ".github/workflows/migration-test.yml" | |
| pull_request: | |
| paths: | |
| - "packages/indexer/src/configs/schema.graphql" | |
| - ".github/workflows/migration-test.yml" | |
| env: | |
| OLD_COMMIT_REF: 'fcee4b2cdd8224ccab618f8afd1f7fc09e5060e8' | |
| jobs: | |
| schema-migration-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch all history for git operations | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v2 | |
| with: | |
| version: "7" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y docker-compose netcat-openbsd curl jq | |
| docker-compose version | |
| - name: Set up environment variables | |
| run: | | |
| cat > .env.local << EOF | |
| BSC_CHAPEL=${{ secrets.BSC_CHAPEL }} | |
| GNOSIS_CHIADO=${{ secrets.GNOSIS_CHIADO }} | |
| HYPERBRIDGE_GARGANTUA=${{ secrets.HYPERBRIDGE_GARGANTUA }} | |
| PASEO_RPC_URL=${{ secrets.PASEO_RPC_URL }} | |
| BIFROST_PASEO=${{ secrets.BIFROST_PASEO }} | |
| CERE_LOCAL=${{ secrets.CERE_RPC_URL }} | |
| INDEXER_URL=${{ secrets.INDEXER_URL }} | |
| PRIVATE_KEY=${{ secrets.PRIVATE_KEY }} | |
| SECRET_PHRASE=${{ secrets.SECRET_PHRASE }} | |
| PING_MODULE_ADDRESS=0xFE9f23F0F2fE83b8B9576d3FC94e9a7458DdDD35 | |
| TOKEN_GATEWAY_ADDRESS=0xFcDa26cA021d5535C3059547390E6cCd8De7acA6 | |
| DB_USER=postgres | |
| DB_PASS=postgres | |
| DB_DATABASE=postgres | |
| DB_HOST=postgres | |
| DB_PORT=5432 | |
| DB_PATH="./local/db/" | |
| EOF | |
| # Step 1: Checkout old schema commit and start indexer | |
| - name: Checkout old schema commit | |
| run: | | |
| git stash push -m "Stash current changes" | |
| git checkout $OLD_COMMIT_REF | |
| git log --oneline -1 | |
| - name: Install dependencies & Build old schema version | |
| env: | |
| ENV: local | |
| run: | | |
| pnpm install | |
| pnpm build | |
| # Step 2: Publish old schema manifest | |
| - name: Publish old schema manifest | |
| env: | |
| ENV: local | |
| SUBQL_ACCESS_TOKEN: ${{ secrets.SUBQL_ACCESS_TOKEN }} | |
| run: | | |
| cd packages/indexer | |
| ./node_modules/.bin/subql publish | |
| pnpm build | |
| # Step 3: Start indexer with old schema | |
| - name: Start indexer with old schema | |
| run: | | |
| cd packages/indexer | |
| docker-compose -f docker/docker-compose.local.yml down --remove-orphans || true | |
| nohup docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local up --force-recreate --remove-orphans > indexer_old_schema.log 2>&1 & | |
| echo $! > indexer_pid.txt | |
| echo "Indexer started with PID: $(cat indexer_pid.txt)" | |
| - name: Wait for GraphQL server and verify old schema indexing | |
| run: | | |
| echo "Waiting for GraphQL server and verifying old schema is indexing data..." | |
| timeout=300 # 5 minutes timeout | |
| elapsed=0 | |
| interval=10 | |
| while true; do | |
| if [ "$elapsed" -ge "$timeout" ]; then | |
| echo "❌ Timed out waiting for GraphQL server and old schema indexing" | |
| echo "=== Database status ===" | |
| PGPASSWORD=${{ env.DB_PASS }} psql -h localhost -p 5432 -U ${{ env.DB_USER }} -d ${{ env.DB_DATABASE }} -c "\dt" || echo "Could not connect to database" | |
| echo "=== Indexer logs (old schema) ===" | |
| cat packages/indexer/indexer_old_schema.log || echo "No log file found" | |
| exit 1 | |
| fi | |
| echo "Attempting to query StateMachineUpdateEvents (elapsed: ${elapsed}s)..." | |
| # Query StateMachineUpdateEvents | |
| state_machine_result=$(curl -s -X POST http://localhost:3100/graphql \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"query": "query StateMachineUpdateEvents { stateMachineUpdateEvents { totalCount } }"}' 2>/dev/null || echo "failed") | |
| if [[ "$state_machine_result" == *"stateMachineUpdateEvents"* ]]; then | |
| echo "GraphQL server is responding with valid schema structure" | |
| echo "StateMachineUpdateEvents Response: $state_machine_result" | |
| total_count=$(echo "$state_machine_result" | jq -r '.data.stateMachineUpdateEvents.totalCount // 0' 2>/dev/null) | |
| if [[ "$total_count" -gt 1 ]]; then | |
| # Query Metadata | |
| metadata_result=$(curl -s -X POST http://localhost:3100/graphql \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"query": "query Metadata { _metadatas { totalCount nodes { chain deployments } } }"}' 2>/dev/null || echo "failed") | |
| echo "Metadata Response: $metadata_result" | |
| echo "Old schema indexing successful: Found $total_count StateMachineUpdateEvents" | |
| break | |
| fi | |
| else | |
| http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3100/graphql 2>/dev/null || echo "000") | |
| if [[ "$http_code" == "200" ]]; then | |
| echo "GraphQL server responding but schema not ready yet..." | |
| elif [[ "$http_code" == "000" ]]; then | |
| echo "GraphQL server not yet available (connection refused)" | |
| else | |
| echo "GraphQL server responding with HTTP $http_code" | |
| fi | |
| echo "StateMachineUpdateEvents Response: $state_machine_result" | |
| fi | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| echo "Old schema verification completed successfully!" | |
| echo "Final verification details:" | |
| echo "$state_machine_result" | jq '.' 2>/dev/null || echo "$state_machine_result" | |
| # Step 4: Update chain _metadata*.deployment values | |
| - name: Update chain _metadata*.deployment values | |
| shell: bash | |
| env: | |
| DB_USER: postgres | |
| DB_PASS: postgres | |
| DB_DATABASE: postgres | |
| DB_PORT: 5432 | |
| run: | | |
| cd packages/indexer | |
| declare CHAINS_DATA=$(cat chains-block-number.json) | |
| declare -A CHAIN_MAP=( | |
| ["97"]="bsc-chapel" | |
| ["10200"]="gnosis-chiado" | |
| ["Hyperbridge(Gargantua)"]="hyperbridge-gargantua" | |
| ) | |
| run_query() { | |
| local query="$1" | |
| local result | |
| result=$(PGPASSWORD="$DB_PASS" psql -d "${DB_DATABASE}" -h "localhost" -p "${DB_PORT}" -U "${DB_USER}" -t -A -c "$query" 2>&1) | |
| if [ $? -ne 0 ]; then | |
| echo "Database query failed '$query', Error: $result" | |
| return 1 | |
| fi | |
| echo "$result" | |
| } | |
| echo "Fetching metadata tables..." | |
| echo "" | |
| tables_result=$(run_query "SELECT tablename FROM pg_tables WHERE tablename LIKE '_metadata_%' AND schemaname = 'app'") | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to fetch tables" | |
| exit 1 | |
| fi | |
| while IFS= read -r table; do | |
| table=$(echo "$table" | tr -d '[:space:]') | |
| [ -z "$table" ] && continue | |
| echo "Processing: app.$table" | |
| chain_result=$(run_query "SELECT value::text FROM app.$table WHERE key = 'chain'") | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to get chain value" | |
| continue | |
| fi | |
| chain_value=$(echo "$chain_result" | tr -d '[:space:]"') | |
| chain_name=${CHAIN_MAP[$chain_value]} | |
| if [ -z "$chain_name" ]; then | |
| echo "Unknown chain: $chain_value" | |
| continue | |
| fi | |
| echo "Mapped '$chain_value' to '$chain_name'" | |
| block_number=$(echo "$CHAINS_DATA" | jq -r ".\"$chain_name\".blockNumber // empty") | |
| cid=$(echo "$CHAINS_DATA" | jq -r ".\"$chain_name\".cid // empty") | |
| if [ -z "$block_number" ] || [ -z "$cid" ]; then | |
| echo "Missing data for $chain_name" | |
| continue | |
| fi | |
| deployment_json=$(jq -nc --arg b "$block_number" --arg c "$cid" '{($b): ("ipfs://" + $c)} | @json') | |
| update_result=$(run_query "UPDATE app.$table SET value = '$deployment_json'::jsonb WHERE key = 'deployments'") | |
| if [ $? -eq 0 ]; then | |
| echo "Updated $chain_name (block: $block_number)" | |
| else | |
| echo "❌ Failed to update $chain_name" | |
| fi | |
| echo "" | |
| done <<< "$tables_result" | |
| echo "Deployment updates completed!" | |
| - name: Stop running indexer (old schema) | |
| run: | | |
| cd packages/indexer | |
| docker-compose -f docker/docker-compose.local.yml down --remove-orphans || true | |
| rm -rf .*-cid | |
| # Step 5: Checkout new schema and perform migration | |
| - name: Checkout new schema | |
| run: | | |
| git stash push -m "Stash old schema changes" || true | |
| git checkout ${{ github.sha }} | |
| git log --oneline -1 | |
| - name: Install dependencies & Build new schema version | |
| env: | |
| ENV: local | |
| run: | | |
| pnpm install | |
| pnpm build | |
| - name: Publish new schema manifest & Perform schema migration | |
| env: | |
| SUBQL_ACCESS_TOKEN: ${{ secrets.SUBQL_ACCESS_TOKEN }} | |
| ENV: local | |
| run: | | |
| cd packages/indexer | |
| ./node_modules/.bin/subql publish | |
| pnpm build | |
| nohup docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local up --force-recreate --remove-orphans > indexer_migration.log 2>&1 & | |
| echo $! > indexer_migration_pid.txt | |
| echo "Migration indexer started with PID: $(cat indexer_migration_pid.txt)" | |
| echo "Log new docker-compose configuration:" | |
| cat docker/docker-compose.local.yml || echo "No docker-compose file found" | |
| - name: Wait for GraphQL server and verify migration success | |
| run: | | |
| echo "Waiting for GraphQL server and verifying migration success..." | |
| timeout=900 # 15 minutes timeout | |
| elapsed=0 | |
| interval=10 | |
| while true; do | |
| if [ "$elapsed" -ge "$timeout" ]; then | |
| echo "❌ Timed out waiting for GraphQL server and verifying migration success" | |
| echo "=== Database status ===" | |
| PGPASSWORD=${{ env.DB_PASS }} psql -h localhost -p 5432 -U ${{ env.DB_USER }} -d ${{ env.DB_DATABASE }} -c "\dt" || echo "Could not connect to database" | |
| echo "=== Indexer Migration (new schema) ===" | |
| cat packages/indexer/indexer_migration.log || echo "No log file found" | |
| exit 1 | |
| fi | |
| echo "Attempting to query Metadata (elapsed: ${elapsed}s)..." | |
| # Query Metadata | |
| metadata_result=$(curl -s -X POST http://localhost:3100/graphql \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"query": "query Metadata { _metadatas { totalCount nodes { chain deployments } } }"}' 2>/dev/null || echo "failed") | |
| if [[ "$metadata_result" == *"_metadatas"* ]]; then | |
| echo "GraphQL server is responding with valid schema structure" | |
| echo "Metadata Response: $metadata_result" | |
| total_count=$(echo "$metadata_result" | jq -r '.data._metadatas.totalCount // 0' 2>/dev/null) | |
| if [[ "$total_count" -gt 0 ]]; then | |
| deployments=$(echo "$metadata_result" | jq -r '.data._metadatas.nodes[].deployments // empty' 2>/dev/null) | |
| ipfs_count=$(echo "$deployments" | grep -c "ipfs://" 2>/dev/null || echo "0") | |
| if [[ "$ipfs_count" -eq "$total_count" ]]; then | |
| echo "Migrated schema indexing successful with IPFS link: Found $total_count Metadata" | |
| break | |
| elif [[ "$ipfs_count" -gt 0 && "$ipfs_count" -lt "$total_count" ]]; then | |
| echo "Partial IPFS migration: $ipfs_count/$total_count deployments have IPFS links" | |
| echo "Waiting for remaining deployments to migrate..." | |
| else | |
| echo "No IPFS links found yet, migration still in progress..." | |
| fi | |
| fi | |
| else | |
| http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3100/graphql 2>/dev/null || echo "000") | |
| if [[ "$http_code" == "200" ]]; then | |
| echo "GraphQL server responding but schema not ready yet..." | |
| elif [[ "$http_code" == "000" ]]; then | |
| echo "GraphQL server not yet available (connection refused)" | |
| else | |
| echo "GraphQL server responding with HTTP $http_code" | |
| fi | |
| echo "Metadata Response: $metadata_result" | |
| fi | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| # echo "Migration schema verification completed successfully!" | |
| # echo "Final verification details:" | |
| # echo "$metadata_result" | jq '.' 2>/dev/null || echo "$metadata_result" | |
| # Step 6: Run test suites | |
| - name: Run SDK tests | |
| run: pnpm --filter="hyperbridge-sdk" test | |
| - name: Run Intent Filler tests | |
| run: pnpm --filter="filler" test | |
| - name: Show migration summary | |
| if: success() | |
| run: | | |
| echo "=== Schema Migration Test Summary ===" | |
| echo "Old schema indexer started successfully" | |
| echo "Indexed data with old schema" | |
| echo "Schema migration completed successfully" | |
| echo "New schema indexer running and stable" | |
| echo "SDK test suites passed" | |
| echo "" | |
| echo "Migration from commit $OLD_COMMIT_REF to ${{ github.sha }} successful!" | |
| - name: Show indexer logs on failure | |
| if: failure() | |
| run: | | |
| cd packages/indexer | |
| echo "=== Schema Migration Test Failed ===" | |
| echo "--- Old schema logs ---" | |
| cat indexer_old_schema.log 2>/dev/null || echo "No old schema log file found" | |
| echo "\n\n" | |
| echo "--- Migration logs ---" | |
| cat indexer_migration.log 2>/dev/null || echo "No migration log file found" | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| cd packages/indexer | |
| if [ -f indexer_pid.txt ]; then | |
| OLD_PID=$(cat indexer_pid.txt) | |
| kill $OLD_PID 2>/dev/null || echo "Old indexer process already stopped" | |
| fi | |
| if [ -f indexer_migration_pid.txt ]; then | |
| MIGRATION_PID=$(cat indexer_migration_pid.txt) | |
| kill $MIGRATION_PID 2>/dev/null || echo "Migration indexer process already stopped" | |
| fi | |
| docker-compose -f docker/docker-compose.local.yml down --remove-orphans || true | |
| rm -f indexer_pid.txt indexer_migration_pid.txt | |
| - name: Upload logs as artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: indexer-logs | |
| path: | | |
| packages/indexer/indexer_old_schema.log | |
| packages/indexer/indexer_migration.log | |
| retention-days: 7 |