opensearch-migrations/cloudbuild.yaml at main · tonypiazza/opensearch-migrations · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
steps:
  # Build Java tools and kafka libraries
  - name: 'amazoncorretto:21'
    entrypoint: './gradlew'
    args:
      - ':CreateSnapshot:installDist'
      - ':MetadataMigration:installDist'
      - ':DocumentsFromSnapshotMigration:installDist'
      - ':DocumentsFromSnapshotMigration:copyDockerRuntimeJars'
      - ':DocumentsFromSnapshotMigration:syncVersionFile_reindexFromSnapshot'
      - ':libraries:kafkaUtils:installDist'
      - ':libraries:kafkaCommandLineFormatter:assemble'
      - '--no-daemon'

  # Run GCS integration tests (requires Docker for testcontainers)
  - name: 'amazoncorretto:21'
    entrypoint: './gradlew'
    args:
      - ':SnapshotReaderGcs:isolatedTest'
      - '--no-daemon'

  # Build config processor and workflow templates (staged into migrationConsole/nodeStaging)
  - name: 'node:22'
    entrypoint: 'bash'
    args:
      - '-c'
      - |
        set -e
        cd orchestrationSpecs
        npm ci
        echo "=== Building config processor ==="
        cd packages/config-processor
        npm run check-and-bundle -- /workspace/migrationConsole/nodeStaging/configProcessor
        echo "=== Building workflow templates ==="
        cd ../migration-workflow-templates
        mkdir -p /workspace/migrationConsole/nodeStaging/workflowTemplates
        npm run make-templates -- --outputDirectory /workspace/migrationConsole/nodeStaging/workflowTemplates
        echo "=== Workflow templates built ==="
        ls /workspace/migrationConsole/nodeStaging/workflowTemplates/
        echo "=== Generating migration resources (CRDs + admission policies) ==="
        cd /workspace/orchestrationSpecs
        mkdir -p /workspace/migrationConsole/nodeStaging/migrationResources
        npm run stage-migration-resources -- --outputDirectory /workspace/migrationConsole/nodeStaging/migrationResources
        echo "=== Migration resources generated ==="
        ls /workspace/migrationConsole/nodeStaging/migrationResources/

  # Build reindex-from-snapshot image (independent of console chain)
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--no-cache'
      - '-t'
      - 'us-central1-docker.pkg.dev/$PROJECT_ID/migrations/reindex-from-snapshot:latest'
      - 'DocumentsFromSnapshotMigration/docker'

  # Layer 1: Build elasticsearch_client_test_console (base linux + java + python venv)
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--no-cache'
      - '-t'
      - 'migrations/elasticsearch_client_test_console:latest'
      - 'TrafficCapture/dockerSolution/src/main/docker/elasticsearchTestConsole'

  # Layer 2: Build migration_console_base (extends test console with Python libs, Node, scripts)
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--no-cache'
      - '-f'
      - 'migrationConsole/Dockerfile'
      - '-t'
      - 'migrations/migration_console_base:latest'
      - 'migrationConsole'

  # Stage Java tools for the final migration console image
  - name: 'bash'
    args:
      - '-c'
      - |
        set -e
        STAGING_DIR=/workspace/migration-console-staging
        mkdir -p "$$STAGING_DIR/staging"
        cp -r /workspace/CreateSnapshot/build/install/CreateSnapshot "$$STAGING_DIR/staging/CreateSnapshot"
        cp -r /workspace/MetadataMigration/build/install/MetadataMigration "$$STAGING_DIR/staging/MetadataMigration"
        cp -r /workspace/libraries/kafkaUtils/build/install/kafkaUtils "$$STAGING_DIR/staging/kafkaUtils"
        mkdir -p "$$STAGING_DIR/staging/kafkaCommandLineFormatter/lib"
        cp /workspace/libraries/kafkaCommandLineFormatter/build/libs/*.jar "$$STAGING_DIR/staging/kafkaCommandLineFormatter/lib/"
        cp /workspace/VERSION "$$STAGING_DIR/VERSION"
        cp /workspace/TrafficCapture/dockerSolution/src/main/docker/migrationConsoleStaging/Dockerfile "$$STAGING_DIR/Dockerfile"
        echo "=== Staging directory contents ==="
        ls -la "$$STAGING_DIR/"
        ls -la "$$STAGING_DIR/staging/"

  # Layer 3: Build final migration_console image (extends base, adds Java tools)
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--no-cache'
      - '--build-arg'
      - 'BASE_IMAGE=migrations/migration_console_base:latest'
      - '-t'
      - 'us-central1-docker.pkg.dev/$PROJECT_ID/migrations/migration-console:latest'
      - 'migration-console-staging'

images:
  - 'us-central1-docker.pkg.dev/$PROJECT_ID/migrations/migration-console:latest'
  - 'us-central1-docker.pkg.dev/$PROJECT_ID/migrations/reindex-from-snapshot:latest'

options:
  machineType: 'E2_HIGHCPU_8'
  diskSizeGb: 100