.gitlab-ci.yml

variables:
  ORBIT_TELEMETRY_ENABLED: "false"
  CANONICAL_PROJECT_ID: "77960826"
  SECURITY_PROJECT_ID: "80999258"
  CLICKHOUSE_VERSION: "26.2"
  DEV_PREFIX: "dev"
  GCLOUD_PROJECT: gitlab-ci-runners-signing
  SERVICE_ACCOUNT: gitlab-orbit-kg-signer@gitlab-ci-runners-signing.iam.gserviceaccount.com
  WI_POOL_PROVIDER: //iam.googleapis.com/projects/8522242139/locations/global/workloadIdentityPools/gitlab-pool-oidc-$CI_PROJECT_ID/providers/gitlab-jwt-$CI_PROJECT_ID
  RUNNER_LARGE: saas-linux-large-amd64
  RUNNER_LARGE_ARM: saas-linux-large-arm64

workflow:
  rules:
    # Security fork
    - if: $CI_PROJECT_ID == $SECURITY_PROJECT_ID && $CI_COMMIT_TAG
      variables:
        DEV_PREFIX: "sec"
    - if: $CI_PROJECT_ID == $SECURITY_PROJECT_ID && $CI_PIPELINE_SOURCE == 'merge_request_event'
      variables:
        DEV_PREFIX: "sec"
    - if: $CI_PROJECT_ID == $SECURITY_PROJECT_ID && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /^chore\(release\)/
      when: never
    - if: $CI_PROJECT_ID == $SECURITY_PROJECT_ID && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      variables:
        DEV_PREFIX: "sec"
    # Non-canonical (other forks)
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID && $CI_COMMIT_TAG
      variables:
        RUNNER_LARGE: saas-linux-medium-amd64
        RUNNER_LARGE_ARM: saas-linux-medium-arm64
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID && $CI_PIPELINE_SOURCE == 'merge_request_event'
      variables:
        RUNNER_LARGE: saas-linux-medium-amd64
        RUNNER_LARGE_ARM: saas-linux-medium-arm64
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /^chore\(release\)/
      when: never
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      variables:
        RUNNER_LARGE: saas-linux-medium-amd64
        RUNNER_LARGE_ARM: saas-linux-medium-arm64
    # Canonical project — original behavior
    - if: $CI_COMMIT_TAG
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /^chore\(release\)/
      when: never
    # Security-template jobs can't reference .skip-on-solo-schedule, so solo
    # schedules mute them through their documented kill-switch variables.
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_ONLY != ""
      variables:
        SAST_DISABLED: "true"
        SECRET_DETECTION_DISABLED: "true"
        DEPENDENCY_SCANNING_DISABLED: "true"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

include:
  - template: Jobs/SAST.gitlab-ci.yml
  - template: Jobs/Secret-Detection.gitlab-ci.yml
  - local: .gitlab/ci/release-local.yml
  - local: .gitlab/ci/stale-mr.yml
  - local: .gitlab/ci/e2e-pin-bump.yml
  - local: .gitlab/ci/release-deploy-mr.yml
  - local: .gitlab/ci/comment-guard.yml
  - project: 'gitlab-com/gl-security/security-operations/infrastructure-security-public/oidc-modules'
    ref: '3.3.0'
    file: 'templates/gcp_auth.yaml'
  - project: 'gitlab-org/orbit/experiments/ai-review-bot'
    file: '/template.yml'
  - component: ${CI_SERVER_FQDN}/gitlab-org/orbit/experiments/ai-review-bot/ai-review-bot@main
    inputs:
      automatic: manual
      labels_enabled: "false"
      model: "anthropic/claude-opus-4-8"
      variant: "high"

stages:
  - lint
  - docs
  - security
  - test
  - benchmark
  - ai
  - build
  - e2e
  - publish
  - release
  - review

# Job Templates

.job-template:
  tags:
    - saas-linux-medium-amd64
  image: registry.gitlab.com/gitlab-org/rust/build-images/orbit-knowledge-graph:latest
  variables:
    MISE_DISABLE_TOOLS: lefthook
  before_script:
    - mise install

.sccache-template:
  variables:
    CARGO_INCREMENTAL: "0"
    SCCACHE_GCS_BUCKET: "gl-knowledgegraph-sccache"
    SCCACHE_GCS_KEY_PATH: "${SCCACHE_GCS_KEY}"
    SCCACHE_GCS_RW_MODE: "READ_WRITE"
    DUCKDB_DOWNLOAD_LIB: "1"
  before_script:
    - mise install
    - |
      if [ -n "${SCCACHE_GCS_KEY}" ]; then
        export RUSTC_WRAPPER="sccache"
        sccache --start-server || true
      fi

.test-template:
  extends:
    - .job-template
    - .sccache-template
  variables:
    RUST_BACKTRACE: "1"
  artifacts:
    when: always
    reports:
      junit:
        - target/nextest/ci/junit.xml

# Identity for jobs acting as the Orbit automation bot (infra-mgmt!3039): the
# glab image plus the bot token from Vault, readable on protected refs only.
# Extend it and add your own rules, needs, and script.
.automation-bot:
  image: registry.gitlab.com/gitlab-org/cli:v1.102.0
  tags:
    - saas-linux-small-amd64
  id_tokens:
    VAULT_ID_TOKEN:
      aud: https://vault.gitlab.net
  secrets:
    AUTOMATION_BOT_TOKEN:
      file: false
      vault: access_tokens/${VAULT_SECRETS_PATH}/automation-bot/token@ci
      token: $VAULT_ID_TOKEN

# "Solo" scheduled pipelines run exactly one job and skip the rest of the CI
# suite. A schedule opts in by setting SCHEDULE_ONLY=<job-name>: the named job
# gates on that value, and every shared job/anchor references this guard so it
# drops out. Adding a new solo schedule needs no edits to existing jobs here.
.skip-on-solo-schedule:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_ONLY != ""
      when: never

.mr-only-rules:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

.mr-and-main-rules:
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

.canonical-only-rules:
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never

.canonical-mr-only-rules:
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

.canonical-main-only-rules:
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# Lint Stage

agent-file-sync-check:
  stage: lint
  image: alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - diff AGENTS.md CLAUDE.md || (echo "AGENTS.md and CLAUDE.md are out of sync. Please ensure they have identical content." && exit 1)

mr-title-check:
  stage: lint
  image: node:26-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  before_script:
    - npm ci --ignore-scripts
  script:
    - ./scripts/check-mr-title.sh

ontology-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - test "$(wc -c < config/schemas/ontology.schema.json)" -le 65536 || (echo "config/schemas/ontology.schema.json must stay at or below 64 KB" && exit 1)
    - find config/ontology -name '*.yaml' ! -name 'reference.yaml' | xargs uvx check-jsonschema --verbose --schemafile config/schemas/ontology.schema.json

named-query-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - find config/named_queries -name '*.yaml' | xargs uvx check-jsonschema --verbose --schemafile config/schemas/named_query.schema.json

migration-ledger-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - uvx check-jsonschema --verbose --schemafile config/schemas/schema-migrations.schema.json config/schema-migrations.yaml

indexer-scenario-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - find crates/integration-tests/tests/indexer/scenarios -name '*.yaml' | xargs uvx check-jsonschema --verbose --schemafile config/schemas/indexer_scenario.schema.json


setup-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - find config/setup/agents -name '*.yaml' | xargs uvx check-jsonschema --verbose --schemafile config/schemas/setup_agent.schema.json
    - uvx check-jsonschema --verbose --schemafile config/schemas/setup.schema.json config/setup/setup.yaml

generate-config-schema:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - mise run schema:generate
    - git diff --exit-code config/schemas/config.schema.json || (echo "config/schemas/config.schema.json is out of date. Run 'mise schema:generate' and commit." && exit 1)
  artifacts:
    paths:
      - config/schemas/config.schema.json
    expire_in: 24 hours

rust-toolchain-sync-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - mise run toolchain:check

metrics-catalog-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - mise run metrics:catalog:check

dashboards-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - mise run dashboards:check

query-language-docs-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - mise run docs:query-language:check

ddl-freshness-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-and-main-rules
  needs: []
  script:
    - bash ./scripts/check-ddl-freshness.sh

migration-ledger-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-only-rules
  needs: []
  script:
    - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=1
    - bash ./scripts/check-migration-ledger.sh "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"

pinned-version-check:
  stage: lint
  image: alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  script:
    - apk add --no-cache git bash
    - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=1
    - bash ./scripts/check-pinned-version.sh "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"

skill-version-bump-check:
  stage: lint
  image: python:3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  script:
    - apk add --no-cache git
    - |
      if ! git fetch origin "$CI_MERGE_REQUEST_DIFF_BASE_SHA" --depth=1; then
        git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=50
        export BASE_REF_OVERRIDE="origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
      fi
      python3 scripts/check-skill-version-bump.py --ci --no-worktree ${BASE_REF_OVERRIDE:+--base-ref "$BASE_REF_OVERRIDE"}

prompt-version-bump-check:
  stage: lint
  image: alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  script:
    - apk add --no-cache git
    - |
      if ! git fetch origin "$CI_MERGE_REQUEST_DIFF_BASE_SHA" --depth=1; then
        git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=50
        export CI_MERGE_REQUEST_DIFF_BASE_SHA="origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
      fi
      sh scripts/check-prompt-version-bump.sh

iglu-schema-check:
  stage: lint
  image: alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  # Full check (local file vs upstream content from the public Iglu
  # Pages CDN). Drift between committed schema JSON and upstream fails
  # the job.
  script:
    - apk add --no-cache bash curl python3
    - bash ./scripts/iglu/check.sh

system-note-actions-check:
  stage: lint
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  # Verifies config/vendored/system_note_metadata.actions matches Rails
  # SystemNoteMetadata::ICON_TYPES at the pinned commit SHA.
  script:
    - apk add --no-cache bash curl python3
    - bash ./scripts/check-system-note-actions.sh

duckdb-fts-sources-sync-check:
  stage: lint
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  needs: []
  script:
    - apk add --no-cache bash git gzip tar
    - bash ./scripts/duckdb/check-duckdb-fts-sources-sync.sh

unused-deps-check:
  stage: lint
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo shear

fmt-check:
  stage: lint
  extends:
    - .job-template
    - .mr-only-rules
  script:
    - cargo fmt --all -- --check

newline-check:
  stage: lint
  extends:
    - .job-template
    - .mr-only-rules
  script:
    - mise run lint:newlines

lint-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-only-rules
  script:
    - cargo clippy --workspace --exclude duckdb-client --all-targets --all-features -- -D warnings
    - cargo clippy -p duckdb-client --all-targets -- -D warnings
    - sccache --show-stats || true

# Security Stage

cargo-audit:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo audit

cargo-deny:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo deny check

cargo-geiger:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name' | while IFS= read -r pkg; do
        if [ "$pkg" = "duckdb-client" ]; then cargo geiger -p "$pkg" || true;
        else cargo geiger -p "$pkg" --all-features || true; fi;
      done
  allow_failure: true

generate-sbom:
  stage: security
  extends:
    - .job-template
  needs: []
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - cargo cyclonedx --format json > sbom.cdx.json
  artifacts:
    paths:
      - sbom.cdx.json
    reports:
      cyclonedx: sbom.cdx.json

semgrep-sast:
  stage: security
  needs: []
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      exists:
        - "**/*.rs"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      exists:
        - "**/*.rs"

gitlab-advanced-sast:
  stage: security
  needs: []

# Override the default secret_detection rules so the job runs on MR pipelines.
# The Jobs/Secret-Detection.gitlab-ci.yml template defaults to branch pipelines
# only, which conflicts with this project's workflow (MR + main branch only).
secret_detection:
  stage: security
  needs: []
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# Docs stage

check_docs_markdown:
  stage: docs
  extends:
    - .mr-only-rules
  image: registry.gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/lint-markdown:alpine-3.23-vale-3.14.1-markdownlint2-0.22.1-lychee-0.24.2
  script:
    # Lint prose
    - vale --minAlertLevel error docs/source
    # Lint Markdown. docs/source is user-facing; root and docs/dev are contributor
    # docs that previously reached CI unlinted (design-documents deferred: #981).
    - markdownlint-cli2 '*.md' 'docs/dev/**/*.md' 'docs/source/**/*.md'
    # Check links
    - lychee --offline --include-fragments docs/source/**/*.md

check_docs_i18n_markdown:
  stage: docs
  image: registry.gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/lint-markdown:alpine-3.23-vale-3.14.1-markdownlint2-0.22.1-lychee-0.24.2
  parallel:
    matrix:
      - LOCALE_CODE: ["fr-fr", "ja-jp", "ko-kr"]
  rules:
    - if: $CI_MERGE_REQUEST_IID
      changes:
        - docs-locale/${LOCALE_CODE}/**/*
  script:
    # Lint prose
    - vale --config="docs-locale/${LOCALE_CODE}/.vale.ini" --minAlertLevel error docs-locale/${LOCALE_CODE}/source/
    # Lint Markdown
    - markdownlint-cli2 "docs-locale/${LOCALE_CODE}/source/**/*.md"
    # Check links
    - lychee --offline --include-fragments docs-locale/${LOCALE_CODE}/source/**/*.md

docs_hugo_build:
  image: registry.gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/docs-gitlab-com-builder:hugo-0.154.5
  stage: docs
  extends:
    - .mr-only-rules
  variables:
    DOCS_BRANCH: "main"
  before_script:
    # Check if this a release branch, which would be the case for a backport.
    # If this is a backport MR, we need to checkout the appropriate version
    # of the Docs website.
    - |
      if [[ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ [0-9]+-[0-9]+-stable ]]; then
        BRANCH_NAME=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
        echo "Detected merge request to stable branch: $BRANCH_NAME"
      # Check if we're directly on a stable branch (direct push/commit)
      elif [[ $CI_COMMIT_BRANCH =~ [0-9]+-[0-9]+-stable ]]; then
        BRANCH_NAME=$CI_COMMIT_BRANCH
        echo "Detected direct commit to stable branch: $BRANCH_NAME"
      fi

      # Extract version info if we found a stable branch
      if [[ -n $BRANCH_NAME ]]; then
        MAJOR=$(echo $BRANCH_NAME | cut -d '-' -f 1)
        MINOR=$(echo $BRANCH_NAME | cut -d '-' -f 2)
        # Convert GitLab style (17-9-stable-ee) to Docs style (17.9)
        DOCS_BRANCH_CANDIDATE="$MAJOR.$MINOR"

        # Check if the branch exists in the Docs website repo, fallback to main if not
        if git ls-remote --heads --exit-code https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com.git "refs/heads/$DOCS_BRANCH_CANDIDATE" >/dev/null 2>&1; then
          DOCS_BRANCH="$DOCS_BRANCH_CANDIDATE"
          echo "Using docs-gitlab-com branch $DOCS_BRANCH for release branch"
        else
          DOCS_BRANCH="main"
          echo "Branch $DOCS_BRANCH_CANDIDATE does not exist, falling back to main"
        fi
      fi
    # Clone the Docs website project
    - git clone --depth 1 --filter=tree:0 --branch $DOCS_BRANCH https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com.git
    - cd docs-gitlab-com
    - make add-latest-icons
  script:
    # Test that Hugo will build
    - hugo --gc --printPathWarnings --panicOnWarning  --environment test
    # Test for invalid index pages
    # See https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/-/blob/main/scripts/check-index-filenames.sh
    - make check-index-pages SEARCH_DIR="../docs/source"
    #- make check-index-pages SEARCH_DIR="../doc-locale/source" # Enable when translation is added

# Test Stage

unit-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  tags:
    - ${RUNNER_LARGE}
  needs: []
  script:
    - ./scripts/run-unit-tests.sh --profile ci
    - sccache --show-stats || true

compiler-integration-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo nextest run --profile ci -p integration-tests --test local --no-fail-fast
    - sccache --show-stats || true

billing-boundary-check:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo nextest run --profile ci -p integration-tests --test billing_boundary
    - sccache --show-stats || true

integration-tests-codegraph:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo nextest run --profile ci -p integration-tests-codegraph --no-fail-fast
    - sccache --show-stats || true

cli-integration-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo build -p orbit-cli
    - cargo nextest run --profile ci --test cli --no-fail-fast
    - sccache --show-stats || true

integration-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  tags:
    - ${RUNNER_LARGE}
  services:
    - name: docker:dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    TESTCONTAINERS_HOST_OVERRIDE: docker
  needs: []
  before_script:
    - !reference [.sccache-template, before_script]
    - |
      docker pull clickhouse/clickhouse-server:${CLICKHOUSE_VERSION} &
      docker pull nats:2.11-alpine &
      wait
  script:
    # corpus-smoke runs as its own parallel job below.
    - cargo nextest run --profile ci --all-features --test containers -E 'not test(corpus_smoke)'
    - sccache --show-stats || true

corpus-smoke-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  tags:
    - ${RUNNER_LARGE}
  services:
    - name: docker:dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    TESTCONTAINERS_HOST_OVERRIDE: docker
  needs: []
  before_script:
    - !reference [.sccache-template, before_script]
    - docker pull clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}
  script:
    # Smoke test: every query in fixtures/queries/corpus/ compiles + runs
    # against a real ClickHouse schema (no correctness assertions).
    - cargo nextest run --profile ci --all-features --test containers -E 'test(corpus_smoke)'
    - sccache --show-stats || true

# Benchmark Stage
#
# Runs code indexing benchmarks when indexing crates change.
# Repos are auto-downloaded by gitlab-xtasks from the benchmark config.

.code-indexing-benchmark-rules:
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      changes:
        - crates/orbit-cli/**/*
        - crates/code-graph/**/*
        - crates/ontology/**/*
        - crates/duckdb-client/**/*
        - crates/utils/**/*
        - code-indexing-benchmark.yaml
        - .gitlab-ci.yml

code-indexing-benchmark-build:
  stage: benchmark
  needs: []
  extends:
    - .job-template
    - .sccache-template
    - .code-indexing-benchmark-rules
  interruptible: true
  tags:
    - ${RUNNER_LARGE}
  script:
    - cargo build --bin orbit --release --features duckdb-client/bundled
  artifacts:
    paths:
      - target/release/orbit
    expire_in: 1 day

.code-indexing-benchmark-template:
  stage: benchmark
  needs:
    - code-indexing-benchmark-build
  extends:
    - .code-indexing-benchmark-rules
  interruptible: true
  cache:
    key: "benchmark-repos-${GROUP}"
    paths:
      - temp/
    policy: pull-push
    when: always
  variables:
    SCENARIO: index
  script:
    - |
      OUTPUT_FILE="gkg-benchmark-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}-${ARCH}-${SCENARIO}-${GROUP}.json"

      for attempt in 1 2 3; do
        if gitlab-xtasks benchmark download-repos \
          --config code-indexing-benchmark.yaml \
          --output-dir temp \
          --group "$GROUP"; then
          break
        fi

        if [ "$attempt" = "3" ]; then
          exit 1
        fi

        delay="$((attempt * 30))"
        echo "Repository download failed; retrying in ${delay}s."
        sleep "$delay"
      done

      gitlab-xtasks benchmark hyperfine \
        --config code-indexing-benchmark.yaml \
        --architecture "$ARCH" \
        --scenario "$SCENARIO" \
        --group "$GROUP" \
        --output-file "$OUTPUT_FILE"
  artifacts:
    paths:
      - gkg-benchmark-*.json
    expire_in: 3 weeks

code-indexing-benchmark-v2-linux:
  extends:
    - .job-template
    - .code-indexing-benchmark-template
  tags:
    - ${RUNNER_LARGE}
  variables:
    ARCH: "linux-x86_64"
    SCENARIO: v2
  parallel:
    matrix:
      - GROUP: [python, java, kotlin, csharp, ruby, go, rust, c, cpp, php]

.code-indexing-benchmark-analyze-template:
  stage: benchmark
  extends:
    - .job-template
    - .code-indexing-benchmark-rules
  interruptible: true
  script:
    - |
      gitlab-xtasks benchmark analyze \
        --pattern "gkg-benchmark-*-${SCENARIO}-*.json" \
        --output-format markdown \
        --pipeline-url "$CI_PIPELINE_URL" \
        --job-url "$CI_JOB_URL" \
        --branch "$CI_COMMIT_REF_NAME" \
        --output-file "gkg-benchmark-${SCENARIO}-results.md"

      gitlab-xtasks benchmark analyze \
        --pattern "gkg-benchmark-*-${SCENARIO}-*.json" \
        --output-format json \
        --pipeline-url "$CI_PIPELINE_URL" \
        --job-url "$CI_JOB_URL" \
        --branch "$CI_COMMIT_REF_NAME" \
        --output-file "gkg-benchmark-${SCENARIO}-results.json"

      if [[ -n "$CI_MERGE_REQUEST_IID" && -n "$BENCHMARK_GITLAB_TOKEN" ]]; then
        gitlab-xtasks comment post \
          --type add-or-edit \
          --unique-id "code-indexing-benchmark-${SCENARIO}-comment" \
          --token "$BENCHMARK_GITLAB_TOKEN" \
          --file-path "gkg-benchmark-${SCENARIO}-results.md"
      else
        echo "Skipping benchmark comment: BENCHMARK_GITLAB_TOKEN unavailable (expected for fork MRs)."
      fi
  artifacts:
    paths:
      - gkg-benchmark-*-results.md
      - gkg-benchmark-*-results.json
      - gkg-benchmark-*.json
    expire_in: 1 week

code-indexing-benchmark-v2-analyze:
  extends: .code-indexing-benchmark-analyze-template
  needs:
    - code-indexing-benchmark-v2-linux
  variables:
    SCENARIO: v2

# Build Stage

.docker-build:
  stage: build
  image: docker:29
  needs: []
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - git fetch --tags
    - BASE_VERSION=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' --sort=-v:refname | sed -n '1s/^v//p')
    - export GKG_VERSION="${BASE_VERSION:-0.0.0}-${DEV_PREFIX}+${CI_COMMIT_SHORT_SHA}"
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${DEV_PREFIX}-${CI_COMMIT_SHORT_SHA}-${ARCH}"

docker-build-amd64:
  extends: .docker-build
  tags:
    - ${RUNNER_LARGE}
  variables:
    ARCH: amd64

docker-build-arm64:
  extends: .docker-build
  tags:
    - ${RUNNER_LARGE_ARM}
  variables:
    ARCH: arm64

docker-manifest:
  stage: build
  image: docker:29
  tags:
    - saas-linux-medium-amd64
  needs:
    - docker-build-amd64
    - docker-build-arm64
  variables:
    IMAGE_NAME: gkg
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    - docker buildx imagetools create
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${DEV_PREFIX}"
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${DEV_PREFIX}-${CI_COMMIT_SHORT_SHA}"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${DEV_PREFIX}-${CI_COMMIT_SHORT_SHA}-amd64"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${DEV_PREFIX}-${CI_COMMIT_SHORT_SHA}-arm64"

docker-build-mr:
  stage: build
  image: docker:29
  tags:
    - ${RUNNER_LARGE}
  needs: []
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  script:
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop-mr-${CI_MERGE_REQUEST_IID}-${CI_COMMIT_SHORT_SHA}-amd64"

# Tag = Dockerfile hash; the e2e job resolves the same tag.
e2e-robot-image:
  stage: build
  image: docker:29
  tags:
    - saas-linux-medium-amd64
  needs: []
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - e2e/Dockerfile.robot
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    - TAG=$(sha256sum e2e/Dockerfile.robot | cut -c1-12)
    - IMAGE="${CI_REGISTRY_IMAGE}/e2e-robot"
    - |
      if docker manifest inspect "${IMAGE}:${TAG}" >/dev/null 2>&1; then
        echo "${IMAGE}:${TAG} already exists"
      else
        docker build -t "${IMAGE}:${TAG}" -f e2e/Dockerfile.robot e2e
        docker push "${IMAGE}:${TAG}"
      fi

check-go-proto:
  needs: []
  image: golang:1.24
  tags:
    - saas-linux-medium-amd64
  stage: lint
  before_script:
    - apt-get update -qq && apt-get install -y -qq unzip
  script:
    - scripts/proto-go.sh check
    - cd clients/orbitpb && go build ./...
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/
    - if: $CI_COMMIT_TAG
      when: never
    - changes:
        - crates/orbit-server/proto/**/*
        - clients/orbitpb/**/*

build-proto-gem:
  needs: []
  image: ruby:3.3
  tags:
    - saas-linux-medium-amd64
  stage: build
  before_script: []
  script:
    - BUNDLE_GEMFILE=clients/protogem/Gemfile bundle install --quiet
    - |
      if [ -n "$CI_COMMIT_TAG" ]; then
        ruby clients/protogem/build-proto-gem --version "${CI_COMMIT_TAG#v}"
      else
        ruby clients/protogem/build-proto-gem
      fi
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/
    - if: $CI_COMMIT_TAG
      when: never
    - changes:
        - crates/orbit-server/proto/**/*
        - clients/protogem/**/*
  artifacts:
    paths:
      - _build/gitlab-orbit-proto-*.gem
    expire_in: 30 days

publish-proto-gem:
  needs:
    - build-proto-gem
  image: ruby:3.3
  tags:
    - saas-linux-medium-amd64
  stage: publish
  before_script: []
  script:
    - gem push "_build/gitlab-orbit-proto-${CI_COMMIT_TAG#v}.gem"
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
  environment:
    name: rubygems
    url: https://rubygems.org/gems/gitlab-orbit-proto

# Release Stage

semantic-release:
  stage: release
  image: node:26
  tags:
    - saas-linux-medium-amd64
  id_tokens:
    VAULT_ID_TOKEN:
      aud: https://vault.gitlab.net
  secrets:
    SEMANTIC_RELEASE_GITLAB_TOKEN:
      file: false
      vault: access_tokens/${VAULT_SECRETS_PATH}/semantic-release/token@ci
      token: $VAULT_ID_TOKEN
  variables:
    GIT_STRATEGY: clone
  needs:
    - job: build-proto-gem
      optional: true
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_ONLY == "auto-release"
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
  before_script:
    - test -n "${SEMANTIC_RELEASE_GITLAB_TOKEN}" || (echo "Missing SEMANTIC_RELEASE_GITLAB_TOKEN from Vault" && exit 1)
    - export GITLAB_TOKEN="${SEMANTIC_RELEASE_GITLAB_TOKEN}"
    - git remote set-url origin "https://semantic-release:${GITLAB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
    - npm ci
  script:
    - npx semantic-release

.docker-release:
  stage: build
  image: docker:29
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/
  script:
    - export GKG_VERSION="${CI_COMMIT_TAG#v}"
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GKG_VERSION}-${ARCH}"

release-build-amd64:
  extends: .docker-release
  tags:
    - ${RUNNER_LARGE}
  variables:
    ARCH: amd64

release-build-arm64:
  extends: .docker-release
  tags:
    - ${RUNNER_LARGE_ARM}
  variables:
    ARCH: arm64

release-manifest:
  stage: build
  image: docker:29
  tags:
    - saas-linux-medium-amd64
  needs:
    - release-build-amd64
    - release-build-arm64
  variables:
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    - VERSION="${CI_COMMIT_TAG#v}"
    - docker buildx imagetools create
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}"
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:latest"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}-amd64"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}-arm64"

# Review stage

.review-docs:
  image: ruby:3.3-alpine3.23
  extends:
    - .canonical-mr-only-rules
  stage: review
  cache: {}
  dependencies: []
  before_script:
  - gem install gitlab --no-doc
  # We need to download the script rather than clone the repo since the
  # review-docs-cleanup job will not be able to run when the branch gets
  # deleted (when merging the MR).
  - apk add --update openssl
  - wget https://gitlab.com/gitlab-org/gitlab/-/raw/master/scripts/trigger-build.rb
  - chmod 755 trigger-build.rb
  variables:
    GIT_STRATEGY: none
    DOCS_GITLAB_REPO_SUFFIX: orbit
    # By default, deploy the Review App using the `main` branch of the `gitlab-org/gitlab-docs` project
    DOCS_BRANCH: main
  when: manual

# Trigger a docs build in docs-gitlab-com project
# Useful to preview the docs changes live
# https://docs.gitlab.com/development/documentation/review_apps/
review-docs-deploy:
  extends:
  - .review-docs
  environment:
    name: review/mr-${CI_MERGE_REQUEST_IID}
    url: https://docs.gitlab.com/upstream-review-mr-${DOCS_GITLAB_REPO_SUFFIX}-${CI_MERGE_REQUEST_IID}/${DOCS_GITLAB_REPO_SUFFIX}
    auto_stop_in: 2 weeks
    on_stop: review-docs-cleanup
  script:
  - ./trigger-build.rb docs deploy

# Cleanup remote environment of docs-gitlab-com
review-docs-cleanup:
  extends:
  - .review-docs
  environment:
    name: review/mr-${CI_MERGE_REQUEST_IID}
    action: stop
  script:
  - ./trigger-build.rb docs cleanup

# AI Review Stage
#
# Manually-triggered AI agents for MR review. Uses the shared
# ai-review-bot framework from gitlab-org/orbit/experiments/ai-review-bot.
#
# Required CI/CD variables (Settings > CI/CD > Variables):
#   GITLAB_REVIEW_TOKEN  — Masked + Hidden
#   AI_PROXY_INIT_TOKEN  — Masked + Hidden
#   GOOGLE_CLOUD_PROJECT — GCP project ID for Vertex AI
#   VERTEX_SA_KEY_B64    — Masked + Hidden

ai:performance:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: performance
    GOOGLE_CLOUD_LOCATION: "global"
    AI_REFS: "https://gitlab.com/gitlab-org/gitlab.git,https://github.com/ClickHouse/clickhouse-docs.git,https://gitlab.com/gitlab-org/analytics-section/siphon.git"

ai:security:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: security
    GOOGLE_CLOUD_LOCATION: "global"
    AI_REFS: "https://gitlab.com/gitlab-org/gitlab.git,https://github.com/ClickHouse/clickhouse-docs.git,https://gitlab.com/gitlab-org/analytics-section/siphon.git"

ai:docs:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: docs
    GOOGLE_CLOUD_LOCATION: "global"

# E2E Stage

e2e:
  stage: e2e
  extends:
    - .job-template
    - .sccache-template
  tags:
    - ${RUNNER_LARGE}
  timeout: 60 minutes
  interruptible: true
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    KCTX: gitlab-org/orbit/orbit-e2e-harness:e2e-harness-agent
    KUBECTL_VERSION: "1.35.2"
    HELM_VERSION: "3.20.0"
    HELMFILE_VERSION: "1.4.2"
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
  needs:
    - job: docker-manifest
      optional: true
    - job: e2e-robot-image
      optional: true
  rules:
    - !reference [.skip-on-solo-schedule, rules]
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    # Scheduled pin-bump MRs exist to be validated, so e2e runs and blocks.
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == 'automation/e2e-pin-bump'
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  before_script:
    - !reference [.sccache-template, before_script]
    - bash e2e/scripts/install-ci-tools.sh
    - kubectl config use-context "$KCTX"
  script:
    - export E2E_SHA="${CI_COMMIT_SHORT_SHA}${E2E_SUFFIX:-}"
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    # Sourced: exports E2E_GKG_IMAGE/E2E_GKG_TAG and backgrounds the MR build
    # so `wait "$E2E_BUILD_PID"` below stays in this shell.
    - source e2e/scripts/build-orbit-image.sh
    - |
      E2E_ROBOT_TAG=$(sha256sum e2e/Dockerfile.robot | cut -c1-12)
      if docker manifest inspect "${CI_REGISTRY_IMAGE}/e2e-robot:${E2E_ROBOT_TAG}" >/dev/null 2>&1; then
        export E2E_ROBOT_IMAGE="${CI_REGISTRY_IMAGE}/e2e-robot:${E2E_ROBOT_TAG}"
      else
        echo "e2e-robot:${E2E_ROBOT_TAG} not in registry; robot job installs at runtime"
      fi
    - bash e2e/scripts/setup.sh || { tail -80 e2e-image-build.log 2>/dev/null; exit 1; }
    - |
      if [ -n "${E2E_BUILD_PID:-}" ]; then
        if ! wait "$E2E_BUILD_PID"; then
          echo "gkg-e2e image build failed:"
          tail -80 e2e-image-build.log
          exit 1
        fi
        tail -3 e2e-image-build.log
      fi
    - bash e2e/scripts/test.sh
  after_script:
    - bash e2e/scripts/install-ci-tools.sh
    - kubectl config use-context "$KCTX"
    # Before teardown wipes the namespaces; failure-only (the dump costs ~3 min).
    - |
      if [ "$CI_JOB_STATUS" != "success" ]; then
        E2E_SHA="${CI_COMMIT_SHORT_SHA}${E2E_SUFFIX:-}" bash e2e/scripts/dump-diagnostics.sh || true
      fi
    - bash e2e/scripts/teardown.sh --sha="${CI_COMMIT_SHORT_SHA}${E2E_SUFFIX:-}" -y
  artifacts:
    when: always
    paths:
      - e2e/diagnostics/
    reports:
      junit: e2e/diagnostics/junit.xml
    expire_in: 3 days

# Same suite against a 3-replica ClickHouse cluster; ch-chaos.sh kills one
# replica during the schema migration and another during the run.
e2e-ha:
  extends: e2e
  timeout: 90 minutes
  variables:
    E2E_CH_REPLICAS: "3"
    E2E_SUFFIX: "-ha"
    # Quorum inserts make the initial Siphon snapshot merge about three times slower.
    E2E_INDEXING_BUDGET_MULTIPLIER: "5"
  rules:
    - if: $CI_PROJECT_ID != $CANONICAL_PROJECT_ID
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_ONLY == "e2e-ha"
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true