Taskfile.yml

version: "3"

tasks:

  bootstrap:
    cmds:
      - sudo -E ./bootstrap.sh
      - . $HOME/.bashrc
      - . $HOME/.venv/bin/activate

  config:
    cmds:
      - python -m ops.tools.config.create_config

  auditor-config:
    cmds:
      - python -m ops.tools.config.create_config --auditor

  miner-config:
    cmds:
      - python -m ops.tools.config.create_config --miner

  trainer-config:
    cmds:
      - python -m ops.tools.config.create_config --trainer

  miner:
    cmds:
      - ENV=DEV uvicorn miner.asgi:app --reload --host 0.0.0.0 --port 7999 --env-file .1.env --log-level debug

  trainer:
    cmds:
      - pm2 delete trainer || true
      - pm2 start bash --name trainer -- -c 'ENV=DEV uvicorn trainer.asgi:app --host 0.0.0.0 --port 8001 --env-file .trainer.env --log-level debug'

  dbdown:
    cmds:
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml run dbmate --wait down

  dbup:
    cmds:
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml run dbmate --wait up

  setup:
    cmds:
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml -f ops/compose/docker-compose.dev.yml up -d --build

  validator_dev:
    cmds:
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml -f ops/compose/docker-compose.dev.yml up -d --build --remove-orphans
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml -f ops/compose/docker-compose.dev.yml  run dbmate --wait up
      - ./ops/validator_ops/start_validator.sh

  autoupdates:
    cmds:
      - pm2 delete autoupdater || true
      - pm2 start "python -m ops.validator_ops.run_validator_auto_update" --name autoupdater

  validator:
    cmds:
      - . $HOME/.venv/bin/activate
      - ./ops/validator_ops/setup_grafana.sh
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml up -d --build --remove-orphans
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml rm -f -v grafana
      - docker compose --env-file .vali.env -f ops/compose/docker-compose.yml up grafana -d
      - ./ops/validator_ops/start_validator.sh

  postgres:
    cmds:
      - |
        export $(cat .vali.env | grep -v '^#' | xargs)
        if [ -n "$DATABASE_URL" ]; then
          echo "Connecting using DATABASE_URL"
          psql "$DATABASE_URL"
        else
          echo "Connecting using individual parameters"
          PGPASSWORD=$POSTGRES_PASSWORD psql -U $POSTGRES_USER -d $POSTGRES_DB -h $POSTGRES_HOST
        fi

  install:
    cmds:
      - . $HOME/.venv/bin/activate
      # fiber v2.7.0+ moved to async-substrate-interface 2.x, which depends on
      # `cyscale` instead of `scalecodec`/`substrate-interface` (same import
      # namespace as scalecodec, so it refuses to load if either is still
      # around). `pip install -e .` won't remove those on its own since
      # they're no longer required by anything, so clear them out first.
      - pip uninstall -y substrate-interface scalecodec cyscale || true
      - pip install -e .

  auditor:
    cmds:
      - . $HOME/.venv/bin/activate
      - pm2 delete auditor || true
      - pm2 start "python -m ops.auditing.audit" --name auditor

  auditor-autoupdates:
    cmds:
      - pm2 delete auditor-autoupdates || true
      - pm2 start "python -m ops.validator_ops.run_auditor_autoupdate" --name auditor-autoupdates

  db-dump:
    cmds:
      - export $(cat .vali.env | grep -v '^#' | xargs) && docker run --rm --network=host -v $(pwd):/backup postgres:17 pg_dump --dbname="${DATABASE_URL:-postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}}" -F c -f /backup/god-db-backup.dump

  db-restore:
    cmds:
      - echo "Restoring database. Make sure your database connection parameters are correct in .vali.env"
      - |
        export $(cat .vali.env | grep -v '^#' | xargs)
        docker run --rm --network=host -v $(pwd):/backup postgres:17 bash -c "pg_restore --dbname=\"${DATABASE_URL:-postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}}\" --clean --if-exists -x -O /backup/god-db-backup.dump || echo 'Restore completed with some errors, which may be safely ignored'"

  deploy-observability-server:
    desc: "Deploy observability server for training logs on validator (uses .vali.env)"
    cmds:
      - echo "Deploying observability server for training logs..."
      - |
        # Load from .vali.env (this runs on the validator)
        if [ ! -f .vali.env ]; then
          echo "ERROR: .vali.env file not found!"
          echo "Please add the following variables to .vali.env:"
          echo "  OBSERVABILITY_DOMAIN=your-domain.com"
          echo "  GRAFANA_TRAINING_PASSWORD=your-password"
          echo "  LOKI_PASSWORD=your-loki-password"
          exit 1
        fi

        # Load variables from .vali.env
        export $(cat .vali.env | grep -v '^#' | xargs)

        # Auto-detect OBSERVABILITY_DOMAIN if not set
        if [ -z "$OBSERVABILITY_DOMAIN" ]; then
          # Try to get public IP
          OBSERVABILITY_DOMAIN=$(curl -s ifconfig.me || curl -s icanhazip.com || hostname -I | awk '{print $1}')
          echo "Auto-detected OBSERVABILITY_DOMAIN: $OBSERVABILITY_DOMAIN"
        fi

        # Use default passwords if not set
        GRAFANA_TRAINING_PASSWORD=${GRAFANA_TRAINING_PASSWORD:-changeme123}
        LOKI_PASSWORD=${LOKI_PASSWORD:-trainerlogs123}

        # Create observability env from .vali.env variables
        cat > .env.observability << EOF
        OBSERVABILITY_DOMAIN=${OBSERVABILITY_DOMAIN}
        GRAFANA_TRAINING_USERNAME=${GRAFANA_TRAINING_USERNAME:-admin}
        GRAFANA_TRAINING_PASSWORD=${GRAFANA_TRAINING_PASSWORD}
        GRAFANA_ANONYMOUS_ENABLED=${GRAFANA_ANONYMOUS_ENABLED:-true}
        LOKI_AUTH_ENABLED=false
        LOKI_USERNAME=${LOKI_USERNAME:-trainer}
        LOKI_PASSWORD=${LOKI_PASSWORD}
        EOF
      - |
        # Auto-create SSL certs if missing
        mkdir -p ops/observability/nginx/ssl ops/observability/nginx
        if [ ! -f ops/observability/nginx/ssl/cert.pem ]; then
          export $(cat .vali.env | grep -v '^#' | xargs)
          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
            -keyout ops/observability/nginx/ssl/key.pem \
            -out ops/observability/nginx/ssl/cert.pem \
            -subj "/C=US/ST=State/L=City/O=Training/CN=${OBSERVABILITY_DOMAIN}" 2>/dev/null
        fi
      - |
        # Auto-create basic auth file with password from .vali.env
        if [ ! -f ops/observability/nginx/.htpasswd ] || [ "$LOKI_PASSWORD_UPDATED" = "true" ]; then
          export $(cat .vali.env | grep -v '^#' | xargs)
          # Check if htpasswd is available, if not use Docker
          if command -v htpasswd >/dev/null 2>&1; then
            htpasswd -b -B -c ops/observability/nginx/.htpasswd ${LOKI_USERNAME:-trainer} ${LOKI_PASSWORD} 2>/dev/null
          else
            # Use Docker to generate htpasswd file
            docker run --rm -v $(pwd)/ops/observability/nginx:/work httpd:alpine htpasswd -b -B -c /work/.htpasswd ${LOKI_USERNAME:-trainer} ${LOKI_PASSWORD}
          fi
        fi
      - docker compose -f ops/compose/docker-compose.observability-server.yml --env-file .env.observability up -d
      - |
        export $(cat .vali.env | grep -v '^#' | xargs)
        echo "========================================="
        echo "Observability server started!"
        echo "Grafana: https://$(hostname -I | awk '{print $1}' 2>/dev/null || echo ${OBSERVABILITY_DOMAIN}):3001"
        echo "Login: ${GRAFANA_TRAINING_USERNAME:-admin} / [password from .vali.env]"
        echo "========================================="

  stop-observability-server:
    desc: "Stop the observability server"
    cmds:
      - docker compose -f ops/compose/docker-compose.observability-server.yml down

  deploy-trainer-logs:
    desc: "Deploy log shipping on trainer node (uses .trainer.env)"
    cmds:
      - echo "Deploying trainer log shipping..."
      - |
        # Load from .trainer.env ONLY
        if [ ! -f .trainer.env ]; then
          echo "ERROR: .trainer.env file not found!"
          echo "Please add the following variables to .trainer.env:"
          echo "  LOKI_ENDPOINT=https://validator-server:3101"
          echo "  LOKI_PASSWORD=your-loki-password"
          exit 1
        fi

        # Load variables from .trainer.env
        export $(cat .trainer.env | grep -v '^#' | xargs)

        # Check required variables
        if [ -z "$LOKI_ENDPOINT" ] || [ -z "$LOKI_PASSWORD" ]; then
          echo "ERROR: Missing required variables in .trainer.env:"
          echo "  LOKI_ENDPOINT=${LOKI_ENDPOINT:-<not set>}"
          echo "  LOKI_PASSWORD=${LOKI_PASSWORD:-<not set>}"
          exit 1
        fi

        # Create trainer env for Vector
        cat > .env.trainer-logs << EOF
        LOKI_ENDPOINT=${LOKI_ENDPOINT}
        LOKI_USERNAME=${LOKI_USERNAME:-trainer}
        LOKI_PASSWORD=${LOKI_PASSWORD}
        VECTOR_LOG_LEVEL=${VECTOR_LOG_LEVEL:-info}
        EOF
      - |
        docker rm -f god-node-exporter god-vector-log-shipper 2>/dev/null || true
        docker compose -p god -f ops/compose/docker-compose.trainer-server.yml --env-file .env.trainer-logs up -d --remove-orphans
      - echo "Log shipping started to ${LOKI_ENDPOINT}"

  stop-trainer-logs:
    desc: "Stop log shipping on trainer node"
    cmds:
      - docker compose -p god -f ops/compose/docker-compose.trainer-server.yml down

  logs-observability:
    desc: "View logs from observability server"
    cmds:
      - docker compose -f ops/compose/docker-compose.observability-server.yml logs -f

  logs-trainer-shipper:
    desc: "View logs from trainer log shipper"
    cmds:
      - docker compose -f ops/compose/docker-compose.trainer-server.yml logs -f vector-log-shipper

  status-trainer-logs:
    desc: "Check status of trainer logging system"
    cmds:
      - echo "=== Observability Server Status ==="
      - docker compose -f ops/compose/docker-compose.observability-server.yml ps 2>/dev/null || echo "Not running on this server"
      - echo ""
      - echo "=== Trainer Log Shipper Status ==="
      - docker compose -f ops/compose/docker-compose.trainer-server.yml ps 2>/dev/null || echo "Not running on this server"
      - echo ""
      - echo "=== Vector Metrics ==="
      - curl -s http://localhost:8686/health 2>/dev/null && echo "Vector is healthy" || echo "Vector not accessible"

  test-trainer-logs:
    desc: "Test log shipping by creating test containers with different labels"
    cmds:
      - ./ops/tools/observability/test_trainer_logs.sh