feat: storokufy sprue

.github/workflows/deploy.yml

@@ -0,0 +1,78 @@
+# storoku:ignore
+
+name: Deploy
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches: [main]
+  workflow_run:
+    workflows: [Releaser]
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        description: Environment
+        options:
+          - warm-staging
+          - forge-production
+          - forge-test
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  # apply staging on pushes to main, plan otherwise
+  warm-staging:
+    uses: ./.github/workflows/terraform.yml
+    with:
+      env: warm-staging
+      workspace: warm-staging
+      network: warm
+      did: did:web:staging.up.warm.storacha.network
+      apply: ${{ github.event_name != 'pull_request' }}
+    secrets:
+      aws-account-id: ${{ secrets.WARM_STAGING_AWS_ACCOUNT_ID }}
+      aws-region: ${{ secrets.WARM_STAGING_AWS_REGION }}
+      region: ${{ secrets.WARM_STAGING_AWS_REGION }}
+      private-key: ${{ secrets.WARM_STAGING_PRIVATE_KEY }}
+      cloudflare-zone-id: ${{ secrets.WARM_STAGING_CLOUDFLARE_ZONE_ID }}
+      cloudflare-api-token: ${{ secrets.WARM_STAGING_CLOUDFLARE_API_TOKEN }}
+
+  # apply prod and test on successful release, plan otherwise
+  forge-production:
+    uses: ./.github/workflows/terraform.yml
+    with:
+      env: forge-production
+      workspace: forge-prod
+      network: forge
+      did: did:web:up.forge.storacha.network
+      apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-production') }}
+    secrets:
+      aws-account-id: ${{ secrets.FORGE_PROD_AWS_ACCOUNT_ID }}
+      aws-region: ${{ secrets.FORGE_PROD_AWS_REGION }}
+      region: ${{ secrets.FORGE_PROD_AWS_REGION }}
+      private-key: ${{ secrets.FORGE_PROD_PRIVATE_KEY }}
+      cloudflare-zone-id: ${{ secrets.FORGE_PROD_CLOUDFLARE_ZONE_ID }}
+      cloudflare-api-token: ${{ secrets.FORGE_PROD_CLOUDFLARE_API_TOKEN }}
+
+  forge-test:
+    uses: ./.github/workflows/terraform.yml
+    with:
+      env: forge-test
+      workspace: forge-test
+      network: test
+      did: did:web:up.test.storacha.network
+      apply: ${{ (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'forge-test') }}
+    secrets:
+      aws-account-id: ${{ secrets.FORGE_TEST_AWS_ACCOUNT_ID }}
+      aws-region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      region: ${{ secrets.FORGE_TEST_AWS_REGION }}
+      private-key: ${{ secrets.FORGE_TEST_PRIVATE_KEY }}
+      cloudflare-zone-id: ${{ secrets.FORGE_TEST_CLOUDFLARE_ZONE_ID }}
+      cloudflare-api-token: ${{ secrets.FORGE_TEST_CLOUDFLARE_API_TOKEN }}

.github/workflows/terraform.yml

@@ -0,0 +1,108 @@
+name: Terraform
+
+on:
+  workflow_call:
+    inputs:
+      env:
+        required: true
+        type: string
+      workspace:
+        required: true
+        type: string
+      network:
+        required: false
+        default: "hot"
+        type: string
+      did:
+        required: true
+        type: string
+      apply:
+        required: true
+        type: boolean
+    secrets:
+      aws-account-id:
+        required: true
+      aws-region:
+        required: true
+      private-key:
+        required: true
+      region:
+        required: true
+      cloudflare-zone-id:
+        required: true
+      cloudflare-api-token:
+        required: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.workspace }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  AWS_ACCOUNT_ID: ${{ secrets.aws-account-id }}
+  AWS_REGION: ${{ secrets.aws-region }}
+  ENV: ${{ inputs.env }}
+  TF_WORKSPACE: ${{ inputs.workspace }}
+  TF_VAR_network: ${{ inputs.network }}
+  TF_VAR_private_key: ${{ secrets.private-key }}
+  TF_VAR_did: ${{ inputs.did }}
+  TF_VAR_app: sprue
+  TF_VAR_domain_base: 
+  TF_VAR_allowed_account_id: ${{ secrets.aws-account-id }}
+  TF_VAR_region: ${{ secrets.region }}
+  TF_VAR_cloudflare_zone_id: ${{ secrets.cloudflare-zone-id }}
+  CLOUDFLARE_API_TOKEN: ${{ secrets.cloudflare-api-token }}
+  DEPLOY_ENV: ci
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform:
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/terraform-ci
+
+      - uses: opentofu/setup-opentofu@v1
+
+      - name: Tofu Init
+        run: |
+          touch .tfworkspace
+          make init
+        working-directory: deploy
+
+      # just plan if !inputs.apply
+      - name: Terraform Plan
+        if: ${{ !inputs.apply }}
+        run: |
+          make plan
+        working-directory: deploy
+
+      # build and push docker image and apply if inputs.apply
+      - name: Set up Docker Buildx
+        if: ${{ inputs.apply }}
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build + Push Docker ECR
+        if: ${{ inputs.apply }}
+        run: |
+          make docker-push
+        working-directory: deploy
+
+      - name: Terraform Apply
+        if: ${{ inputs.apply }}
+        run: |
+          make apply
+        working-directory: deploy
+
+      - name: Wait For Deployment
+        if: ${{ inputs.apply }}
+        run: |
+          make wait-deploy
+        working-directory: deploy

.storoku.json

@@ -0,0 +1,22 @@
+{
+  "app": "sprue",
+  "privateKeyEnvVar": "SPRUE_IDENTITY_PRIVATE_KEY",
+  "didEnvVar": "SPRUE_IDENTITY_SERVICE_DID",
+  "port": 0,
+  "js": null,
+  "domainBase": "",
+  "cloudflare": true,
+  "createDB": false,
+  "caches": null,
+  "topics": null,
+  "queues": null,
+  "buckets": null,
+  "secrets": null,
+  "tables": null,
+  "networks": [
+    "forge",
+    "warm",
+    "test"
+  ],
+  "writeToContainer": false
+}

deploy/.env.production.local.tpl

@@ -0,0 +1,39 @@
+<%
+if [ "$TF_WORKSPACE" == "forge-prod" ]; then
+  DEPLOYMENT_PREFIX="forge-prod-upload-api"
+
+  AGENT_MESSAGE_BUCKET="forge-prod-upload-api-workflow-store-0"
+  DELEGATION_BUCKET="forge-prod-upload-api-delegation-0"
+  UPLOAD_SHARDS_BUCKET="forge-prod-upload-api-upload-shards-0"
+elif [ "$TF_WORKSPACE" == "forge-test" ]; then
+  DEPLOYMENT_PREFIX="forge-test-w3infra"
+
+  AGENT_MESSAGE_BUCKET="workflow-store-forge-test-0"
+  DELEGATION_BUCKET="delegation-forge-test-0"
+  UPLOAD_SHARDS_BUCKET="upload-shards-forge-test-0"
+else
+  DEPLOYMENT_PREFIX="staging-warm-upload-api"
+
+  AGENT_MESSAGE_BUCKET="staging-warm-upload-api-workflow-store-0"
+  DELEGATION_BUCKET="staging-warm-upload-api-delegation-0"
+  UPLOAD_SHARDS_BUCKET="staging-warm-upload-api-upload-shards-0"
+fi
+%>
+
+SPRUE_DYNAMODB_AGENT_INDEX_TABLE=<%= $DEPLOYMENT_PREFIX %>-agent-index
+SPRUE_DYNAMODB_BLOB_REGISTRY_TABLE=<%= $DEPLOYMENT_PREFIX %>-blob-registry
+SPRUE_DYNAMODB_CONSUMER_TABLE=<%= $DEPLOYMENT_PREFIX %>-consumer
+SPRUE_DYNAMODB_CUSTOMER_TABLE=<%= $DEPLOYMENT_PREFIX %>-customer
+SPRUE_DYNAMODB_DELEGATION_TABLE=<%= $DEPLOYMENT_PREFIX %>-delegation
+SPRUE_DYNAMODB_SPACE_METRICS_TABLE=<%= $DEPLOYMENT_PREFIX %>-space-metrics
+SPRUE_DYNAMODB_ADMIN_METRICS_TABLE=<%= $DEPLOYMENT_PREFIX %>-admin-metrics
+SPRUE_DYNAMODB_REPLICA_TABLE=<%= $DEPLOYMENT_PREFIX %>-replica
+SPRUE_DYNAMODB_REVOCATION_TABLE=<%= $DEPLOYMENT_PREFIX %>-revocation
+SPRUE_DYNAMODB_STORAGE_PROVIDER_TABLE=<%= $DEPLOYMENT_PREFIX %>-storage-provider
+SPRUE_DYNAMODB_SUBSCRIPTION_TABLE=<%= $DEPLOYMENT_PREFIX %>-subscription
+SPRUE_DYNAMODB_SPACE_DIFF_TABLE=<%= $DEPLOYMENT_PREFIX %>-space-diff
+SPRUE_DYNAMODB_UPLOAD_TABLE=<%= $DEPLOYMENT_PREFIX %>-upload
+
+SPRUE_S3_AGENT_MESSAGE_BUCKET=<%= AGENT_MESSAGE_BUCKET %>
+SPRUE_S3_DELEGATION_BUCKET=<%= DELEGATION_BUCKET %>
+SPRUE_S3_UPLOAD_SHARDS_BUCKET=<%= UPLOAD_SHARDS_BUCKET %>

deploy/.env.terraform.tpl

@@ -0,0 +1,9 @@
+# copy to .env.terraform and set missing vars
+TF_WORKSPACE= # your name here
+TF_VAR_app=sprue
+TF_VAR_did= # did for your env
+TF_VAR_private_key= # private_key or your env -- do not commit to repo!
+TF_VAR_allowed_account_id=505595374361
+TF_VAR_region=us-east-2
+TF_VAR_cloudflare_zone_id=37783d6f032b78cd97ce37ab6fd42848
+CLOUDFLARE_API_TOKEN= # enter a cloudflare api token

deploy/.gitignore

@@ -0,0 +1,8 @@
+# Deployment
+.env.production.local
+.env.terraform
+.terraform
+.tfworkspace
+app/code_deploy.sh
+shared/builds
+app/builds