Build on remote server

Author: justinlittman

.kamal/hooks/post-deploy

@@ -21,7 +21,7 @@ REPO_URL=$(git config --get remote.origin.url)
 API_KEY=$(vault kv get -field=content puppet/application/argo-b3/honeybadger-api-key)
 
 curl --no-progress-meter --data \
-  "deploy[environment]=${KAMAL_DESTINATION}&deploy[local_username]=${KAMAL_PERFORMER}&deploy[revision]=${KAMAL_VERSION}&deploy[repository]=${REPO_URL}&api_key=${API_KEY}" \
+  "deploy[environment]=${KAMAL_DESTINATION}&deploy[local_username]=$(whoami)&deploy[revision]=${KAMAL_VERSION}&deploy[repository]=${REPO_URL}&api_key=${API_KEY}" \
   https://api.honeybadger.io/v1/deploys
 
 echo "Honeybadger notification complete."

Dockerfile

@@ -88,10 +88,11 @@ USER 1000:1000
 COPY --chown=rails:rails --from=build "${BUNDLE_PATH}" "${BUNDLE_PATH}"
 COPY --chown=rails:rails --from=build /rails /rails
 
-# Rails creates tmp/{cache,pids,sockets} at boot; some destinations (see
-# config/deploy.qa.yml) run the container as a different UID than the image's
-# baked-in rails user, so make tmp/ writable regardless of runtime UID.
-RUN chmod a+rwX tmp
+# Rails creates tmp/{cache,pids,sockets} at boot and writes log/production.log;
+# some destinations (see config/deploy.qa.yml) run the container as a different
+# UID than the image's baked-in rails user, so make these writable regardless
+# of runtime UID.
+RUN chmod -R a+rwX tmp log
 
 # Entrypoint prepares the database.
 ENTRYPOINT ["/rails/bin/docker-entrypoint"]

bin/kamal-otk

@@ -5,9 +5,15 @@
 # (https://github.com/cbeer/capistrano-one_time_key). Generates a fresh, passphrase-less
 # keypair, adds the public key to each target host's ~/.ssh/authorized_keys (relying on
 # whatever trusted channel -- e.g. Kerberos/ksu -- already lets that connection succeed),
-# runs `bin/kamal deploy` using that key, then always removes it from authorized_keys.
+# runs `bin/kamal` using that key, then always removes it from authorized_keys.
 #
-# Usage: bin/deploy DESTINATION [kamal deploy args...]
+# The key is both configured via config/deploy.<destination>.yml's `ssh.keys` (read from
+# KAMAL_SSH_KEY_PATH) and loaded into a scratch ssh-agent: Kamal's remote-builder port
+# forwarding (lib/kamal/cli/build.rb#remote_builder_ssh_options) builds its own SSH options
+# from the `builder.remote` URI and doesn't include configured keys, so it only succeeds
+# via agent-based auth.
+#
+# Usage: bin/kamal-otk DESTINATION [kamal args...]
 
 require 'bundler/setup'
 require 'kamal'
@@ -29,6 +35,21 @@ def ssh_exec!(user, host, command)
   system('ssh', "#{user}@#{host}", command, exception: true)
 end
 
+# Same mechanism as dlss-capistrano's controlmaster:setup/:start tasks:
+# https://github.com/sul-dlss/dlss-capistrano/blob/main/lib/dlss/capistrano/tasks/control_master.rake
+def ensure_controlmaster!
+  return unless ENV.fetch('USE_CAPISTRANO_CONTROLMASTER', false) == 'true'
+
+  controlmaster_host = ENV.fetch('CONTROLMASTER_HOST', 'dlss-jump')
+  controlmaster_socket = ENV.fetch('CONTROLMASTER_SOCKET', '~/.ssh/%r@%h:%p')
+
+  return if system('ssh', '-O', 'check', '-S', controlmaster_socket, controlmaster_host)
+
+  system('ssh', '-f', '-N', '-S', controlmaster_socket, controlmaster_host, exception: true)
+end
+
+ensure_controlmaster!
+
 Dir.mktmpdir do |dir|
   key_path = File.join(dir, 'deploy_otk')
   comment = "kamal-otk-#{SecureRandom.hex(6)}"
@@ -42,10 +63,19 @@ Dir.mktmpdir do |dir|
 
   hosts.each { |host| ssh_exec!(user, host, add_key_command) }
 
+  agent_pid = nil
   begin
+    `ssh-agent -s`.each_line do |line|
+      ENV[Regexp.last_match(1)] = Regexp.last_match(2) if line =~ /^(SSH_AUTH_SOCK|SSH_AGENT_PID)=([^;]+);/
+    end
+    agent_pid = ENV['SSH_AGENT_PID']
+    system('ssh-add', key_path, exception: true)
+
     ENV['KAMAL_SSH_KEY_PATH'] = key_path
     system('bin/kamal', *ARGV, '-d', destination, exception: true)
   ensure
+    system('ssh-agent', '-k') if agent_pid
+
     # Best-effort cleanup: don't let a removal failure mask the deploy's own result.
     remove_key_command = "sed -i.bak -e '/#{comment}$/d' ~/.ssh/authorized_keys && rm -f ~/.ssh/authorized_keys.bak"
     hosts.each do |host|

config/deploy.qa.yml

@@ -1,22 +1,41 @@
 # Deploy to the qa server. Apache (with Shibboleth SSO) already handles SSL
 # termination and reverse proxying on this host -- it proxies to localhost:3000,
-# same as it does for Passenger on poc -- so Kamal's own proxy is disabled.
+# same as it does for Passenger on poc. kamal-proxy is still used (left enabled
+# for the primary "web" role), but rebound to plain HTTP on 3000 instead of its
+# default 80/443 (which Apache already owns) -- this lets kamal-proxy do its
+# normal zero-downtime container swap over the internal "kamal" network,
+# without the app container needing to publish (and collide on) a host port.
 # Kamal always attaches app containers to its own "kamal" bridge network (no
 # config to disable that), so host networking isn't possible here: instead we
-# publish the container's port directly and reach the host's Postgres via
-# host.docker.internal (see the pg_hba.conf note for the Puppet-side change
-# this requires).
+# reach the host's Postgres via host.docker.internal (see the pg_hba.conf note
+# for the Puppet-side change this requires).
+proxy:
+  ssl: false
+  run:
+    http_port: 3000
+    https_port: 3001 # unused (ssl disabled), just must not collide with Apache's 443
+
 servers:
   web:
     hosts:
       - argo-b3-qa-a.stanford.edu
-    proxy: false
     options:
-      publish: "3000:80"
       add-host: "host.docker.internal:host-gateway"
       # Overrides the image's baked-in rails (1000:1000) user so bind-mounted
       # writes (bulk actions, logs) land as lyberadmin, matching host ownership.
       user: "503:503"
+  # Runs SolidQueue standalone (bin/jobs) rather than in-Puma, so it registers
+  # its own graceful-shutdown signal handlers (see solid_queue/supervisor/signals.rb)
+  # and can drain in-flight jobs on its own timeline, independent of the web
+  # role -- analogous to Sidekiq's hotswap deploys on poc/legacy argo hosts.
+  job:
+    hosts:
+      - argo-b3-qa-a.stanford.edu
+    cmd: bin/jobs
+    stop_timeout: <%= ENV.fetch("SOLID_QUEUE_SHUTDOWN_TIMEOUT", 86400) %>
+    options:
+      add-host: "host.docker.internal:host-gateway"
+      user: "503:503"
 
 ssh:
   user: lyberadmin
@@ -25,6 +44,11 @@ ssh:
     - <%= ENV["KAMAL_SSH_KEY_PATH"] %>
 <% end -%>
 
+# Build natively on the deploy target itself (already has Docker via Puppet)
+# rather than emulating amd64 via QEMU on an arm64 developer laptop.
+builder:
+  remote: ssh://lyberadmin@argo-b3-qa-a.stanford.edu
+
 # Bind-mount host paths for durable bulk action output (NFS-backed, shared
 # with poc) and Rails log output (for host-level log shipping).
 volumes:
@@ -44,6 +68,14 @@ env:
     SETTINGS__PRESERVATION_CATALOG__URL: https://preservation-catalog-qa.stanford.edu
     SETTINGS__STACKS__URL: https://stacks-stage.stanford.edu/image
     SETTINGS__PURL_FETCHER__URL: https://purl-fetcher-stage.stanford.edu
+
+    HONEYBADGER_ENV: qa
+
+    # Governs how long SolidQueue's supervisor waits for in-flight jobs to
+    # finish on shutdown (see config/initializers/solid_queue.rb); the job
+    # role's stop_timeout above is set to match, so Docker's SIGKILL doesn't
+    # cut the graceful drain short.
+    SOLID_QUEUE_SHUTDOWN_TIMEOUT: <%= ENV.fetch("SOLID_QUEUE_SHUTDOWN_TIMEOUT", 86400) %>
   secret:
     - HONEYBADGER_API_KEY
     - SECRET_KEY_BASE

config/deploy.yml

@@ -10,10 +10,17 @@ image: suldlss/argo-b3
 registry:
   server: localhost:5555
 
-# Configure builder setup.
+# Configure builder setup. Destination-specific config (config/deploy.<destination>.yml)
+# points `remote` at the deploy target itself, for a native (non-QEMU) amd64 build.
 builder:
   arch: amd64
 
+logging:
+  driver: json-file
+  options:
+    max-size: "100m"
+    max-file: "5"
+
 # Destination-specific configuration (servers, env, secrets, volumes) lives in
 # config/deploy.<destination>.yml, e.g. config/deploy.qa.yml. Run with:
 #   bin/kamal deploy -d qa

config/environments/production.rb

@@ -33,9 +33,12 @@
   # Skip http-to-https redirect for the default health check endpoint.
   # config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/up" } } }
 
-  # Log to STDOUT with the current request id as a default log tag.
+  # Log to STDOUT (so Kamal/Docker can collect container logs) and to
+  # log/production.log, with the current request id as a default log tag.
   config.log_tags = [ :request_id ]
-  config.logger   = ActiveSupport::TaggedLogging.logger(ENV["RAILS_LOG_TO_FILE"] || STDOUT)
+  file_logger = ActiveSupport::Logger.new(Rails.root.join("log", "production.log"))
+  stdout_logger = ActiveSupport::Logger.new(STDOUT)
+  config.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::BroadcastLogger.new(file_logger, stdout_logger))
 
   # Change to "debug" to log everything (including potentially personally-identifiable information!).
   config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info")

config/initializers/solid_queue.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# How long the SolidQueue supervisor waits for in-flight jobs to finish on
+# shutdown before force-terminating them. Kamal's job role stop_timeout
+# (config/deploy.qa.yml) is set to match, so Docker's SIGKILL doesn't cut
+# this grace period short.
+SolidQueue.shutdown_timeout = ENV.fetch('SOLID_QUEUE_SHUTDOWN_TIMEOUT', 86_400).to_i