INS-11420 | Make deploy-tag recording collision-safe and idempotent

The deploy tag was numbered from a `git fetch --tags` snapshot at the START of a
promote, then pushed at the END — after a minutes-long deploy — so any deploy
tag that appeared meanwhile made the final push fail ("reference already
exists"), turning a successful production deploy red. Re-running then minted a
second numbered tag for the same commit.

- Number + create + push the tag AFTER the deploy, so the number reflects origin
  state seconds before the push (not minutes).
- Idempotent push: a losing-race collision is accepted only when it is a
  "reference already exists" collision (decided from the push OUTPUT, not the
  executor's generic exception) AND the deployed artifact matches what is already
  recorded. Promote verifies binary identity via Heroku slug checksum
  (heroku.rb same_slug?/slug_checksum); the deploy path falls back to the git
  commit the existing tag points at. A same-name / different-slug collision fails
  loudly.

Adds spec/stairstep/common/git_spec.rb (the repo had no specs before): fresh tag,
numbering, deploy ref-name path, failed-deploy -> no tag, deploy-before-tag
ordering, fetch re-run, slug-verify both branches, non-collision re-raise, and
ls-remote empty/multi-line.

Tool: Claude Code (claude-opus-4-8[1m])

Author: JasonRTodd-SRP

lib/stairstep/common/git.rb

@@ -1,4 +1,5 @@
 require "date"
+require "stringio"
 require_relative "../../stairstep"
 
 module Stairstep::Common
@@ -16,13 +17,15 @@ def commit_sha(ref_name)
       git("rev-parse", "--verify", ref_name, capture_stdout: true).chomp
     end
 
-    def with_tag(to_remote, commit:, message:, tag:)
-      tag_name = build_tag_name(to_remote) if tag
-      git("tag", "-a", "-m", message, tag_name, commit) if tag
+    # Records the deploy as an annotated tag on origin, AFTER the deploy runs, so
+    # a tag that appears on origin during the — potentially minutes-long — deploy
+    # can't make us commit to a stale number and then collide on push. If the push
+    # still loses a race, the collision is only accepted when the deployed
+    # artifact matches what is already recorded (see #push_deploy_tag); promote
+    # passes verify_deploy for a Heroku slug-checksum comparison.
+    def with_tag(to_remote, commit:, message:, tag:, verify_deploy: nil)
       yield
-      save_tag(tag_name) if tag
-    ensure
-      delete_tag(tag_name) if tag
+      save_deploy_tag(to_remote, commit, message, verify_deploy) if tag
     end
 
     def verify_clean_working_directory
@@ -85,8 +88,64 @@ def existing_tags
         end
     end
 
-    def save_tag(tag_name)
-      git("push", "origin", tag_name)
+    # Number, create, and push the deploy tag — all after the deploy — so the
+    # number reflects origin state seconds before the push (not minutes) and a tag
+    # that appeared mid-deploy can't force a stale collision. Each deploy records
+    # its own tag; #push_deploy_tag decides whether a losing-race collision is
+    # benign.
+    def save_deploy_tag(remote, commit, message, verify_deploy)
+      refresh_existing_tags!
+      tag_name = build_tag_name(remote)
+      git("tag", "-a", "-m", message, tag_name, commit)
+      push_deploy_tag(tag_name, commit_sha(commit), verify_deploy)
+    ensure
+      delete_tag(tag_name) if tag_name
+    end
+
+    # A rejected push is treated as success only when BOTH hold:
+    #   * it is a "tag already exists" collision — decided from the push output,
+    #     not the (generic) exception, and NOT an auth/network/hook failure; and
+    #   * the deployed artifact matches what the colliding tag records
+    #     (see #deployed_artifact_matches?).
+    # Anything else re-raises.
+    def push_deploy_tag(tag_name, deployed_sha, verify_deploy)
+      output = StringIO.new
+      succeeded = executor.execute("git", "push", "origin", tag_name, output: output)
+      print(output.string)
+      return if succeeded
+
+      unless tag_collision?(output.string) && deployed_artifact_matches?(tag_name, deployed_sha, verify_deploy)
+        raise("Command failed: `git push origin #{tag_name}`")
+      end
+
+      logger.info("Tag #{tag_name} already on origin and the deployed artifact matches; treating as success")
+    end
+
+    def tag_collision?(push_output)
+      push_output.match?(/reference already exists|cannot lock ref/)
+    end
+
+    # Whether the just-deployed artifact matches what the colliding tag already
+    # records. Promote supplies verify_deploy — a Heroku slug-checksum comparison
+    # (deployed vs. live) — so a same-name / different-slug deploy fails loudly
+    # (INS-11420 AC1). Without it, the deploy path (no from/to slug pair) falls
+    # back to the git commit the existing tag points at.
+    def deployed_artifact_matches?(tag_name, deployed_sha, verify_deploy)
+      return verify_deploy.call if verify_deploy
+
+      remote_tag_commit(tag_name) == deployed_sha
+    end
+
+    def remote_tag_commit(tag_name)
+      # ls-remote prints "<sha>\t<ref>" per matching ref; take the sha off the ref
+      # line (line-anchored, so a leading warning/redirect line doesn't derail it).
+      git("ls-remote", "origin", "refs/tags/#{tag_name}^{}", capture_stdout: true)[/^(\h{7,})\s/, 1]
+    rescue
+      nil
+    end
+
+    def refresh_existing_tags!
+      @existing_tags = nil
     end
 
     def delete_tag(tag_name)

lib/stairstep/common/heroku.rb

@@ -46,6 +46,18 @@ def slug_id(remote)
       JSON.parse(release_json).dig("slug", "id") || fail
     end
 
+    # True when the two remotes are running the same slug binary, compared by the
+    # slug's content checksum (not its git commit — one commit can back several
+    # rebuilt slugs). Used to decide whether a deploy-tag collision is benign.
+    def same_slug?(pipeline, from_remote, to_remote)
+      slug_checksum(pipeline, from_remote) == slug_checksum(pipeline, to_remote)
+    end
+
+    def slug_checksum(pipeline, remote)
+      path = "/apps/#{app_name(pipeline, remote)}/slugs/#{slug_id(remote)}"
+      JSON.parse(heroku_api("GET", path)).fetch("checksum")
+    end
+
     def scale_dynos(remote, initial_deploy:)
       heroku(remote, "ps:scale", *worker_dyno_counts(remote).collect { |type, _| "#{type}=0" }) unless initial_deploy
       yield

lib/stairstep/promote.rb

@@ -48,7 +48,15 @@ def from_commit
   end
 
   def promote_slug
-    git.with_tag(to_remote, commit: from_commit, message: "Deploy to #{to_remote} from #{from_remote} at #{Time.now}", tag: tag?) do
+    git.with_tag(
+      to_remote,
+      commit: from_commit,
+      message: "Deploy to #{to_remote} from #{from_remote} at #{Time.now}",
+      tag: tag?,
+      # Only consulted if the tag push loses a race: is prod running the slug we
+      # just promoted? A same-name / different-slug collision then fails loudly.
+      verify_deploy: -> { heroku.same_slug?(pipeline, from_remote, to_remote) }
+    ) do
       heroku.manage_deploy(to_remote, downtime: downtime?, initial_deploy: initial_deploy?) do
         heroku.promote_slug(pipeline, from_remote, to_remote)
       end

spec/stairstep/common/git_spec.rb

@@ -0,0 +1,190 @@
+require "date"
+require "stairstep/common/git"
+
+# Fake CommandExecutor modelling the real contract Git relies on:
+#   * #execute(*cmd, output:) runs a command, writes its (fake) output to the
+#     given IO, and returns a success boolean — used only for the tag push.
+#   * #execute!(*cmd) runs a command and raises on failure — used for the other
+#     mutating git commands.
+#   * #fetch_stdout(:execute!, *cmd) returns canned stdout for a read command.
+# Every command is recorded so examples can assert what git ran.
+class FakeGitExecutor
+  attr_reader :calls
+
+  def initialize(stdout: {}, fail_on: [], push_fails: false, push_output: "")
+    @stdout = stdout
+    @fail_on = fail_on
+    @push_fails = push_fails
+    @push_output = push_output
+    @calls = []
+  end
+
+  def execute(*command, output: nil, **)
+    @calls << command
+    output&.write(@push_output)
+    !@push_fails
+  end
+
+  def execute!(*command, **)
+    @calls << command
+    raise "Command failed: `#{command.join(" ")}`" if @fail_on.include?(command)
+
+    true
+  end
+
+  def fetch_stdout(_exec, *command, **)
+    @calls << command
+    @stdout.fetch(command, "")
+  end
+
+  def ran?(*command) = @calls.include?(command)
+
+  def pushed_any_tag? = @calls.any? { |c| c[0, 2] == ["git", "push"] }
+
+  def tagged_any? = @calls.any? { |c| c[0, 2] == ["git", "tag"] }
+end
+
+RSpec.describe Stairstep::Common::Git do
+  subject(:git) { described_class.new(executor, logger) }
+
+  let(:logger) { double("Logger", info: nil) }
+  let(:executor) do
+    FakeGitExecutor.new(stdout: stdout, fail_on: fail_on, push_fails: push_fails, push_output: push_output)
+  end
+  let(:fail_on) { [] }
+  let(:push_fails) { false }
+  let(:push_output) { "" }
+
+  let(:base) { "deploy-production-#{Date.today}" }
+  let(:commit) { "abc123abc123abc123abc123abc123abc123abcd" }
+
+  # By default `commit` resolves to itself and today has no deploy tags yet.
+  let(:stdout) do
+    {
+      ["git", "rev-parse", "--verify", commit] => commit,
+      ["git", "tag"] => existing_tag_list.join("\n")
+    }
+  end
+  let(:existing_tag_list) { [] }
+
+  def record_tag(ref: commit, verify_deploy: nil, &block)
+    block ||= -> { :deployed }
+    git.with_tag("production", commit: ref, message: "Deploy", tag: true, verify_deploy: verify_deploy, &block)
+  end
+
+  describe "ordering / gating" do
+    it "runs the deploy (yield) before any tagging or push" do
+      calls_at_yield = nil
+      record_tag { calls_at_yield = executor.calls.dup }
+
+      # nothing git-tag/push related had run by the time the deploy block executed
+      expect(calls_at_yield.any? { |c| c[0, 2] == ["git", "tag"] || c[0, 2] == ["git", "push"] }).to be(false)
+      # ...and tagging did happen afterwards
+      expect(executor.tagged_any?).to be(true)
+    end
+
+    it "does nothing when tagging is disabled" do
+      git.with_tag("production", commit: commit, message: "Deploy", tag: false) { :deployed }
+
+      expect(executor.calls).to be_empty
+    end
+
+    it "records no tag when the deploy fails" do
+      expect { record_tag { raise "boom" } }.to raise_error(/boom/)
+
+      expect(executor.tagged_any?).to be(false)
+      expect(executor.pushed_any_tag?).to be(false)
+    end
+  end
+
+  describe "recording a new deploy tag" do
+    it "re-fetches tags, creates a numbered tag, pushes it, and cleans up locally" do
+      record_tag
+
+      expect(executor.ran?("git", "fetch", "--tags")).to be(true) # AC2: numbering off a fresh fetch
+      expect(executor.ran?("git", "tag", "-a", "-m", "Deploy", base, commit)).to be(true)
+      expect(executor.ran?("git", "push", "origin", base)).to be(true)
+      expect(executor.ran?("git", "tag", "-d", base)).to be(true)
+    end
+
+    it "picks the next free number when earlier tags for today already exist" do
+      allow_existing(base, "#{base}.1")
+
+      record_tag
+
+      expect(executor.ran?("git", "tag", "-a", "-m", "Deploy", "#{base}.2", commit)).to be(true)
+    end
+
+    it "tags the resolved commit on the deploy ref-name path" do
+      ref = "refs/deploys/production-#{Date.today}"
+      stdout[["git", "rev-parse", "--verify", ref]] = commit # ref peels to a real SHA
+
+      record_tag(ref: ref)
+
+      expect(executor.ran?("git", "tag", "-a", "-m", "Deploy", base, ref)).to be(true)
+    end
+  end
+
+  describe "push collision — promote path (slug identity)" do
+    let(:push_fails) { true }
+    let(:push_output) { " ! [remote rejected] #{base} -> #{base} (cannot lock ref 'refs/tags/#{base}': reference already exists)" }
+
+    it "treats the collision as success when the deployed slug matches (verify_deploy true)" do
+      expect { record_tag(verify_deploy: -> { true }) }.not_to raise_error
+      expect(executor.ran?("git", "tag", "-d", base)).to be(true) # still cleans up local
+    end
+
+    it "fails loudly when the deployed slug differs (verify_deploy false)" do
+      expect { record_tag(verify_deploy: -> { false }) }.to raise_error(/Command failed/)
+    end
+  end
+
+  describe "push collision — deploy path (commit identity fallback)" do
+    let(:push_fails) { true }
+    let(:push_output) { "cannot lock ref 'refs/tags/#{base}': reference already exists" }
+
+    it "treats a same-commit collision as success" do
+      stub_remote_peeled(base, commit)
+
+      expect { record_tag }.not_to raise_error
+    end
+
+    it "re-raises when the existing tag points at a different commit" do
+      stub_remote_peeled(base, "def456def456def456def456def456def456abcd")
+
+      expect { record_tag }.to raise_error(/Command failed/)
+    end
+
+    it "re-raises when the tag is absent on origin (ls-remote empty)" do
+      # no stub_remote_peeled → ls-remote returns "" → nil
+      expect { record_tag }.to raise_error(/Command failed/)
+    end
+
+    it "tolerates a multi-line ls-remote (leading warning) when extracting the sha" do
+      stdout[["git", "ls-remote", "origin", "refs/tags/#{base}^{}"]] =
+        "warning: redirecting to https://...\n#{commit}\trefs/tags/#{base}^{}\n"
+
+      expect { record_tag }.not_to raise_error
+    end
+  end
+
+  describe "non-collision push failure (auth / network)" do
+    let(:push_fails) { true }
+    let(:push_output) { "fatal: unable to access 'https://...': Could not resolve host: github.com" }
+
+    it "re-raises even if the deployed artifact would have matched" do
+      # verify would say "matches", but the failure is NOT a tag collision
+      expect { record_tag(verify_deploy: -> { true }) }.to raise_error(/Command failed/)
+    end
+  end
+
+  # --- helpers ---
+
+  def allow_existing(*names)
+    stdout[["git", "tag"]] = names.join("\n")
+  end
+
+  def stub_remote_peeled(tag_name, sha)
+    stdout[["git", "ls-remote", "origin", "refs/tags/#{tag_name}^{}"]] = "#{sha}\trefs/tags/#{tag_name}^{}"
+  end
+end