Stop counting stale ELF strings when bottling

Library/Homebrew/keg_relocate.rb

@@ -488,6 +488,137 @@ def self.text_matches_in_file(file, string, ignores, linked_libraries, formula_a
         end
       end
     end
+    without_stale_elf_matches(file, string, text_matches)
+  end
+
+  # Growing an ELF RPATH or interpreter moves the dynamic string table or the
+  # interpreter elsewhere in the file, which can leave the old prefix string
+  # behind as dead bytes that `strings`-based scanning still finds, wrongly
+  # pinning bottles whose live linkage is fully placeholdered. Map each match
+  # offset back to the ELF structures and drop matches the loader can no
+  # longer see: unreferenced strings in loader-owned string regions and bytes
+  # no section covers any more.
+  # This deliberately prefers assuming relocatability: a match the current
+  # ELF structures cannot account for is treated as dead even though the
+  # bytes are still in the file, because a wrongly pinned bottle forces
+  # source builds for every non-default-prefix user, while a wrongly
+  # dropped match surfaces as a per-formula bug report and fix.
+  sig {
+    params(file: Pathname, string: String,
+           text_matches: T::Array[[String, String]]).returns(T::Array[[String, String]])
+  }
+  def self.without_stale_elf_matches(file, string, text_matches)
+    return text_matches if text_matches.empty?
+
+    require "os/linux/elf"
+    return text_matches unless T.cast(Pathname.new(file.to_s).extend(ELFShim), ELFShim).elf?
+
+    require "elftools"
+
+    stream = file.open("rb")
+    begin
+      elf = ELFTools::ELFFile.new(stream)
+      # A file may legally keep section headers while `e_shstrndx` is
+      # `SHN_UNDEF` (no section-name table); sections cannot be classified
+      # without names, so keep every match.
+      return text_matches unless elf.strtab_section.is_a?(ELFTools::Sections::StrTabSection)
+
+      section_ranges = elf.sections.filter_map do |section|
+        header = section.header
+        # SHT_NOBITS sections (e.g. `.bss`) occupy no file bytes.
+        next if header.sh_type.to_i == ELFTools::Constants::SHT::SHT_NOBITS
+        next if header.sh_size.to_i.zero?
+
+        [section.name, header.sh_offset.to_i...(header.sh_offset.to_i + header.sh_size.to_i)]
+      end
+      # Without a section header table nothing can be classified as dead.
+      return text_matches if section_ranges.empty?
+
+      interp_range = if (interp = elf.segment_by_type(:interp))
+        interp.header.p_offset.to_i...(interp.header.p_offset.to_i + interp.header.p_filesz.to_i)
+      end
+
+      live_string_offsets = []
+      string_table_range = T.let(nil, T.nilable(T::Range[Integer]))
+      if (dynamic = elf.segment_by_type(:dynamic))
+        # Dynamic tags whose value is an offset into the dynamic string
+        # table, i.e. the strings the loader can actually see.
+        string_tags = [
+          ELFTools::Constants::DT::DT_NEEDED,
+          ELFTools::Constants::DT::DT_SONAME,
+          ELFTools::Constants::DT::DT_RPATH,
+          ELFTools::Constants::DT::DT_RUNPATH,
+          ELFTools::Constants::DT::DT_AUXILIARY,
+          # elftools 1.3.1 mislabels `DT_USED` (0x7ffffffe) as `DT_FILTER`;
+          # both hold string-table offsets, so keep the mislabelled value
+          # and add the ELF ABI's real `DT_FILTER`.
+          ELFTools::Constants::DT::DT_FILTER,
+          0x7fffffff, # DT_FILTER
+          ELFTools::Constants::DT::DT_AUDIT,
+          ELFTools::Constants::DT::DT_DEPAUDIT,
+          ELFTools::Constants::DT::DT_CONFIG,
+        ]
+        string_table_vaddr = T.let(nil, T.nilable(Integer))
+        string_table_size = T.let(nil, T.nilable(Integer))
+        string_offsets = []
+        dynamic.tags.each do |tag|
+          case tag.header.d_tag.to_i
+          when ELFTools::Constants::DT::DT_STRTAB then string_table_vaddr = tag.header.d_val.to_i
+          when ELFTools::Constants::DT::DT_STRSZ then string_table_size = tag.header.d_val.to_i
+          when *string_tags then string_offsets << tag.header.d_val.to_i
+          end
+        end
+        if string_table_vaddr && (string_table_offset = elf.offset_from_vma(string_table_vaddr))
+          string_table_range = string_table_offset...(string_table_offset + string_table_size) if string_table_size
+
+          # Dynamic symbol names are loader-visible strings in the same
+          # table, referenced by `.dynsym` `st_name` rather than by dynamic
+          # tags. Version table strings also index the table but hold
+          # version names, never paths, so they are not collected.
+          elf.sections_by_type(ELFTools::Constants::SHT::SHT_DYNSYM).each do |section|
+            section.symbols.each do |symbol|
+              name_offset = symbol.header.st_name.to_i
+              string_offsets << name_offset unless name_offset.zero?
+            end
+          end
+
+          live_string_offsets = string_offsets.map { |offset| string_table_offset + offset }
+        end
+      end
+
+      # The current `.dynstr` and `.interp` sections plus the `DT_STRTAB`
+      # table cover both the live loader string regions and any abandoned
+      # copies whose section headers were left behind.
+      loader_ranges = section_ranges.filter_map { |name, range| range if [".dynstr", ".interp"].include?(name) }
+      loader_ranges << string_table_range if string_table_range
+
+      text_matches.select do |match, offset|
+        match_start = offset.to_i(16)
+        match_range = match_start...(match_start + match.bytesize)
+
+        # The interpreter the loader actually uses is live.
+        next true if interp_range&.cover?(match_start)
+
+        # A referenced string reaches from its offset to the end of the
+        # printable run, so the prefix is live only when some reference
+        # starts at or before a prefix occurrence: a suffix-merged
+        # reference to the interior `libfoo.so` of `/old/prefix/libfoo.so`
+        # leaves the prefix bytes before it dead.
+        last_prefix_position = match_start + (match.b.rindex(string) || 0)
+        next true if live_string_offsets.any? { |live| match_range.cover?(live) && live <= last_prefix_position }
+        # Anything else inside a loader-owned string region is a dead copy.
+        next false if loader_ranges.any? { |range| range.cover?(match_start) }
+
+        # Strings in ordinary sections are compiled-in content; bytes outside
+        # every section are dead copies left behind by a moved section.
+        section_ranges.any? { |_name, range| range.cover?(match_start) }
+      end
+    ensure
+      stream.close
+    end
+  rescue ELFTools::ELFError, IOError
+    # A file that is not valid ELF, or whose program, section or dynamic
+    # tables are truncated mid-parse, keeps all its matches.
     text_matches
   end
 

Library/Homebrew/plans/relocatable-bottles.md

@@ -172,10 +172,6 @@ length limit and no new machinery.
 4. Reconcile checker divergences: pull `brew bottle --verbose` CI logs for
    the `abseil` class, fix whatever diverges, then batch re-mark provably
    clean pins `cellar :any` with no rebuild (sha256 unchanged).
-5. ELF-aware checker: map string offsets to sections (elftools is already
-   vendored via patchelf.rb) and stop counting stale `.dynstr` corpses.
-   Flips the glib/gobject-introspection cluster and much of the Linux-only
-   pinned set on their next rebottle.
 6. node-gyp debris: delete `build/**/obj.target`, `*.o` and `*.d` from
    npm-installed trees and strip or debug-prefix-map compiled `.node`
    addons. Flips the npm cluster.

Library/Homebrew/test/keg_relocate/elf_checker_spec.rb

@@ -0,0 +1,136 @@
+# typed: true
+# frozen_string_literal: true
+
+require "keg_relocate"
+require "patchelf"
+require "elftools"
+
+RSpec.describe Keg do
+  let(:dir) { HOMEBREW_CELLAR/"foo/1.0.0" }
+  let(:file) { dir/"bin/program" }
+  let(:baked_rpath) { "#{dir}/lib/#{"deep/" * 10}end" }
+
+  def patch_rpath(rpath)
+    patcher = PatchELF::Patcher.new(file.to_s, on_error: :silent)
+    patcher.rpath = rpath
+    patcher.save(patchelf_compatible: true)
+  end
+
+  def patch_interpreter(interpreter)
+    patcher = PatchELF::Patcher.new(file.to_s, on_error: :silent)
+    patcher.interpreter = interpreter
+    patcher.save(patchelf_compatible: true)
+  end
+
+  before do
+    Pathname(baked_rpath).mkpath
+    file.dirname.mkpath
+    FileUtils.cp TEST_FIXTURE_DIR/"elf/hello", file
+    patch_rpath baked_rpath
+  end
+
+  describe "::text_matches_in_file" do
+    it "keeps prefix strings the dynamic loader still references" do
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil).size).to eq 1
+    end
+
+    it "drops dead loader strings nothing references any more" do
+      # Shrinking the RPATH leaves an in-place `X` run inside `.dynstr`;
+      # writing a prefix string over it recreates the dead bytes a moved or
+      # rewritten string table leaves behind.
+      patch_rpath "#{Keg::PREFIX_PLACEHOLDER}/lib"
+      corpse_offset = File.binread(file).index("X" * (dir.to_s.length + 2))
+      raise "no X padding run found" if corpse_offset.nil?
+
+      File.open(file, "r+b") do |f|
+        f.seek(corpse_offset)
+        f.write("#{dir}\x00")
+      end
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil)).to be_empty
+    end
+
+    it "drops prefix strings outside every section" do
+      patch_rpath "#{Keg::PREFIX_PLACEHOLDER}/lib"
+      file.open("ab") { |f| f.write("\x00#{dir}\x00") }
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil)).to be_empty
+    end
+
+    it "drops prefixes before the substring a dynamic tag references" do
+      # Point DT_RPATH into the interior of the baked string, as a linker
+      # does for tail-merged entries: the prefix bytes before the
+      # referenced substring are then dead.
+      value_offset = file.open("rb") do |stream|
+        dynamic = ELFTools::ELFFile.new(stream).segment_by_type(:dynamic)
+        index = dynamic.tags.find_index do |tag|
+          [ELFTools::Constants::DT::DT_RPATH, ELFTools::Constants::DT::DT_RUNPATH].include?(tag.header.d_tag.to_i)
+        end
+        dynamic.header.p_offset.to_i + (index * 16) + 8
+      end
+      referenced = File.binread(file, 8, value_offset).unpack1("Q<")
+      File.open(file, "r+b") do |f|
+        f.seek(value_offset)
+        f.write([referenced + dir.to_s.length + 1].pack("Q<"))
+      end
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil)).to be_empty
+    end
+
+    it "keeps matches when the section-name table is unavailable" do
+      # `e_shstrndx` may legally be `SHN_UNDEF`, leaving sections unnameable.
+      File.open(file, "r+b") do |f|
+        f.seek(62)
+        f.write("\x00\x00")
+      end
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil).size).to eq 1
+    end
+
+    it "keeps the interpreter the loader still uses" do
+      interpreter = "#{dir}/ld.so"
+      FileUtils.touch interpreter
+      patch_rpath "#{Keg::PREFIX_PLACEHOLDER}/lib"
+      patch_interpreter interpreter
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil).size).to eq 1
+    end
+
+    it "drops old interpreter bytes the loader no longer uses" do
+      patch_rpath "#{Keg::PREFIX_PLACEHOLDER}/lib"
+      # Growing the interpreter moves it; shrinking it back leaves an
+      # in-place padding run where dead interpreter bytes can survive.
+      patch_interpreter "#{dir}/#{"deep/" * 10}ld.so"
+      patch_interpreter "#{Keg::PREFIX_PLACEHOLDER}/lib/ld.so"
+      anchor = "#{Keg::PREFIX_PLACEHOLDER}/lib/ld.so\x00"
+      corpse_offset = File.binread(file).index(anchor)
+      raise "no placeholdered interpreter found" if corpse_offset.nil?
+
+      File.open(file, "r+b") do |f|
+        f.seek(corpse_offset + anchor.bytesize)
+        f.write("#{dir}\x00")
+      end
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil)).to be_empty
+    end
+
+    it "keeps matches in files whose ELF tables are truncated" do
+      file.binwrite "\x7fELF\x02\x01\x01#{"\x00" * 9}\x00#{dir}\x00"
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil).size).to eq 1
+    end
+
+    it "keeps prefix strings compiled into ordinary sections" do
+      patch_rpath "#{Keg::PREFIX_PLACEHOLDER}/lib"
+      text_offset = file.open("rb") do |stream|
+        ELFTools::ELFFile.new(stream).section_by_name(".text").header.sh_offset.to_i
+      end
+      File.open(file, "r+b") do |f|
+        f.seek(text_offset)
+        f.write("\x00#{dir}\x00")
+      end
+
+      expect(described_class.text_matches_in_file(file, dir.to_s, [], [], nil).size).to eq 1
+    end
+  end
+end