Optionally use DownloadQueue for brew install

Allowing using `HOMEBREW_DOWNLOAD_CONCURRENCY` to use the
`DownloadQueue` for `brew install` by downloading and extracting
bottles in parallel.

This requires some fixes in e.g. `Dependency` and `FormulaInstaller`
to be able to front-load all downloads and handle parallelisation of
bottle pouring.

Behaviour without `HOMEBREW_DOWNLOAD_CONCURRENCY` set should be
unchanged.

Attestations are not handled for now and the UI should be improved
before we roll this out to users.

Post-install upgrades are not yet parallelised.

Co-authored-by: Carlo Cabrera <github@carlo.cab>

Author: MikeMcQuaid

Library/Homebrew/dependency.rb

@@ -51,13 +51,13 @@ def installed?(minimum_version: nil, minimum_revision: nil)
     end
     return false unless formula
 
+    # If the opt prefix doesn't exist: we likely have an incomplete installation.
+    return false unless formula.opt_prefix.exist?
+
     return true if formula.latest_version_installed?
 
     return false if minimum_version.blank?
 
-    # If the opt prefix doesn't exist: we likely have an incomplete installation.
-    return false unless formula.opt_prefix.exist?
-
     installed_keg = formula.any_installed_keg
     return false unless installed_keg
 

Library/Homebrew/download_queue.rb

@@ -8,12 +8,13 @@
 
 module Homebrew
   class DownloadQueue
-    sig { params(retries: Integer, force: T::Boolean).void }
-    def initialize(retries: 0, force: false)
+    sig { params(retries: Integer, force: T::Boolean, pour: T::Boolean).void }
+    def initialize(retries: 0, force: false, pour: false)
       @concurrency = T.let(EnvConfig.download_concurrency, Integer)
       @quiet = T.let(@concurrency > 1, T::Boolean)
       @tries = T.let(retries + 1, Integer)
       @force = force
+      @pour = pour
       @pool = T.let(Concurrent::FixedThreadPool.new(concurrency), Concurrent::FixedThreadPool)
     end
 
@@ -24,6 +25,10 @@ def enqueue(downloadable)
       ) do |download, force, quiet|
         download.clear_cache if force
         download.fetch(quiet:)
+        if pour && download.bottle?
+          UnpackStrategy.detect(download.cached_download, prioritize_extension: true)
+                        .extract_nestedly(to: HOMEBREW_CELLAR)
+        end
       end
     end
 
@@ -168,6 +173,9 @@ def cancel
     sig { returns(T::Boolean) }
     attr_reader :quiet
 
+    sig { returns(T::Boolean) }
+    attr_reader :pour
+
     sig { returns(T::Hash[Downloadable, Concurrent::Promises::Future]) }
     def downloads
       @downloads ||= T.let({}, T.nilable(T::Hash[Downloadable, Concurrent::Promises::Future]))

Library/Homebrew/download_strategy.rb

@@ -30,19 +30,6 @@ class AbstractDownloadStrategy
 
   abstract!
 
-  # Extension for bottle downloads.
-  module Pourable
-    extend T::Helpers
-
-    requires_ancestor { AbstractDownloadStrategy }
-
-    sig { params(block: T.nilable(T.proc.params(arg0: String).returns(T.anything))).returns(T.nilable(T.anything)) }
-    def stage(&block)
-      ohai "Pouring #{basename}"
-      super
-    end
-  end
-
   # The download URL.
   #
   # @api public
@@ -74,7 +61,6 @@ def initialize(url, name, version, **meta)
     @cache = T.let(meta.fetch(:cache, HOMEBREW_CACHE), Pathname)
     @meta = T.let(meta, T::Hash[Symbol, T.untyped])
     @quiet = T.let(false, T.nilable(T::Boolean))
-    extend Pourable if meta[:bottle]
   end
 
   # Download and cache the resource at {#cached_location}.
@@ -826,7 +812,6 @@ class LocalBottleDownloadStrategy < AbstractFileDownloadStrategy
   sig { params(path: Pathname).void }
   def initialize(path)
     @cached_location = T.let(path, Pathname)
-    extend Pourable
   end
   # rubocop:enable Lint/MissingSuper
 

Library/Homebrew/formula_installer.rb

@@ -44,6 +44,9 @@ class FormulaInstaller
   sig { returns(T::Boolean) }
   attr_accessor :link_keg
 
+  sig { returns(T.nilable(Homebrew::DownloadQueue)) }
+  attr_accessor :download_queue
+
   sig {
     params(
       formula:                    Formula,
@@ -136,9 +139,12 @@ def initialize(
     @hold_locks = T.let(false, T::Boolean)
     @show_summary_heading = T.let(false, T::Boolean)
     @etc_var_preinstall = T.let([], T::Array[Pathname])
+    @download_queue = T.let(nil, T.nilable(Homebrew::DownloadQueue))
 
     # Take the original formula instance, which might have been swapped from an API instance to a source instance
     @formula = T.let(T.must(previously_fetched_formula), Formula) if previously_fetched_formula
+
+    @ran_prelude_fetch = T.let(false, T::Boolean)
   end
 
   sig { returns(T::Boolean) }
@@ -294,7 +300,7 @@ def install_bottle_for?(dep, build)
   end
 
   sig { void }
-  def prelude
+  def prelude_fetch
     deprecate_disable_type = DeprecateDisable.type(formula)
     if deprecate_disable_type.present?
       message = "#{formula.full_name} has been #{DeprecateDisable.message(formula)}"
@@ -312,8 +318,24 @@ def prelude
       end
     end
 
+    # Needs to be done before expand_dependencies for compute_dependencies
+    fetch_bottle_tab if pour_bottle?
+
+    @ran_prelude_fetch = true
+  end
+
+  sig { void }
+  def prelude
+    prelude_fetch unless @ran_prelude_fetch
+
     Tab.clear_cache
 
+    # Setup bottle_tab_runtime_dependencies for compute_dependencies
+    @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes
+                                              .fetch("runtime_dependencies", []).then { |deps| deps || [] }
+                                              .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep }
+                                              .freeze
+
     verify_deps_exist unless ignore_deps?
 
     forbidden_license_check
@@ -778,13 +800,15 @@ def install_dependencies(deps)
     if deps.empty? && only_deps?
       puts "All dependencies for #{formula.full_name} are satisfied."
     elsif !deps.empty?
-      oh1 "Installing dependencies for #{formula.full_name}: " \
-          "#{deps.map(&:first).map { Formatter.identifier(_1) }.to_sentence}",
-          truncate: false
+      if deps.length > 1
+        oh1 "Installing dependencies for #{formula.full_name}: " \
+            "#{deps.map(&:first).map { Formatter.identifier(_1) }.to_sentence}",
+            truncate: false
+      end
       deps.each { |dep, options| install_dependency(dep, options) }
     end
 
-    @show_header = true unless deps.empty?
+    @show_header = true if deps.length > 1
   end
 
   sig { params(dep: Dependency).void }
@@ -808,6 +832,7 @@ def fetch_dependency(dep)
       quiet:                      quiet?,
       verbose:                    verbose?,
     )
+    fi.download_queue = download_queue
     fi.prelude
     fi.fetch
   end
@@ -830,7 +855,7 @@ def install_dependency(dep, inherited_options)
       installed_keg = Keg.new(df.prefix)
       tab ||= installed_keg.tab
       tmp_keg = Pathname.new("#{installed_keg}.tmp")
-      installed_keg.rename(tmp_keg)
+      installed_keg.rename(tmp_keg) unless tmp_keg.directory?
     end
 
     if df.tap.present? && tab.present? && (tab_tap = tab.source["tap"].presence) &&
@@ -867,6 +892,7 @@ def install_dependency(dep, inherited_options)
       verbose:                    verbose?,
     )
     oh1 "Installing #{formula.full_name} dependency: #{Formatter.identifier(dep.name)}"
+    fi.prelude
     fi.install
     fi.finish
   # Handle all possible exceptions installing deps.
@@ -1337,9 +1363,13 @@ def fetch_dependencies
 
     return if deps.empty?
 
-    oh1 "Fetching dependencies for #{formula.full_name}: " \
-        "#{deps.map(&:first).map { Formatter.identifier(_1) }.to_sentence}",
-        truncate: false
+    unless download_queue
+      dependencies_string = deps.map(&:first)
+                                .map { Formatter.identifier(_1) }
+                                .to_sentence
+      oh1 "Fetching dependencies for #{formula.full_name}: #{dependencies_string}",
+          truncate: false
+    end
 
     deps.each { |(dep, _options)| fetch_dependency(dep) }
   end
@@ -1360,15 +1390,18 @@ def previously_fetched_formula
   def fetch_bottle_tab(quiet: false)
     return if @fetch_bottle_tab
 
-    begin
-      formula.fetch_bottle_tab(quiet: quiet)
-      @bottle_tab_runtime_dependencies = formula.bottle_tab_attributes
-                                                .fetch("runtime_dependencies", []).then { |deps| deps || [] }
-                                                .each_with_object({}) { |dep, h| h[dep["full_name"]] = dep }
-                                                .freeze
-    rescue DownloadError, Resource::BottleManifest::Error
-      # do nothing
+    if (download_queue = self.download_queue) &&
+       (bottle = formula.bottle) &&
+       (manifest_resource = bottle.github_packages_manifest_resource)
+      download_queue.enqueue(manifest_resource)
+    else
+      begin
+        formula.fetch_bottle_tab(quiet: quiet)
+      rescue DownloadError, Resource::BottleManifest::Error
+        # do nothing
+      end
     end
+
     @fetch_bottle_tab = T.let(true, T.nilable(TrueClass))
   end
 
@@ -1381,7 +1414,7 @@ def fetch
     return if only_deps?
     return if formula.local_bottle_path.present?
 
-    oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip
+    oh1 "Fetching #{Formatter.identifier(formula.full_name)}".strip unless download_queue
 
     downloadable_object = downloadable
     check_attestation = if pour_bottle?(output_warning: true)
@@ -1391,19 +1424,31 @@ def fetch
     else
       @formula = Homebrew::API::Formula.source_download(formula) if formula.loaded_from_api?
 
-      formula.fetch_patches
-      formula.resources.each(&:fetch)
+      if (download_queue = self.download_queue)
+        formula.enqueue_resources_and_patches(download_queue:)
+      else
+        formula.fetch_patches
+        formula.resources.each(&:fetch)
+      end
+
       downloadable_object = downloadable
 
       false
     end
-    downloadable_object.fetch
+
+    if (download_queue = self.download_queue)
+      download_queue.enqueue(downloadable_object)
+    else
+      downloadable_object.fetch
+    end
 
     # We skip `gh` to avoid a bootstrapping cycle, in the off-chance a user attempts
     # to explicitly `brew install gh` without already having a version for bootstrapping.
     # We also skip bottle installs from local bottle paths, as these are done in CI
     # as part of the build lifecycle before attestations are produced.
     if check_attestation &&
+       # TODO: support this for download queues at some point
+       download_queue.nil? &&
        Homebrew::Attestation.enabled? &&
        formula.tap&.core_tap? &&
        formula.name != "gh"
@@ -1489,7 +1534,10 @@ def downloadable
   sig { void }
   def pour
     HOMEBREW_CELLAR.cd do
-      downloadable.downloader.stage
+      # download queue has already done the actual staging but we'll lie about
+      # pouring now for nicer output
+      ohai "Pouring #{downloadable.downloader.basename}"
+      downloadable.downloader.stage unless download_queue
     end
 
     Tab.clear_cache

Library/Homebrew/install.rb

@@ -6,6 +6,7 @@
 require "hardware"
 require "development_tools"
 require "upgrade"
+require "download_queue"
 
 module Homebrew
   # Helper module for performing (pre-)install checks.
@@ -313,30 +314,43 @@ def install_formulae(
         skip_post_install: false,
         skip_link: false
       )
-        unless dry_run
+        formulae_names_to_install = formula_installers.map { |fi| fi.formula.name }
+        return if formulae_names_to_install.empty?
+
+        if dry_run
+          ohai "Would install #{Utils.pluralize("formula", formulae_names_to_install.count,
+                                                plural: "e", include_count: true)}:"
+          puts formulae_names_to_install.join(" ")
+
           formula_installers.each do |fi|
-            fi.prelude
-            fi.fetch
-          rescue CannotInstallFormulaError => e
-            ofail e.message
-            next
-          rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e
-            ofail "#{fi.formula}: #{e}"
-            next
+            print_dry_run_dependencies(fi.formula, fi.compute_dependencies, &:name)
           end
+          return
         end
 
-        if dry_run
-          if (formulae_name_to_install = formula_installers.map { |fi| fi.formula.name })
-            ohai "Would install #{Utils.pluralize("formula", formulae_name_to_install.count,
-                                                  plural: "e", include_count: true)}:"
-            puts formulae_name_to_install.join(" ")
-
+        formula_sentence = formulae_names_to_install.map { |name| Formatter.identifier(name) }.to_sentence
+        oh1 "Fetching downloads for: #{formula_sentence}", truncate: false
+        if EnvConfig.download_concurrency > 1
+          download_queue = Homebrew::DownloadQueue.new(pour: true)
+          formula_installers.each do |fi|
+            fi.download_queue = download_queue
+          end
+        end
+        begin
+          [:prelude_fetch, :prelude, :fetch].each do |step|
             formula_installers.each do |fi|
-              print_dry_run_dependencies(fi.formula, fi.compute_dependencies, &:name)
+              fi.public_send(step)
+            rescue UnsatisfiedRequirements, DownloadError, ChecksumMismatchError => e
+              ofail "#{fi.formula}: #{e}"
+              next
             end
+            download_queue&.fetch
+          rescue CannotInstallFormulaError => e
+            ofail e.message
+            next
           end
-          return
+        ensure
+          download_queue&.shutdown
         end
 
         formula_installers.each do |fi|

Library/Homebrew/retryable_download.rb

@@ -89,6 +89,9 @@ def verify_download_integrity(filename) = downloadable.verify_download_integrity
     sig { override.returns(String) }
     def download_name = downloadable.download_name
 
+    sig { returns(T::Boolean) }
+    def bottle? = downloadable.is_a?(Bottle)
+
     private
 
     sig { returns(Downloadable) }

Library/Homebrew/test/cmd/deps_spec.rb

@@ -29,10 +29,13 @@
     setup_test_formula "recommended_test"
     setup_test_formula "installed"
 
-    # Mock `Formula#any_version_installed?` by creating the tab in a plausible keg directory
-    keg_dir = HOMEBREW_CELLAR/"installed"/"1.0"
+    # Mock `Formula#any_version_installed?` by creating the tab in a plausible keg directory and opt link
+    keg_dir = HOMEBREW_CELLAR/"installed/1.0"
     keg_dir.mkpath
     touch keg_dir/AbstractTab::FILENAME
+    opt_link = HOMEBREW_PREFIX/"opt/installed"
+    opt_link.parent.mkpath
+    FileUtils.ln_sf keg_dir, opt_link
 
     expect { brew "deps", "baz", "--include-test", "--missing", "--skip-recommended" }
       .to be_a_success