Merge pull request #22301 from Homebrew/embed-executables-api-json

Embed executables in API JSON

Author: MikeMcQuaid

Library/Homebrew/api.rb

@@ -214,13 +214,13 @@ def self.fetch_api_files!
       end
     end
 
-    sig { void }
-    def self.write_names_and_aliases
+    sig { params(legacy_executables_fallback: T::Boolean).void }
+    def self.write_names_and_aliases(legacy_executables_fallback: false)
       if Homebrew::EnvConfig.use_internal_api?
-        Homebrew::API::Internal.write_formula_names_and_aliases
+        Homebrew::API::Internal.write_formula_names_and_aliases(legacy_executables_fallback:)
         Homebrew::API::Internal.write_cask_names
       else
-        Homebrew::API::Formula.write_names_and_aliases
+        Homebrew::API::Formula.write_names_and_aliases(legacy_executables_fallback:)
         Homebrew::API::Cask.write_names
       end
     end
@@ -252,6 +252,80 @@ def self.write_aliases_file!(aliases, type, regenerate:)
       false
     end
 
+    sig {
+      params(
+        formulae:                    T::Hash[String, T::Hash[String, T.untyped]],
+        regenerate:                  T::Boolean,
+        legacy_executables_fallback: T::Boolean,
+      ).returns(T::Boolean)
+    }
+    def self.write_executables_file!(formulae, regenerate:, legacy_executables_fallback: false)
+      executables_path = HOMEBREW_CACHE_API/"internal/executables.txt"
+      executables_lines = formulae.filter_map do |name, hash|
+        executables = T.cast(hash["executables"], T.nilable(T::Array[String]))
+        next if executables.blank?
+
+        "#{name}:#{executables.join(" ")}"
+      end
+      if executables_lines.empty?
+        return false if executables_path.exist? && !executables_path.empty?
+        return false unless legacy_executables_fallback
+
+        # TODO: Remove this fallback once formula JSON always includes executables.
+        return download_executables_file_from_github_packages!(executables_path)
+      end
+
+      contents = "#{executables_lines.sort.join("\n")}\n"
+      if !executables_path.exist? || regenerate || executables_path.read != contents
+        executables_path.dirname.mkpath
+        executables_path.unlink if executables_path.exist?
+        executables_path.write(contents)
+        return true
+      end
+
+      false
+    end
+
+    sig { params(target: Pathname).returns(T::Boolean) }
+    def self.download_executables_file_from_github_packages!(target)
+      github_packages_url = "https://ghcr.io/v2/homebrew/command-not-found/executables"
+      manifest_args = [
+        "--fail", "--location",
+        "--header", "Accept: application/vnd.oci.image.manifest.v1+json",
+        "#{github_packages_url}/manifests/latest"
+      ]
+      if HOMEBREW_GITHUB_PACKAGES_AUTH.present?
+        manifest_args.insert(-2, "--header", "Authorization: #{HOMEBREW_GITHUB_PACKAGES_AUTH}")
+      end
+
+      manifest_output = Utils::Curl.curl_output(*manifest_args, show_error: false)
+      return false unless manifest_output.success?
+
+      manifest = JSON.parse(manifest_output.stdout)
+      layers = T.cast(manifest.fetch("layers"), T::Array[T::Hash[String, T.untyped]])
+      layer = layers.find do |candidate|
+        candidate.dig("annotations", "org.opencontainers.image.title") == target.basename.to_s
+      end
+      return false if layer.nil?
+
+      digest = T.cast(layer["digest"], T.nilable(String))
+      return false if digest.blank?
+
+      download_args = ["--fail"]
+      if HOMEBREW_GITHUB_PACKAGES_AUTH.present?
+        download_args += ["--header", "Authorization: #{HOMEBREW_GITHUB_PACKAGES_AUTH}"]
+      end
+      download_args << "#{github_packages_url}/blobs/#{digest}"
+      target.dirname.mkpath
+      Utils::Curl.curl_download(*download_args, to: target, show_error: false)
+      FileUtils.touch(target)
+      true
+    rescue ErrorDuringExecution, JSON::ParserError, KeyError, TypeError
+      target.unlink if target.exist? && target.empty?
+
+      false
+    end
+
     sig {
       params(json_data: T::Hash[String, T.untyped])
         .returns([T::Boolean, T.any(String, T::Array[T.untyped], T::Hash[String, T.untyped])])

Library/Homebrew/api/formula.rb

@@ -201,12 +201,13 @@ def self.tap_migrations
         cache.fetch("tap_migrations")
       end
 
-      sig { params(regenerate: T::Boolean).void }
-      def self.write_names_and_aliases(regenerate: false)
+      sig { params(regenerate: T::Boolean, legacy_executables_fallback: T::Boolean).void }
+      def self.write_names_and_aliases(regenerate: false, legacy_executables_fallback: false)
         download_and_cache_data! unless cache.key?("formulae")
 
         Homebrew::API.write_names_file!(all_formulae.keys, "formula", regenerate:)
         Homebrew::API.write_aliases_file!(all_aliases, "formula", regenerate:)
+        Homebrew::API.write_executables_file!(all_formulae, regenerate:, legacy_executables_fallback:)
       end
     end
   end

Library/Homebrew/api/formula_struct.rb

@@ -93,6 +93,7 @@ def self.from_hash(formula_hash)
       const :deprecate_args, T::Hash[Symbol, T.nilable(T.any(String, Symbol))], default: {}
       const :desc, String
       const :disable_args, T::Hash[Symbol, T.nilable(T.any(String, Symbol))], default: {}
+      const :executables, T::Array[String], default: []
       const :head_dependencies, T::Array[DependsOnArgs], default: []
       const :head_url_args, [String, T::Hash[Symbol, T.anything]], default: ["", {}]
       const :head_uses_from_macos, T::Array[UsesFromMacOSArgs], default: []

Library/Homebrew/api/internal.rb

@@ -90,12 +90,13 @@ def self.download_and_cache_data!
       end
       private_class_method :download_and_cache_data!
 
-      sig { params(regenerate: T::Boolean).void }
-      def self.write_formula_names_and_aliases(regenerate: false)
+      sig { params(regenerate: T::Boolean, legacy_executables_fallback: T::Boolean).void }
+      def self.write_formula_names_and_aliases(regenerate: false, legacy_executables_fallback: false)
         download_and_cache_data! unless cache.key?("formula_hashes")
 
         Homebrew::API.write_names_file!(formula_hashes.keys, "formula", regenerate:)
         Homebrew::API.write_aliases_file!(formula_aliases, "formula", regenerate:)
+        Homebrew::API.write_executables_file!(formula_hashes, regenerate:, legacy_executables_fallback:)
       end
 
       sig { params(regenerate: T::Boolean).void }

Library/Homebrew/cmd/exec.rb

@@ -11,7 +11,7 @@ class Exec < AbstractCommand
 
       cmd_args do
         usage_banner <<~EOS
-          `exec`, `x` [`--skip-update`] [`--formulae=`<formulae>] [`--`] <command> [<args> ...]
+          `exec`, `x` [`--formulae=`<formulae>] [`--`] <command> [<args> ...]
 
           Run <command> in an environment populated by Homebrew formulae.
 
@@ -29,8 +29,6 @@ class Exec < AbstractCommand
           `#!/usr/bin/env -S brew exec --formulae=jq,yq --`
         EOS
 
-        switch "--skip-update",
-               description: "Skip updating the executables database if any version exists on disk, no matter how old."
         comma_array "--formulae",
                     description: "Comma-separated formulae to install and add to `PATH` before running " \
                                  "<command>."

Library/Homebrew/cmd/exec.sh

@@ -84,10 +84,6 @@ homebrew-exec() {
   while [[ "$#" -gt 0 ]]
   do
     case "$1" in
-      --skip-update)
-        HOMEBREW_SKIP_UPDATE=1
-        shift
-        ;;
       --formulae=*)
         formulae_arg="${1#--formulae=}"
         formulae_seen=1
@@ -148,7 +144,7 @@ homebrew-exec() {
     [[ "${executable}" != */* ]] || odie "Executable name must not contain path separators without \`--formulae\`."
 
     provider_lookup=1
-    download_and_cache_executables_file >&2
+    ensure_executables_file >&2
 
     local -a matching_formulae=()
     local selected_formula formula

Library/Homebrew/cmd/update-report.rb

@@ -101,7 +101,9 @@ def output_update_report
         end
 
         # Check if we can parse the JSON and do any Ruby-side follow-up.
-        Homebrew::API.write_names_and_aliases unless Homebrew::EnvConfig.no_install_from_api?
+        unless Homebrew::EnvConfig.no_install_from_api?
+          Homebrew::API.write_names_and_aliases(legacy_executables_fallback: true)
+        end
 
         Homebrew.failed = true if ENV["HOMEBREW_UPDATE_FAILED"]
         return if Homebrew::EnvConfig.disable_load_formula?

Library/Homebrew/cmd/update.sh

@@ -11,7 +11,7 @@
 # HOMEBREW_FORCE_BREWED_CURL, HOMEBREW_FORCE_BREWED_GIT,
 # HOMEBREW_SYSTEM_CURL_TOO_OLD, HOMEBREW_USER_AGENT_CURL are set by brew.sh
 # shellcheck disable=SC2154
-source "${HOMEBREW_LIBRARY}/Homebrew/utils/executables.sh"
+source "${HOMEBREW_LIBRARY}/Homebrew/utils/api.sh"
 source "${HOMEBREW_LIBRARY}/Homebrew/utils/lock.sh"
 
 macos_version_name() {
@@ -428,8 +428,6 @@ fetch_api_file() {
     do
       time_cond+=("${arg}")
     done < <(api_time_cond_args "${cache_path}")
-    # Keep curl request handling in sync with `download_and_cache_executables_file`
-    # in Library/Homebrew/utils/executables.sh.
     curl \
       "${CURL_DISABLE_CURLRC_ARGS[@]}" \
       --fail --compressed --silent \
@@ -685,12 +683,6 @@ EOS
 
   safe_cd "${HOMEBREW_REPOSITORY}"
 
-  # This means the user has run `brew which-formula` before and we should fetch executables.txt
-  if [[ "$(git config get --type=bool homebrew.commandnotfound 2>/dev/null)" == "true" ]]
-  then
-    export HOMEBREW_FETCH_EXECUTABLES_TXT=1
-  fi
-
   # if an older system had a newer curl installed, change each repo's remote URL from git to https
   if [[ -n "${HOMEBREW_SYSTEM_CURL_TOO_OLD}" && -x "${HOMEBREW_PREFIX}/opt/curl/bin/curl" ]] &&
      [[ "$(git config remote.origin.url)" =~ ^git:// ]]
@@ -1032,13 +1024,6 @@ EOS
     fi
   fi
 
-  # Update executables.txt if the user has ever run `brew which-formula` before.
-  if [[ -n "${HOMEBREW_FETCH_EXECUTABLES_TXT}" ]]
-  then
-    ohai "Downloading executables.txt"
-    fetch_api_file "${HOMEBREW_EXECUTABLES_TXT_ENDPOINT}" "${update_failed_file}"
-  fi
-
   if [[ -f "${update_failed_file}" ]]
   then
     onoe <"${update_failed_file}"

Library/Homebrew/cmd/which-formula.rb

@@ -22,8 +22,6 @@ class WhichFormula < AbstractCommand
         EOS
         switch "--explain",
                description: "Output explanation of how to get <command> by installing one of the providing formulae."
-        switch "--skip-update",
-               description: "Skip updating the executables database if any version exists on disk, no matter how old."
         named_args :command, min: 1
       end
     end

Library/Homebrew/cmd/which-formula.sh

@@ -13,10 +13,6 @@ homebrew-which-formula() {
         HOMEBREW_EXPLAIN=1
         shift
         ;;
-      --skip-update)
-        HOMEBREW_SKIP_UPDATE=1
-        shift
-        ;;
       --*)
         echo "Unknown option: $1" >&2
         brew help which-formula
@@ -37,7 +33,7 @@ homebrew-which-formula() {
 
   for cmd in "${args[@]}"
   do
-    download_and_cache_executables_file
+    ensure_executables_file
 
     local formulae=()
     local formula