Merge pull request #16638 from apainintheneck/load-internal-json-v3

Load internal json v3

Library/Homebrew/api.rb

@@ -184,6 +184,11 @@ def self.tap_from_source_download(path)
 
       Tap.fetch(org, repo)
     end
+
+    sig { returns(T::Boolean) }
+    def self.internal_json_v3?
+      ENV["HOMEBREW_INTERNAL_JSON_V3"].present?
+    end
   end
 
   # @api private

Library/Homebrew/api/formula.rb

@@ -39,19 +39,24 @@ def self.source_download(formula)
 
       sig { returns(T::Boolean) }
       def self.download_and_cache_data!
-        json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.jws.json"
-
-        cache["aliases"] = {}
-        cache["renames"] = {}
-        cache["formulae"] = json_formulae.to_h do |json_formula|
-          json_formula["aliases"].each do |alias_name|
-            cache["aliases"][alias_name] = json_formula["name"]
-          end
-          (json_formula["oldnames"] || [json_formula["oldname"]].compact).each do |oldname|
-            cache["renames"][oldname] = json_formula["name"]
+        if Homebrew::API.internal_json_v3?
+          json_formulae, updated = Homebrew::API.fetch_json_api_file "internal/v3/homebrew-core.jws.json"
+          overwrite_cache! T.cast(json_formulae, T::Hash[String, T.untyped])
+        else
+          json_formulae, updated = Homebrew::API.fetch_json_api_file "formula.jws.json"
+
+          cache["aliases"] = {}
+          cache["renames"] = {}
+          cache["formulae"] = json_formulae.to_h do |json_formula|
+            json_formula["aliases"].each do |alias_name|
+              cache["aliases"][alias_name] = json_formula["name"]
+            end
+            (json_formula["oldnames"] || [json_formula["oldname"]].compact).each do |oldname|
+              cache["renames"][oldname] = json_formula["name"]
+            end
+
+            [json_formula["name"], json_formula.except("name")]
           end
-
-          [json_formula["name"], json_formula.except("name")]
         end
 
         updated
@@ -88,6 +93,28 @@ def self.all_renames
         cache["renames"]
       end
 
+      sig { returns(Hash) }
+      def self.tap_migrations
+        # Not sure that we need to reload here.
+        unless cache.key?("tap_migrations")
+          json_updated = download_and_cache_data!
+          write_names_and_aliases(regenerate: json_updated)
+        end
+
+        cache["tap_migrations"]
+      end
+
+      sig { returns(String) }
+      def self.tap_git_head
+        # Note sure we need to reload here.
+        unless cache.key?("tap_git_head")
+          json_updated = download_and_cache_data!
+          write_names_and_aliases(regenerate: json_updated)
+        end
+
+        cache["tap_git_head"]
+      end
+
       sig { params(regenerate: T::Boolean).void }
       def self.write_names_and_aliases(regenerate: false)
         download_and_cache_data! unless cache.key?("formulae")

Library/Homebrew/dev-cmd/generate-formula-api.rb

@@ -58,14 +58,6 @@ def generate_formula_api
       Formulary.enable_factory_cache!
       Formula.generating_hash!
 
-      homebrew_core_tap_hash = {
-        "tap_git_head"   => tap.git_head,
-        "aliases"        => tap.alias_table,
-        "renames"        => tap.formula_renames,
-        "tap_migrations" => tap.tap_migrations,
-        "formulae"       => {},
-      }
-
       tap.formula_names.each do |name|
         formula = Formulary.factory(name)
         name = formula.name
@@ -77,15 +69,12 @@ def generate_formula_api
           File.write("api/formula/#{name}.json", FORMULA_JSON_TEMPLATE)
           File.write("formula/#{name}.html", html_template_name)
         end
-
-        homebrew_core_tap_hash["formulae"][formula.name] =
-          formula.to_hash_with_variations(hash_method: :to_api_hash)
       rescue
         onoe "Error while generating data for formula '#{name}'."
         raise
       end
 
-      homebrew_core_tap_json = JSON.generate(homebrew_core_tap_hash)
+      homebrew_core_tap_json = JSON.generate(tap.to_api_hash)
       File.write("api/internal/v3/homebrew-core.json", homebrew_core_tap_json) unless args.dry_run?
       canonical_json = JSON.pretty_generate(tap.formula_renames.merge(tap.alias_table))
       File.write("_data/formula_canonical.json", "#{canonical_json}\n") unless args.dry_run?

Library/Homebrew/extend/cachable.rb

@@ -11,4 +11,11 @@ def cache
   def clear_cache
     cache.clear
   end
+
+  private
+
+  sig { params(hash: T::Hash[T.untyped, T.untyped]).void }
+  def overwrite_cache!(hash)
+    @cache = hash
+  end
 end

Library/Homebrew/formulary.rb

@@ -251,8 +251,8 @@ def self.load_formula_from_api(name, flags:)
       desc json_formula["desc"]
       homepage json_formula["homepage"]
       license SPDX.string_to_license_expression(json_formula["license"])
-      revision json_formula["revision"]
-      version_scheme json_formula["version_scheme"]
+      revision json_formula.fetch("revision", 0)
+      version_scheme json_formula.fetch("version_scheme", 0)
 
       if (urls_stable = json_formula["urls"]["stable"].presence)
         stable do
@@ -262,7 +262,7 @@ def self.load_formula_from_api(name, flags:)
             using:    urls_stable["using"]&.to_sym,
           }.compact
           url urls_stable["url"], **url_spec
-          version json_formula["versions"]["stable"]
+          version Homebrew::API.internal_json_v3? ? json_formula["version"] : json_formula["versions"]["stable"]
           sha256 urls_stable["checksum"] if urls_stable["checksum"].present?
 
           instance_exec(:stable, &add_deps)
@@ -289,7 +289,13 @@ def self.load_formula_from_api(name, flags:)
         end
       end
 
-      if (bottles_stable = json_formula["bottle"]["stable"].presence)
+      bottles_stable = if Homebrew::API.internal_json_v3?
+        json_formula["bottle"]
+      else
+        json_formula["bottle"]["stable"]
+      end.presence
+
+      if bottles_stable
         bottle do
           if Homebrew::EnvConfig.bottle_domain == HOMEBREW_BOTTLE_DEFAULT_DOMAIN
             root_url HOMEBREW_BOTTLE_DEFAULT_DOMAIN
@@ -373,19 +379,26 @@ def caveats
             &.gsub(HOMEBREW_HOME_PLACEHOLDER, Dir.home)
       end
 
-      @tap_git_head_string = json_formula["tap_git_head"]
+      @tap_git_head_string = if Homebrew::API.internal_json_v3?
+        Homebrew::API::Formula.tap_git_head
+      else
+        json_formula["tap_git_head"]
+      end
+
       def tap_git_head
         self.class.instance_variable_get(:@tap_git_head_string)
       end
 
-      @oldnames_array = json_formula["oldnames"] || [json_formula["oldname"]].compact
-      def oldnames
-        self.class.instance_variable_get(:@oldnames_array)
-      end
+      unless Homebrew::API.internal_json_v3?
+        @oldnames_array = json_formula["oldnames"] || [json_formula["oldname"]].compact
+        def oldnames
+          self.class.instance_variable_get(:@oldnames_array)
+        end
 
-      @aliases_array = json_formula.fetch("aliases", [])
-      def aliases
-        self.class.instance_variable_get(:@aliases_array)
+        @aliases_array = json_formula.fetch("aliases", [])
+        def aliases
+          self.class.instance_variable_get(:@aliases_array)
+        end
       end
 
       @versioned_formulae_array = json_formula.fetch("versioned_formulae", [])

Library/Homebrew/tap.rb

@@ -126,6 +126,7 @@ def clear_cache
     @formula_dir = nil
     @cask_dir = nil
     @command_dir = nil
+    @formula_names = nil
     @formula_files = nil
     @cask_files = nil
     @alias_dir = nil
@@ -1101,6 +1102,8 @@ def tap_migrations
     @tap_migrations ||= if Homebrew::EnvConfig.no_install_from_api?
       ensure_installed!
       super
+    elsif Homebrew::API.internal_json_v3?
+      Homebrew::API::Formula.tap_migrations
     else
       migrations, = Homebrew::API.fetch_json_api_file "formula_tap_migrations.jws.json",
                                                       stale_seconds: TAP_MIGRATIONS_STALE_SECONDS
@@ -1188,6 +1191,23 @@ def formula_files_by_name
       hash[name] = Pathname(new_path) if existing_path.nil? || existing_path.to_s.length < new_path.length
     end
   end
+
+  sig { returns(T::Hash[String, T.untyped]) }
+  def to_api_hash
+    formulae_api_hash = formula_names.to_h do |name|
+      formula = Formulary.factory(name)
+      formula_hash = formula.to_hash_with_variations(hash_method: :to_api_hash)
+      [name, formula_hash]
+    end
+
+    {
+      "tap_git_head"   => git_head,
+      "aliases"        => alias_table,
+      "renames"        => formula_renames,
+      "tap_migrations" => tap_migrations,
+      "formulae"       => formulae_api_hash,
+    }
+  end
 end
 
 # A specialized {Tap} class for homebrew-cask.

Library/Homebrew/test/api/internal_tap_json/formula_spec.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+RSpec.describe "Internal Tap JSON -- Formula" do
+  let(:internal_tap_json) { File.read(TEST_FIXTURE_DIR/"internal_tap_json/homebrew-core.json").chomp }
+  let(:tap_git_head) { "9977471165641744a829d3e494fa563407503297" }
+
+  context "when generating JSON", :needs_macos do
+    before do
+      cp_r(TEST_FIXTURE_DIR/"internal_tap_json/homebrew-core", Tap::TAP_DIRECTORY/"homebrew")
+
+      # NOTE: Symlinks can't be copied recursively so we create them manually here.
+      (Tap::TAP_DIRECTORY/"homebrew/homebrew-core").tap do |core_tap|
+        mkdir(core_tap/"Aliases")
+        ln_s(core_tap/"Formula/f/fennel.rb", core_tap/"Aliases/fennel-lang")
+        ln_s(core_tap/"Formula/p/ponyc.rb", core_tap/"Aliases/ponyc-lang")
+      end
+    end
+
+    it "creates the expected hash" do
+      api_hash = CoreTap.instance.to_api_hash
+      api_hash["tap_git_head"] = tap_git_head # tricky to mock
+
+      expect(JSON.pretty_generate(api_hash)).to eq(internal_tap_json)
+    end
+  end
+
+  context "when loading JSON" do
+    before do
+      ENV["HOMEBREW_INTERNAL_JSON_V3"] = "1"
+      ENV.delete("HOMEBREW_NO_INSTALL_FROM_API")
+
+      allow(Homebrew::API).to receive(:fetch_json_api_file)
+        .with("internal/v3/homebrew-core.jws.json")
+        .and_return([JSON.parse(internal_tap_json), false])
+
+      # `Tap.reverse_tap_migrations_renames` looks for renames in every
+      # tap so `CoreCaskTap.tap_migrations` gets called and tries to
+      # fetch stuff from the API. This just avoids errors.
+      allow(Homebrew::API).to receive(:fetch_json_api_file)
+        .with("cask_tap_migrations.jws.json", anything)
+        .and_return([{}, false])
+
+      # To allow `formula_names.txt` to be written to the cache.
+      (HOMEBREW_CACHE/"api").mkdir
+
+      Homebrew::API::Formula.clear_cache
+    end
+
+    after do
+      Homebrew::API::Formula.clear_cache
+    end
+
+    it "loads tap aliases" do
+      expect(CoreTap.instance.alias_table).to eq({
+        "fennel-lang" => "fennel",
+        "ponyc-lang"  => "ponyc",
+      })
+    end
+
+    it "loads formula renames" do
+      expect(CoreTap.instance.formula_renames).to eq({
+        "advancemenu" => "advancemame",
+        "amtk"        => "libgedit-amtk",
+        "annie"       => "lux",
+        "antlr2"      => "antlr@2",
+        "romanesco"   => "fennel",
+      })
+    end
+
+    it "loads tap migrations" do
+      expect(CoreTap.instance.tap_migrations).to eq({
+        "adobe-air-sdk"          => "homebrew/cask",
+        "android-ndk"            => "homebrew/cask",
+        "android-platform-tools" => "homebrew/cask",
+        "android-sdk"            => "homebrew/cask",
+        "app-engine-go-32"       => "homebrew/cask/google-cloud-sdk",
+      })
+    end
+
+    it "loads tap git head" do
+      expect(Homebrew::API::Formula.tap_git_head)
+        .to eq(tap_git_head)
+    end
+
+    context "when loading formulae" do
+      let(:fennel_metadata) do
+        {
+          "dependencies"     => ["lua"],
+          "desc"             => "Lua Lisp Language",
+          "full_name"        => "fennel",
+          "homepage"         => "https://fennel-lang.org",
+          "license"          => "MIT",
+          "name"             => "fennel",
+          "ruby_source_path" => "Formula/f/fennel.rb",
+          "tap"              => "homebrew/core",
+          "tap_git_head"     => tap_git_head,
+          "versions"         => { "bottle"=>true, "head"=>nil, "stable"=>"1.4.0" },
+        }
+      end
+
+      let(:ponyc_metadata) do
+        {
+          "desc"                   => "Object-oriented, actor-model, capabilities-secure programming language",
+          "full_name"              => "ponyc",
+          "homepage"               => "https://www.ponylang.io/",
+          "license"                => "BSD-2-Clause",
+          "name"                   => "ponyc",
+          "ruby_source_path"       => "Formula/p/ponyc.rb",
+          "tap"                    => "homebrew/core",
+          "tap_git_head"           => tap_git_head,
+          # TODO: improve this API before we ship internal API v3 to users
+          "uses_from_macos"        => [{ "llvm"=>[:build, :test] }, "zlib"],
+          "uses_from_macos_bounds" => [{}, {}],
+          "versions"               => { "bottle"=>true, "head"=>nil, "stable"=>"0.58.1" },
+        }
+      end
+
+      it "loads fennel" do
+        fennel = Formulary.factory("fennel")
+        expect(fennel.to_hash).to include(**fennel_metadata)
+      end
+
+      it "loads fennel from rename" do
+        fennel = Formulary.factory("romanesco")
+        expect(fennel.to_hash).to include(**fennel_metadata)
+      end
+
+      it "loads fennel from alias" do
+        fennel = Formulary.factory("fennel-lang")
+        expect(fennel.to_hash).to include(**fennel_metadata)
+      end
+
+      it "loads ponyc" do
+        ponyc = Formulary.factory("ponyc")
+        expect(ponyc.to_hash).to include(**ponyc_metadata)
+      end
+
+      it "loads ponyc from alias" do
+        ponyc = Formulary.factory("ponyc-lang")
+        expect(ponyc.to_hash).to include(**ponyc_metadata)
+      end
+    end
+  end
+end