Perf: cache the ontology list, removing ~80% of /search response time

config/config.rb

@@ -25,6 +25,8 @@ def config(&block)
     @settings.enable_unicorn_workerkiller ||= false
     @settings.req_per_second_per_ip       ||= 15
     @settings.ontology_report_path        ||= "../ontologies_report.json"
+    # TTL (seconds) for the process-level ontology list cache; 0 disables it
+    @settings.ontology_list_cache_ttl     ||= 300
 
     if @settings.enable_monitoring
       puts "(API) >> Slow queries log enabled: #{@settings.slow_request_log}"

config/newrelic_method_tracers.rb

@@ -37,7 +37,10 @@ module NewRelicMethodTracers
     'LinkedData::Serializer' => %i[build_response serialize],
     # Solr query execution incl. Ruby-side response parsing (the HTTP
     # call itself already appears as an External segment)
-    'LinkedData::Models::Class' => %i[search]
+    'LinkedData::Models::Class' => %i[search],
+    # Ontology list cache: `all` shows per-request cost (near-zero on a
+    # hit), `load_ontologies` fires only on cache misses
+    'OntologyListCache' => %i[all load_ontologies]
   }.freeze
 
   def self.register

helpers/application_helper.rb

@@ -254,15 +254,14 @@ def restricted_ontologies(params = nil)
 
         found_onts = false
         if params["ontologies"] && !params["ontologies"].empty?
+          # Access-control attributes are preloaded by OntologyListCache
           onts = ontology_objects_from_params(params)
           found_onts = onts.length > 0
-          Ontology.where.models(onts).include(*Ontology.access_control_load_attrs).all
         else
-          onts = if params["also_include_views"] == "true"
-            Ontology.where.include(Ontology.goo_attrs_to_load()).to_a
-          else
-            Ontology.where.filter(Goo::Filter.new(:viewOf).unbound).include(Ontology.goo_attrs_to_load()).to_a
-                 end
+          onts = OntologyListCache.all
+          # Views are excluded in Ruby rather than via a Goo viewOf filter
+          # so the cached list can serve both variants
+          onts = onts.select { |o| o.viewOf.nil? } unless params["also_include_views"] == "true"
 
           found_onts = onts.length > 0
 
@@ -367,15 +366,14 @@ def ontology_from_acronym(acronym)
       # Replies 400 if the ontology does not have a parsed submission
       def ontology_objects_from_params(params = nil)
         ontologies = Set.new(ontologies_param(params))
-        all_onts = LinkedData::Models::Ontology.where.include(LinkedData::Models::Ontology.goo_attrs_to_load).to_a
-        all_onts.select { |o| ontologies.include?(o.id.to_s) }
+        OntologyListCache.all.select { |o| ontologies.include?(o.id.to_s) }
       end
 
       def ontology_uri_acronym_map
         cached_map = naive_expiring_cache_read(__method__)
         return cached_map if cached_map
         map = {}
-        LinkedData::Models::Ontology.where.include(:acronym).all.each { |o| map[o.acronym] = o.id.to_s }
+        OntologyListCache.all.each { |o| map[o.acronym] = o.id.to_s }
         naive_expiring_cache_write(__method__, map)
         map
       end
@@ -384,7 +382,7 @@ def acronym_ontology_uri_map
         cached_map = naive_expiring_cache_read(__method__)
         return cached_map if cached_map
         map = {}
-        LinkedData::Models::Ontology.where.include(:acronym).all.each { |o| map[o.id.to_s] = o.acronym }
+        OntologyListCache.all.each { |o| map[o.id.to_s] = o.acronym }
         naive_expiring_cache_write(__method__, map)
         map
       end

lib/ontology_list_cache.rb

@@ -0,0 +1,66 @@
+# Process-level TTL cache of the full ontology list (issue #244).
+#
+# Loading all ~1,200 ontologies from the backend accounted for ~97% of
+# /search response time because both branches of restricted_ontologies
+# reloaded the list on every request. The list changes on the scale of
+# days, so it is loaded once per TTL window per worker instead.
+#
+# Staleness contract: ontology additions/removals and ACL or
+# viewingRestriction changes take up to the TTL to propagate (per
+# worker). Per-user visibility filtering (slice/ACL) remains per-request
+# in the callers; only the raw list is cached.
+#
+# TTL comes from LinkedData::OntologiesAPI.settings.ontology_list_cache_ttl
+# (seconds, default 300). A value of 0 disables caching entirely.
+#
+# NOTE: the returned array is a defensive copy, but the Ontology objects
+# in it are shared across requests within a worker. Loading further
+# attributes on them (bring) is safe; assigning attributes is not.
+class OntologyListCache
+  @lock = Mutex.new
+  @cached = nil
+
+  class << self
+    def all
+      ttl = ttl_seconds
+      return load_ontologies if ttl <= 0
+
+      @lock.synchronize do
+        if @cached.nil? || clock_now >= @cached[:expires_at]
+          objects = load_ontologies
+          @cached = { objects: objects, expires_at: clock_now + ttl }
+        end
+        @cached[:objects].dup
+      end
+    end
+
+    def invalidate!
+      @lock.synchronize { @cached = nil }
+    end
+
+    private
+
+    def ttl_seconds
+      LinkedData::OntologiesAPI.settings.ontology_list_cache_ttl.to_i
+    end
+
+    def load_ontologies
+      LinkedData::Models::Ontology.where.include(*load_attributes).to_a
+    end
+
+    # Union of the attributes every caller of the cached list needs:
+    # serialization defaults (previous behavior of the load-all sites),
+    # access-control attrs (so restricted_ontologies needs no second
+    # query), and viewOf (site-wide view filtering moved into Ruby).
+    def load_attributes
+      attrs = LinkedData::Models::Ontology.goo_attrs_to_load
+      attrs += LinkedData::Models::Ontology.access_control_load_attrs.to_a
+      attrs << :viewOf
+      attrs.uniq
+    end
+
+    def clock_now
+      Time.now
+    end
+  end
+end

script/benchmark_search_latency.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+# Before/after latency benchmark for the ontology list cache (issue #244,
+# PR #245). Alternates identical requests between two running API
+# instances so environmental drift (shared staging backends, cache
+# warming) affects both revisions equally.
+#
+# Usage:
+#   ruby script/benchmark_search_latency.rb BEFORE_URL AFTER_URL [N]
+#
+# Example (before = merge-base worktree on 9394, after = branch on 9393):
+#   ruby script/benchmark_search_latency.rb http://localhost:9394 http://localhost:9393 30
+#
+# Prints median/p95/mean per scenario per revision and verifies both
+# revisions return identical (port-normalized) response bodies.
+require 'net/http'
+require 'uri'
+require 'json'
+
+before_url, after_url = ARGV[0], ARGV[1]
+n = (ARGV[2] || 30).to_i
+abort "Usage: #{$0} BEFORE_URL AFTER_URL [N]" unless before_url && after_url
+
+WARMUPS = 5
+
+SCENARIOS = {
+  'site-wide search'   => '/search?q=melanoma',
+  'scoped search'      => '/search?q=melanoma&ontologies=NCIT,SNOMEDCT,MESH',
+  'search incl. views' => '/search?q=cancer&also_include_views=true'
+}.freeze
+
+def get(base, path)
+  uri = URI.parse(base + path)
+  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  res = Net::HTTP.get_response(uri)
+  elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000
+  raise "#{uri} returned #{res.code}" unless res.code == '200'
+  [elapsed, res.body]
+end
+
+def stats(times)
+  sorted = times.sort
+  {
+    median: sorted[sorted.length / 2],
+    p95: sorted[(sorted.length * 0.95).floor.clamp(0, sorted.length - 1)],
+    mean: times.sum / times.length
+  }
+end
+
+# Hypermedia links embed LinkedData.settings.rest_url_prefix (a fixed
+# host:port from the shared config), not the serving port, so normalize
+# every localhost:<port> occurrence on both sides before comparing.
+def normalize(body, _base)
+  body.gsub(/localhost:\d+/, 'HOST')
+end
+
+puts "#{WARMUPS} warmups + #{n} alternating measured requests per revision per scenario"
+puts "before: #{before_url}   after: #{after_url}\n\n"
+
+SCENARIOS.each do |name, path|
+  WARMUPS.times { get(before_url, path); get(after_url, path) }
+
+  before_times, after_times = [], []
+  before_body = after_body = nil
+  n.times do
+    t, before_body = get(before_url, path)
+    before_times << t
+    t, after_body = get(after_url, path)
+    after_times << t
+  end
+
+  parity = normalize(before_body, before_url) == normalize(after_body, after_url)
+  b, a = stats(before_times), stats(after_times)
+  delta = ->(k) { format('%+.1f%%', (a[k] - b[k]) / b[k] * 100) }
+
+  puts "== #{name} (#{path})"
+  puts format('   median: %8.1f ms -> %8.1f ms  (%s)', b[:median], a[:median], delta.call(:median))
+  puts format('   p95:    %8.1f ms -> %8.1f ms  (%s)', b[:p95], a[:p95], delta.call(:p95))
+  puts format('   mean:   %8.1f ms -> %8.1f ms  (%s)', b[:mean], a[:mean], delta.call(:mean))
+  puts "   response parity: #{parity ? 'IDENTICAL' : 'DIFFERENT (INVESTIGATE!)'}"
+  puts
+end

test/lib/test_ontology_list_cache.rb

@@ -0,0 +1,139 @@
+require_relative '../test_case'
+
+# Process-level TTL cache for the full ontology list (issue #244).
+# Logic tests swap the loader/clock singletons (repo convention, see
+# test_search_helper.rb); the final test exercises the real loader
+# against the test backend.
+class TestOntologyListCache < TestCase
+
+  def setup
+    OntologyListCache.invalidate!
+  end
+
+  def teardown
+    OntologyListCache.invalidate!
+  end
+
+  # Fetch the method defined on the class's own singleton, skipping any
+  # modules prepended above it (the New Relic tracer wraps traced methods
+  # via prepend; capturing the wrapper and later re-defining it as the
+  # base method would leave its `super` dangling).
+  def own_singleton_method(target, name)
+    m = target.method(name)
+    m = m.super_method while m && m.owner != target.singleton_class
+    refute_nil m, "#{target}.#{name} has no definition on its own singleton"
+    m
+  end
+
+  # Swap OntologyListCache.load_ontologies for the duration of the block,
+  # yielding a counter of loader invocations. Restores the original
+  # definition (and its private visibility) afterward.
+  def with_loader(result_for_call)
+    calls = 0
+    original = own_singleton_method(OntologyListCache, :load_ontologies)
+    OntologyListCache.define_singleton_method(:load_ontologies) do
+      calls += 1
+      result_for_call.call(calls)
+    end
+    yield -> { calls }
+  ensure
+    OntologyListCache.define_singleton_method(:load_ontologies, original)
+    OntologyListCache.singleton_class.send(:private, :load_ontologies)
+  end
+
+  def with_clock(time)
+    original = own_singleton_method(OntologyListCache, :clock_now)
+    OntologyListCache.define_singleton_method(:clock_now) { time }
+    yield
+  ensure
+    OntologyListCache.define_singleton_method(:clock_now, original)
+    OntologyListCache.singleton_class.send(:private, :clock_now)
+  end
+
+  def test_caches_loaded_list_across_calls
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 300) do
+      with_loader(->(_n) { [:a, :b, :c] }) do |calls|
+        assert_equal [:a, :b, :c], OntologyListCache.all
+        assert_equal [:a, :b, :c], OntologyListCache.all
+        assert_equal 1, calls.call, 'loader should run once for two reads inside the TTL'
+      end
+    end
+  end
+
+  def test_returns_isolated_copies
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 300) do
+      with_loader(->(_n) { [:a, :b, :c] }) do |_calls|
+        first = OntologyListCache.all
+        first.select! { |o| o == :a }   # destructive caller, as in get_term_search_query
+        first.clear
+        assert_equal [:a, :b, :c], OntologyListCache.all,
+                     'mutating a returned array must not poison the cache'
+      end
+    end
+  end
+
+  def test_reloads_after_ttl_expires
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 300) do
+      with_loader(->(n) { [n] }) do |calls|
+        start = Time.now
+        with_clock(start)       { assert_equal [1], OntologyListCache.all }
+        with_clock(start + 299) { assert_equal [1], OntologyListCache.all }
+        with_clock(start + 301) { assert_equal [2], OntologyListCache.all }
+        assert_equal 2, calls.call
+      end
+    end
+  end
+
+  def test_ttl_zero_disables_caching
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 0) do
+      with_loader(->(n) { [n] }) do |calls|
+        assert_equal [1], OntologyListCache.all
+        assert_equal [2], OntologyListCache.all
+        assert_equal 2, calls.call, 'ttl 0 must bypass the cache entirely'
+      end
+    end
+  end
+
+  def test_ttl_defaults_to_300
+    assert_equal 300, LinkedData::OntologiesAPI.settings.ontology_list_cache_ttl
+  end
+
+  def test_invalidate_forces_reload
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 300) do
+      with_loader(->(n) { [n] }) do |calls|
+        assert_equal [1], OntologyListCache.all
+        OntologyListCache.invalidate!
+        assert_equal [2], OntologyListCache.all
+        assert_equal 2, calls.call
+      end
+    end
+  end
+
+  def test_load_failures_propagate_and_are_not_cached
+    with_settings(LinkedData::OntologiesAPI.settings, ontology_list_cache_ttl: 300) do
+      with_loader(->(n) { n == 1 ? raise(StandardError, 'backend down') : [:recovered] }) do |_calls|
+        assert_raises(StandardError) { OntologyListCache.all }
+        assert_equal [:recovered], OntologyListCache.all,
+                     'a failed load must not leave a poisoned cache entry'
+      end
+    end
+  end
+
+  def test_real_loader_returns_ontologies_with_needed_attributes
+    count, _acronyms, _onts = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(
+      ont_count: 2, submission_count: 0)
+    assert_operator count, :>=, 2
+
+    onts = OntologyListCache.all
+    assert_operator onts.length, :>=, 2
+    ont = onts.first
+    assert_kind_of LinkedData::Models::Ontology, ont
+
+    # Attributes needed by all downstream consumers must be preloaded:
+    # acronym (Solr filter), access-control attrs (filter_access),
+    # viewOf (site-wide view filtering in Ruby).
+    [:acronym, :viewingRestriction, :administeredBy, :acl, :viewOf].each do |attr|
+      refute ont.bring?(attr), "#{attr} must be preloaded on cached ontology objects"
+    end
+  end
+end

test/test_case.rb

@@ -162,6 +162,9 @@ def with_settings(settings = LinkedData.settings, **overrides)
   def before_setup
     super
     @settings_snapshot = snapshot_settings
+    # Tests create/delete ontologies at will; a list cached by a previous
+    # test would make them invisible to search filters (issue #244 cache)
+    OntologyListCache.invalidate!
   end
 
   def after_teardown