Assign installation_id to the OTEL resource (#2434)

Follow-up to #2409. This change makes the installation_id available in OTEL traces and metrics.

Author: alco

packages/sync-service/config/runtime.exs

@@ -50,7 +50,11 @@ instance_id = env!("ELECTRIC_INSTANCE_ID", :string, Electric.Utils.uuid4())
 version = Electric.version()
 
 config :opentelemetry,
-  resource_detectors: [:otel_resource_env_var, :otel_resource_app_env],
+  resource_detectors: [
+    :otel_resource_env_var,
+    :otel_resource_app_env,
+    Electric.Telemetry.OpenTelemetry.ResourceDetector
+  ],
   resource: %{service: %{name: service_name, version: version}, instance: %{id: instance_id}}
 
 otlp_endpoint = env!("ELECTRIC_OTLP_ENDPOINT", :string, nil)

packages/sync-service/lib/electric/config.ex

@@ -149,6 +149,12 @@ defmodule Electric.Config do
     Application.fetch_env!(:electric, key)
   end
 
+  def persistent_kv do
+    with {m, f, a} <- get_env(:persistent_kv) do
+      apply(m, f, [a])
+    end
+  end
+
   @doc ~S"""
   Parse a PostgreSQL URI into a keyword list.
 

packages/sync-service/lib/electric/telemetry/open_telemetry/resource_detector.ex

@@ -0,0 +1,26 @@
+defmodule Electric.Telemetry.OpenTelemetry.ResourceDetector do
+  @behaviour :otel_resource_detector
+
+  # A resource detector is called inside a one-off process and has 5-6 seconds to return
+  # the resource.
+  #
+  # We need to wait just long enough for the OTP application callback to persist
+  # an installation_id to the KV store.
+  def get_resource(_config) do
+    {m, f, a} = Electric.Config.get_env(:persistent_kv)
+    kv = apply(m, f, [a])
+
+    installation_id = poll_installation_id(kv, 100)
+    :otel_resource.create("instance.installation_id": installation_id)
+  end
+
+  defp poll_installation_id(kv, sleep_ms) do
+    try do
+      Electric.Config.installation_id!(kv)
+    rescue
+      _ ->
+        Process.sleep(sleep_ms)
+        poll_installation_id(kv, sleep_ms)
+    end
+  end
+end

packages/sync-service/lib/electric/telemetry/stack_telemetry.ex

@@ -64,11 +64,16 @@ with_telemetry [OtelMetricExporter, Telemetry.Metrics] do
     end
 
     defp otel_reporter_child_spec(%{otel_metrics?: true} = opts) do
+      kv = Electric.Config.persistent_kv()
+
       {OtelMetricExporter,
        name: :"stack_otel_telemetry_#{opts.stack_id}",
        metrics: otel_metrics(opts),
        export_period: opts.otel_export_period,
-       resource: %{stack_id: opts.stack_id}}
+       resource: %{
+         stack_id: opts.stack_id,
+         instance: %{installation_id: Electric.Config.installation_id!(kv)}
+       }}
     end
 
     defp otel_reporter_child_spec(_), do: nil

packages/sync-service/mix.exs

@@ -138,7 +138,7 @@ defmodule Electric.MixProject do
       {:sentry, "~> 10.0"},
       {:opentelemetry, "~> 1.5"},
       {:opentelemetry_exporter, "~> 1.8"},
-      {:otel_metric_exporter, "~> 0.2"},
+      {:otel_metric_exporter, "~> 0.3"},
       # For debugging the otel_metric_exporter check it out locally and uncomment the line below
       # {:otel_metric_exporter, path: "../../../elixir-otel-metric-exporter"},
       {:telemetry_metrics_prometheus_core, "~> 1.1"},

packages/sync-service/mix.lock

@@ -39,7 +39,7 @@
   "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.8.0", "5d546123230771ef4174e37bedfd77e3374913304cd6ea3ca82a2add49cd5d56", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.5.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "a1f9f271f8d3b02b81462a6bfef7075fd8457fdb06adff5d2537df5e2264d9af"},
   "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"},
   "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.1", "4a73bfa29d7780ffe33db345465919cef875034854649c37ac789eb8e8f38b21", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ee43b14e6866123a3ee1344e3c0d3d7591f4537542c2a925fcdbf46249c9b50b"},
-  "otel_metric_exporter": {:hex, :otel_metric_exporter, "0.2.4", "1e476848c52e7967d4e938603b0db840798ae3525407e85596508885e68374ce", [:mix], [{:finch, "~> 0.19", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.13.0", [hex: :protobuf, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "bbb8667cae543df822def74625d1e1bf62206a5349da14604b8a9a563507a080"},
+  "otel_metric_exporter": {:hex, :otel_metric_exporter, "0.3.4", "801e99db107c99eabe57a6bc0750c0fa375c84ab1012183fae3c378bd6fb55be", [:mix], [{:finch, "~> 0.19", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.13.0", [hex: :protobuf, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "db7473226ee36cb16d546124f751e94c0a7a071ab9a2a278185a589ed6f40f4d"},
   "pg_query_ex": {:hex, :pg_query_ex, "0.5.3", "84bf09f4ea10ada6e1cbfd739ccb9ffb6e5b21d87ab81cf52a42fefcc1808566", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:protox, "~> 1.7", [hex: :protox, repo: "hexpm", optional: false]}], "hexpm", "ec0554d6d287da4cc15cc773577ef61cf41d5d6fcc784bb85f6209439cb246a7"},
   "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"},
   "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},