fix: keep the serve API key out of the operator's APM

The FastAPI server span records url.query verbatim, and lfx serve accepts
its API key as a query parameter (APIKeyQuery). That scope is allowlisted
for export, so any deployment whose callers pass the key that way has been
shipping it to whatever OTLP backend the operator configured. A probe
against the real instrumented app exports url.query = "x-api-key=<the key>".

The export boundary now strips query strings and userinfo from URL
attributes on the way out. Scheme, host, port and path survive, so the route
an operator needs is unchanged. Done here rather than in an instrumentor
hook because this is the one place that already decides what leaves.

Also adds SQLAlchemy spans, so a slow request can be attributed to the
queries it made. Allowlisted only after checking what they carry:
db.statement keeps bound parameters as placeholders, so chat message text
stays in the database.

httpx and requests are deliberately NOT instrumented and NOT allowlisted,
even though the ticket asks for outbound spans. They are the transports the
LLM vendor SDKs ride on, and those SDKs instrument them globally against
whichever provider is global, so admitting them would put one span per
outbound provider call into the APM and reverse a documented decision.
Outbound provider health has a leak-safe metrics path instead. Left for a
separate decision.

Sampling needed no change: the provider takes no explicit sampler, so
OTEL_TRACES_SAMPLER already applies. Pinned with a determinism check that
exports 0 spans at ratio 0.0 and all 20 at 1.0.

Author: ogabrielluiz

.secrets.baseline

@@ -6696,7 +6696,7 @@
         "filename": "src/lfx/src/lfx/cli/serve_app.py",
         "hashed_secret": "b894b81be94cf8fa8d7536475aaec876addf05c8",
         "is_verified": false,
-        "line_number": 70,
+        "line_number": 71,
         "is_secret": false
       }
     ],
@@ -7309,5 +7309,5 @@
       }
     ]
   },
-  "generated_at": "2026-08-04T18:09:09Z"
+  "generated_at": "2026-08-05T14:27:06Z"
 }

src/backend/base/langflow/main.py

@@ -21,7 +21,12 @@
 from filelock import FileLock
 from lfx.interface.utils import setup_llm_caching
 from lfx.log.logger import configure, logger
-from lfx.observability import instrument_fastapi_app, start_event_loop_lag_monitor, stop_event_loop_lag_monitor
+from lfx.observability import (
+    instrument_dependencies,
+    instrument_fastapi_app,
+    start_event_loop_lag_monitor,
+    stop_event_loop_lag_monitor,
+)
 from pydantic import PydanticDeprecatedSince20
 from pydantic_core import PydanticSerializationError
 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
@@ -988,6 +993,9 @@ async def exception_handler(_request: Request, exc: Exception):
     # lazy-include route patch + instrument_app). The helper lives in lfx so lfx serve
     # instruments its own app the same way.
     instrument_fastapi_app(app)
+    # The rest of the trace tree: DB queries and outbound HTTP made during a run. The engine is
+    # not built yet at app-factory time, so SQLAlchemy instruments its own dispatcher globally.
+    instrument_dependencies()
 
     add_pagination(app)
 

src/backend/base/pyproject.toml

@@ -69,6 +69,7 @@ dependencies = [
     "opentelemetry-exporter-prometheus>=0.50b0,<1.0.0",
     "opentelemetry-exporter-otlp>=1.30.0,<2.0.0",
     "opentelemetry-instrumentation-fastapi>=0.50b0,<1.0.0",
+    "opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1.0.0",
     "opentelemetry-instrumentation-requests>=0.50b0,<1.0.0",
     "opentelemetry-instrumentation-urllib3>=0.50b0,<1.0.0",
     # Process CPU, memory, GC and thread counts. Without it the only metrics the APM gets are

src/backend/tests/unit/services/telemetry/test_dependency_span_redaction.py

@@ -0,0 +1,212 @@
+"""Dependency spans must not carry credentials out of the process.
+
+This is the reason ``requests`` and ``urllib3`` were pulled from the export allowlist once
+before: their spans write the full request URL, and providers put API keys in the query
+string (Gemini's ``?key=``, among others). httpx is the transport the OpenAI and Anthropic
+SDKs ride on, so admitting these scopes is only safe while the export boundary redacts URLs.
+
+Each case runs in a subprocess because the tracer provider is process-global.
+"""
+
+import gzip
+import http.server
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+from pathlib import Path
+
+import pytest
+from lfx.observability import APPLICATION_INSTRUMENTATION_SCOPES
+
+
+class _Collector(http.server.BaseHTTPRequestHandler):
+    def do_POST(self) -> None:
+        body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
+        if self.headers.get("Content-Encoding") == "gzip":
+            body = gzip.decompress(body)
+        self.server.requests.append((self.path, body))
+        self.send_response(200)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
+    def log_message(self, *args) -> None:
+        pass
+
+
+@pytest.fixture
+def collector():
+    server = http.server.HTTPServer(("127.0.0.1", 0), _Collector)
+    server.requests = []
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    yield server
+    server.shutdown()
+    server.server_close()
+    thread.join(timeout=5)
+
+
+SECRET = "lfx-serve-api-key-NOT-REAL"  # noqa: S105  # pragma: allowlist secret
+
+# The server span is already allowlisted, and ``lfx serve`` accepts its API key as a query
+# parameter, so this is a live path today rather than a hypothetical one.
+SERVER_SPAN_PROBE = f"""
+import asyncio, json
+
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from lfx.observability import ApplicationOnlySpanProcessor, instrument_fastapi_app
+
+exporter = InMemorySpanExporter()
+provider = TracerProvider()
+provider.add_span_processor(ApplicationOnlySpanProcessor(exporter))
+trace.set_tracer_provider(provider)
+
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+
+app = FastAPI()
+
+
+@app.get("/flows/run")
+async def run():
+    return {{"ok": True}}
+
+
+instrument_fastapi_app(app)
+SECRET = {SECRET!r}
+
+
+async def main():
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://probe") as client:
+        await client.get(f"/flows/run?x-api-key={{SECRET}}&flow=abc")
+    provider.force_flush()
+    spans = [
+        {{"name": s.name, "attrs": dict(s.attributes)}}
+        for s in exporter.get_finished_spans()
+        if s.attributes and "url.path" in s.attributes
+    ]
+    print("PROBE_RESULT " + json.dumps({{"spans": spans}}))
+
+
+asyncio.run(main())
+"""
+
+
+def run_probe(source: str) -> dict:
+    env = {k: v for k, v in os.environ.items() if not k.startswith("OTEL_")}
+    with tempfile.TemporaryDirectory() as tmp:
+        probe_path = Path(tmp) / "probe.py"
+        probe_path.write_text(source, encoding="utf-8")
+        completed = subprocess.run(  # noqa: S603
+            [sys.executable, str(probe_path)],
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=300,
+            check=False,
+        )
+    assert completed.returncode == 0, completed.stderr
+    line = next(ln for ln in completed.stdout.splitlines() if ln.startswith("PROBE_RESULT "))
+    return json.loads(line.removeprefix("PROBE_RESULT "))
+
+
+def test_the_serve_api_key_never_reaches_the_apm():
+    """Regression for a live leak: the key is accepted as a query param and the span kept it."""
+    result = run_probe(SERVER_SPAN_PROBE)
+
+    assert result["spans"], "expected a server span carrying url.path"
+    blob = json.dumps(result["spans"])
+    assert SECRET not in blob, f"serve API key reached the APM: {blob}"
+
+
+def test_redaction_keeps_the_route_an_operator_needs():
+    """Blanking everything would be safe and useless; path and method must survive."""
+    result = run_probe(SERVER_SPAN_PROBE)
+    attrs = result["spans"][0]["attrs"]
+
+    assert attrs["url.path"] == "/flows/run"
+    assert attrs["url.query"] == ""
+    assert attrs["http.request.method"] == "GET"
+
+
+def test_outbound_http_scopes_stay_out_of_the_allowlist():
+    """Redacting URLs does not make the LLM transports safe to export; that boundary stands."""
+    for scope in (
+        "opentelemetry.instrumentation.httpx",
+        "opentelemetry.instrumentation.requests",
+        "opentelemetry.instrumentation.urllib3",
+    ):
+        assert scope not in APPLICATION_INSTRUMENTATION_SCOPES
+
+
+def test_database_spans_are_allowlisted():
+    """Verified separately to carry bound-parameter placeholders, never row values."""
+    assert "opentelemetry.instrumentation.sqlalchemy" in APPLICATION_INSTRUMENTATION_SCOPES
+
+
+# The ticket's sampler criterion, run against the real bootstrap rather than a hand-built
+# provider: the same request at ratio 0.0 must export nothing and at 1.0 must export the lot.
+SAMPLER_PROBE = """
+import json
+from opentelemetry import trace
+from lfx.observability import APPLICATION_TRACER_NAME, bootstrap_application_telemetry
+
+telemetry = bootstrap_application_telemetry()
+tracer = trace.get_tracer(APPLICATION_TRACER_NAME)
+for _ in range(20):
+    with tracer.start_as_current_span("flow.execute"):
+        pass
+telemetry.shutdown()
+print("PROBE_RESULT done")
+"""
+
+
+def _run_sampler_probe(endpoint: str, ratio: str) -> None:
+    env = {k: v for k, v in os.environ.items() if not k.startswith("OTEL_")}
+    env["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
+    env["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf"
+    env["OTEL_TRACES_SAMPLER"] = "traceidratio"
+    env["OTEL_TRACES_SAMPLER_ARG"] = ratio
+    completed = subprocess.run(  # noqa: S603
+        [sys.executable, "-c", SAMPLER_PROBE],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=300,
+        check=False,
+    )
+    assert completed.returncode == 0, completed.stderr
+
+
+def _exported_span_count(requests_seen) -> int:
+    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+
+    total = 0
+    for path, body in requests_seen:
+        if path != "/v1/traces":
+            continue
+        request = ExportTraceServiceRequest()
+        request.ParseFromString(body)
+        for rs in request.resource_spans:
+            for ss in rs.scope_spans:
+                total += len(ss.spans)
+    return total
+
+
+def test_sampling_off_exports_nothing(collector):
+    port = collector.server_address[1]
+    _run_sampler_probe(f"http://127.0.0.1:{port}", "0.0")
+
+    assert _exported_span_count(collector.requests) == 0
+
+
+def test_sampling_on_exports_every_span(collector):
+    port = collector.server_address[1]
+    _run_sampler_probe(f"http://127.0.0.1:{port}", "1.0")
+
+    assert _exported_span_count(collector.requests) == 20

src/lfx/pyproject.toml

@@ -90,6 +90,7 @@ otel = [
     "opentelemetry-exporter-otlp>=1.30.0,<2.0.0",
     "opentelemetry-exporter-prometheus>=0.50b0,<1.0.0",
     "opentelemetry-instrumentation-fastapi>=0.50b0,<1.0.0",
+    "opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1.0.0",
     "opentelemetry-instrumentation-system-metrics>=0.50b0,<1.0.0",
 ]
 # Engine-only by default: ``pip install lfx`` ships NO bundles.  This optional

src/lfx/src/lfx/cli/serve_app.py

@@ -47,6 +47,7 @@
 from lfx.log.logger import logger
 from lfx.observability import (
     bootstrap_application_telemetry,
+    instrument_dependencies,
     instrument_fastapi_app,
     start_event_loop_lag_monitor,
     stop_event_loop_lag_monitor,
@@ -707,6 +708,7 @@ async def _lifespan(_app: FastAPI):
     )
 
     instrument_fastapi_app(app)
+    instrument_dependencies()
 
     app.state.registry = registry
     # Snapshot the API key once so per-request auth (verify_api_key, run on a threadpool