Skip to content

Commit 3d19cca

Browse files
committed
fix: Disable tracing for crates that cause stack overflows on exit
Due to open-telemetry/opentelemetry-rust#2877, we have to turn off tracing for a couple of crates, to avoid a stack overflow when exiting a binary using opentelemetry-rust and an OTLP endpoint. I don't love this fix personally, but the alternative seems worse.
1 parent ec0ec43 commit 3d19cca

3 files changed

Lines changed: 65 additions & 0 deletions

File tree

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ tower-otel-http-metrics = { version = "0.16.0", features = [
5656
opentelemetry-resource-detectors = "0.9.0"
5757

5858
[dev-dependencies]
59+
axum = { version = "^0.8" }
5960
tokio = { version = "1.43.0", features = ["full"] }
6061

6162
[features]

src/lib.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,23 @@ pub fn init_otel(
218218
.transpose()?
219219
.unwrap_or((None, None));
220220

221+
// We have to ignore these specific crates to avoid them emitting traces
222+
// after our OTLP collector has been turned off, causing an infinite loop.
223+
//
224+
// See the issue: https://github.com/open-telemetry/opentelemetry-rust/issues/2877
225+
// See the source of the workaround:
226+
// https://github.com/open-telemetry/opentelemetry-rust/blob/v0.31.0/opentelemetry-otlp/examples/basic-otlp/src/main.rs#L69-L86
227+
#[allow(clippy::expect_used)]
228+
let filter_otel = EnvFilter::new("info")
229+
.add_directive("hyper=off".parse().expect("Could not parse constant directive"))
230+
.add_directive("tonic=off".parse().expect("Could not parse constant directive"))
231+
.add_directive("h2=off".parse().expect("Could not parse constant directive"))
232+
.add_directive("reqwest=off".parse().expect("Could not parse constant directive"));
233+
221234
// Initialize the tracing subscriber with the stdout layer and
222235
// layers for exporting over OpenTelemetry the logs, traces and metrics.
223236
let subscriber = tracing_subscriber::registry()
237+
.with(filter_otel)
224238
.with(logs_layer)
225239
.with(stdout_layer)
226240
.with(meter_layer)

tests/otlp.rs

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// SPDX-FileCopyrightText: 2025 Famedly GmbH (info@famedly.com)
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
//! Tests for OTLP
6+
7+
#![allow(clippy::expect_used)]
8+
9+
use axum::routing::post;
10+
use rust_telemetry::{
11+
config::{ExporterConfig, OtelConfig, StdoutLogsConfig},
12+
init_otel,
13+
};
14+
use url::Url;
15+
16+
/// The current version of the OTLP exporter also induces traces after it's been
17+
/// shut down, causing a feedback loop which ends in a stack overflow
18+
///
19+
/// This test checks that we can start rust-telemetry with an OTLP endpoint
20+
/// enabled, output a log line, and exit our program without a stack overflow.
21+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22+
async fn test_otlp_exporter() {
23+
let router = post(|| async { Vec::new() });
24+
let listener =
25+
tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("Could not bind TCPListener");
26+
let endpoint = Url::parse(&format!(
27+
"http://{}",
28+
listener.local_addr().expect("Failed getting address for listener")
29+
))
30+
.expect("Failed parsing URL for endpoint");
31+
let _mock_otlp = tokio::spawn(async move { axum::serve(listener, router).await });
32+
33+
let provider_config = Some(rust_telemetry::config::ProviderConfig {
34+
enabled: true,
35+
general_level: famedly_rust_utils::LevelFilter(tracing::level_filters::LevelFilter::DEBUG),
36+
..Default::default()
37+
});
38+
39+
let config = OtelConfig {
40+
stdout: Some(StdoutLogsConfig { enabled: true, ..Default::default() }),
41+
exporter: Some(ExporterConfig {
42+
endpoint: endpoint.into(),
43+
logs: provider_config.clone(),
44+
traces: provider_config.clone(),
45+
..Default::default()
46+
}),
47+
};
48+
let _guard = init_otel!(&config).expect("Error initializing Otel");
49+
tracing::info!("Output a line");
50+
}

0 commit comments

Comments
 (0)