otelcol/grpclog: suppress benign gRPC client-disconnect warnings (#15024)

Fixes #5169

## Problem

When a gRPC client disconnects normally (e.g., a .NET app shutting
down its `TracerProvider`), the collector logs spurious WARN messages
from the gRPC transport layer:

These are expected during graceful client shutdown but appear
alarming to operators and pollute production logs.

## Root cause

grpc-go is doing the right thing: messages like this are gated
behind `LoggerV2.V(2)` in `internal/transport/http2_server.go`,
i.e. they are explicitly classified as supplemental verbosity, not
severity-level warnings. They should never reach a WARN sink.

The bug is in zapgrpc. `zapgrpc.Logger.V` is implemented as
`levelEnabler.Enabled(zapcore.Level(level - 1))`, which conflates
grpclog's *verbosity* with zap's *severity*. With WARN enabled,
`V(2)` returns true, so grpc-go thinks verbosity level 2 is
permitted and emits the message — which then gets formatted by
zapgrpc as a WARN entry. See
[uber-go/zap#1544](uber-go/zap#1544).

## Solution

Wrap the installed `grpclog.LoggerV2` with a `fixedVerbosityLogger`
whose `V()` compares the requested level against a fixed threshold
(default 0) instead of the underlying zap severity enabler. This
restores grpclog's intended verbosity gating, so verbose messages
stop at grpc-go before formatting rather than being filtered out
of the log stream after the fact.

The default threshold of `0` matches grpclog when
`GRPC_GO_LOG_VERBOSITY_LEVEL` is unset.

This approach:

- Addresses the root cause (broken `V()` mapping) rather than
  string-matching specific message contents.
- Stops chatty messages at the source, before grpc-go formats them.
- Is generic — kills *all* `V(2)+` noise, including future grpc-go
  additions, not just two cherry-picked patterns.
- Leaves severity-level emission (`Info` / `Warning` / `Error`)
  completely unchanged.
- Tiny: ~10 lines of wrapper plus comment.

## Testing

- `TestFixedVerbosity` — asserts `V(0)` is enabled and `V(1)`/`V(2)`
  are not at the default threshold.
- `TestSeverityLevelsUnaffected` — direct `Warning("…")` still
  fires at WARN, confirming the wrapper does not gate severity.
- Existing `TestGRPCLogger` cases continue to pass unchanged.

---------

Signed-off-by: Sai Kruthi Wusirika <kruthiwusirika@gmail.com>

Author: kruthiwusirika5

.chloggen/fix-zapgrpc-client-close-warning.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: pkg/otelcol
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Stop emitting verbose gRPC transport messages at WARN during normal client disconnect.
+
+# One or more tracking issues or pull requests related to the change
+issues: [5169]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  grpc-go gates chatty per-RPC notices (e.g. "HandleStreams failed to read frame:
+  connection reset by peer") behind `LoggerV2.V(2)`. zapgrpc.Logger.V conflates
+  grpclog verbosity with zap severity, so V(2) returns true whenever WARN is
+  enabled and these messages emit at WARN. Wrap the installed grpclog.LoggerV2
+  with a corrected V() that compares against a fixed verbosity threshold,
+  matching grpclog's intended semantics. See uber-go/zap#1544.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/workflows/utils/cspell.json

@@ -260,6 +260,8 @@
       "groupbytrace",
       "groupbytraceprocessor",
       "grpclb",
+      "grpclog",
+      "grpclog's",
       "guiton",
       "healthcheck",
       "healthcheckextension",
@@ -295,6 +297,7 @@
       "koanf",
       "labeldrop",
       "ldflags",
+      "lifecycles",
       "limitermiddleware",
       "localhostgate",
       "loggingexporter",
@@ -533,6 +536,7 @@
       "yamlprovider",
       "yamls",
       "zapcore",
+      "zapgrpc",
       "zipkin",
       "zipkinexporter",
       "zipkinreceiver",

otelcol/internal/grpclog/logger.go

@@ -10,6 +10,26 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+// fixedVerbosityLogger wraps a grpclog.LoggerV2 and reports verbosity against a
+// fixed threshold instead of zap's severity enabler.
+//
+// zapgrpc.Logger.V(level) is implemented as levelEnabler.Enabled(zapcore.Level(level-1)),
+// which conflates grpclog's supplemental verbosity with zap's severity. With WARN
+// enabled, V(2) returns true, so grpc-go emits chatty per-RPC messages — including
+// transport-layer notices logged on normal client disconnect — at WARN. See
+// uber-go/zap#1544.
+//
+// Comparing against a fixed threshold restores grpclog's intended semantics.
+// The default of 0 matches grpclog when GRPC_GO_LOG_VERBOSITY_LEVEL is unset.
+type fixedVerbosityLogger struct {
+	grpclog.LoggerV2
+	verbosity int
+}
+
+func (l *fixedVerbosityLogger) V(level int) bool {
+	return level <= l.verbosity
+}
+
 // SetLogger constructs a zapgrpc.Logger instance, and installs it as grpc logger, cloned from baseLogger with
 // exact configuration. The minimum level of gRPC logs is set to WARN should the loglevel of the collector is set to
 // INFO to avoid copious logging from grpc framework.
@@ -30,6 +50,6 @@ func SetLogger(baseLogger *zap.Logger) *zapgrpc.Logger {
 		return c.With([]zapcore.Field{zap.Bool("grpc_log", true)})
 	}), zap.AddCallerSkip(5)))
 
-	grpclog.SetLoggerV2(logger)
+	grpclog.SetLoggerV2(&fixedVerbosityLogger{LoggerV2: logger})
 	return logger
 }

otelcol/internal/grpclog/logger_test.go

@@ -83,6 +83,50 @@ func TestGRPCLogger(t *testing.T) {
 	}
 }
 
+func TestFixedVerbosity(t *testing.T) {
+	// After SetLogger, the installed grpclog.LoggerV2 must report verbosity
+	// against a fixed threshold (default 0), not against the zap severity
+	// enabler. With the default threshold, V(0) is enabled and V(1+) is not,
+	// regardless of the underlying zap level.
+	logger, err := zap.NewProduction()
+	require.NoError(t, err)
+
+	SetLogger(logger)
+
+	v2 := grpclog.V(0)
+	assert.True(t, v2, "V(0) should be enabled at the default verbosity threshold")
+
+	v2 = grpclog.V(1)
+	assert.False(t, v2, "V(1) should be disabled at the default verbosity threshold")
+
+	v2 = grpclog.V(2)
+	assert.False(t, v2, "V(2) should be disabled at the default verbosity threshold")
+}
+
+func TestSeverityLevelsUnaffected(t *testing.T) {
+	// Severity-level emission must remain unchanged: a direct Warning call
+	// still fires regardless of the verbosity threshold.
+	warnLogged := false
+	hook := zap.Hooks(func(entry zapcore.Entry) error {
+		if entry.Level == zapcore.WarnLevel {
+			warnLogged = true
+		}
+		return nil
+	})
+
+	cfg := zap.Config{
+		Level:    zap.NewAtomicLevelAt(zapcore.WarnLevel),
+		Encoding: "console",
+	}
+	logger, err := cfg.Build(hook)
+	require.NoError(t, err)
+
+	SetLogger(logger)
+	component := &mockComponent{logger: grpclog.Component("channelz")}
+	component.Warning("real warning unrelated to verbosity")
+	assert.True(t, warnLogged, "severity-level warnings must still be emitted")
+}
+
 type mockComponent struct {
 	logger grpclog.DepthLoggerV2
 }