otelcol/grpclog: suppress benign gRPC client-disconnect warnings

When a gRPC client closes normally, the transport layer emits warnings
like "HandleStreams failed to read frame: EOF" and "connection reset by
peer". These are expected during graceful shutdown and create noise for
operators monitoring production collector deployments.

Add a benignFilterCore zapcore.Core wrapper that intercepts warn-level
entries matching known client-disconnect patterns and drops them before
they reach the output sink. Non-benign warnings are unaffected.

Fixes #5169

Author: kruthiwusirika5

otelcol/internal/grpclog/logger.go

@@ -4,15 +4,50 @@
 package grpclog // import "go.opentelemetry.io/collector/otelcol/internal/grpclog"
 
 import (
+	"strings"
+
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zapgrpc"
 	"google.golang.org/grpc/grpclog"
 )
 
+// benignClientClosePatterns are gRPC transport-layer messages that occur during normal
+// client disconnection and should not be surfaced as warnings to operators.
+var benignClientClosePatterns = []string{
+	"HandleStreams failed to read frame",
+	"connection reset by peer",
+}
+
+func isBenignClientCloseMessage(msg string) bool {
+	for _, pattern := range benignClientClosePatterns {
+		if strings.Contains(msg, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// benignFilterCore wraps a zapcore.Core to suppress known benign gRPC client-disconnect
+// warnings, reducing noise in production logs.
+type benignFilterCore struct {
+	zapcore.Core
+}
+
+func (c *benignFilterCore) With(fields []zapcore.Field) zapcore.Core {
+	return &benignFilterCore{c.Core.With(fields)}
+}
+
+func (c *benignFilterCore) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry {
+	if entry.Level == zapcore.WarnLevel && isBenignClientCloseMessage(entry.Message) {
+		return checked
+	}
+	return c.Core.Check(entry, checked)
+}
+
 // SetLogger constructs a zapgrpc.Logger instance, and installs it as grpc logger, cloned from baseLogger with
 // exact configuration. The minimum level of gRPC logs is set to WARN should the loglevel of the collector is set to
-// INFO to avoid copious logging from grpc framework.
+// INFO to avoid copious logging from grpc framework. Benign client-disconnect warnings are suppressed.
 func SetLogger(baseLogger *zap.Logger) *zapgrpc.Logger {
 	logger := zapgrpc.NewLogger(baseLogger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core {
 		var c zapcore.Core
@@ -27,7 +62,7 @@ func SetLogger(baseLogger *zap.Logger) *zapgrpc.Logger {
 		if err != nil {
 			c = core
 		}
-		return c.With([]zapcore.Field{zap.Bool("grpc_log", true)})
+		return &benignFilterCore{c.With([]zapcore.Field{zap.Bool("grpc_log", true)})}
 	}), zap.AddCallerSkip(5)))
 
 	grpclog.SetLoggerV2(logger)

otelcol/internal/grpclog/logger_test.go

@@ -83,6 +83,68 @@ func TestGRPCLogger(t *testing.T) {
 	}
 }
 
+func TestBenignClientCloseWarningsSuppressed(t *testing.T) {
+	benignMessages := []string{
+		"transport: http2Server.HandleStreams failed to read frame: EOF",
+		"transport: http2Server.HandleStreams failed to read frame: read tcp: connection reset by peer",
+		"[transport] connection reset by peer",
+	}
+
+	for _, msg := range benignMessages {
+		t.Run(msg, func(t *testing.T) {
+			warnLogged := false
+			hook := zap.Hooks(func(entry zapcore.Entry) error {
+				if entry.Level == zapcore.WarnLevel {
+					warnLogged = true
+				}
+				return nil
+			})
+
+			cfg := zap.Config{
+				Level:    zap.NewAtomicLevelAt(zapcore.WarnLevel),
+				Encoding: "console",
+			}
+			logger, err := cfg.Build(hook)
+			require.NoError(t, err)
+
+			SetLogger(logger)
+			component := &mockComponent{logger: grpclog.Component("transport")}
+			component.Warning(msg)
+
+			assert.False(t, warnLogged, "benign client-close message should be suppressed: %s", msg)
+		})
+	}
+}
+
+func TestNonBenignWarningsPreserved(t *testing.T) {
+	warnLogged := false
+	hook := zap.Hooks(func(entry zapcore.Entry) error {
+		if entry.Level == zapcore.WarnLevel {
+			warnLogged = true
+		}
+		return nil
+	})
+
+	cfg := zap.Config{
+		Level:    zap.NewAtomicLevelAt(zapcore.WarnLevel),
+		Encoding: "console",
+	}
+	logger, err := cfg.Build(hook)
+	require.NoError(t, err)
+
+	SetLogger(logger)
+	component := &mockComponent{logger: grpclog.Component("channelz")}
+	component.Warning("some unexpected grpc error unrelated to client close")
+	assert.True(t, warnLogged, "non-benign warning should still be logged at warn level")
+}
+
+func TestIsBenignClientCloseMessage(t *testing.T) {
+	assert.True(t, isBenignClientCloseMessage("HandleStreams failed to read frame: EOF"))
+	assert.True(t, isBenignClientCloseMessage("read tcp: connection reset by peer"))
+	assert.False(t, isBenignClientCloseMessage("unexpected internal error"))
+	assert.False(t, isBenignClientCloseMessage("dial tcp: connection refused"))
+}
+
 type mockComponent struct {
 	logger grpclog.DepthLoggerV2
 }