[Logging] Add support for component scoped log-levels

.gitignore

@@ -6,6 +6,9 @@
 /cmd/util/util
 /cmd/bootstrap/bootstrap
 
+# omit agent plans files
+/docs/plans
+
 # Test ouput of  bootstrapping CLI
 cmd/bootstrap/bootstrap-example
 

admin/commands/common/component_log_level_reset.go

@@ -0,0 +1,79 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/utils/logging"
+)
+
+var _ commands.AdminCommand = (*ResetComponentLogLevelCommand)(nil)
+
+// ResetComponentLogLevelCommand removes runtime log level overrides for components matching
+// the specified patterns, restoring them to static config or global default.
+//
+// Input is a JSON array of patterns. ["*"] resets all registered components.
+// "*" may not be mixed with other patterns.
+//
+// Example input:
+//
+//	["hotstuff.voter", "hotstuff.*"]
+//	["*"]
+type ResetComponentLogLevelCommand struct {
+	registry *logging.LogRegistry
+}
+
+// NewResetComponentLogLevelCommand constructs a ResetComponentLogLevelCommand.
+func NewResetComponentLogLevelCommand(registry *logging.LogRegistry) *ResetComponentLogLevelCommand {
+	return &ResetComponentLogLevelCommand{registry: registry}
+}
+
+// Validator validates that the input is a non-empty array of pattern strings. "*" must be
+// the sole element if present.
+//
+// Returns [admin.InvalidAdminReqError] for invalid or malformed requests.
+func (r *ResetComponentLogLevelCommand) Validator(req *admin.CommandRequest) error {
+	raw, ok := req.Data.([]interface{})
+	if !ok {
+		return admin.NewInvalidAdminReqFormatError("input must be a JSON array of pattern strings")
+	}
+	if len(raw) == 0 {
+		return admin.NewInvalidAdminReqFormatError("input must not be empty")
+	}
+
+	patterns := make([]string, 0, len(raw))
+	for _, v := range raw {
+		pattern, ok := v.(string)
+		if !ok {
+			return admin.NewInvalidAdminReqFormatError("each element must be a string")
+		}
+		pattern = logging.NormalizePattern(strings.TrimSpace(pattern))
+		if pattern == "*" {
+			if len(patterns) > 1 {
+				return admin.NewInvalidAdminReqErrorf("\"*\" must be the only pattern when resetting all components")
+			}
+		} else {
+			if err := logging.ValidatePattern(pattern); err != nil {
+				return admin.NewInvalidAdminReqErrorf("invalid pattern %q: %w", pattern, err)
+			}
+		}
+		patterns = append(patterns, pattern)
+	}
+
+	req.ValidatorData = patterns
+	return nil
+}
+
+// Handler removes the specified runtime overrides and returns "ok".
+//
+// No error returns are expected during normal operation.
+func (r *ResetComponentLogLevelCommand) Handler(_ context.Context, req *admin.CommandRequest) (interface{}, error) {
+	patterns := req.ValidatorData.([]string)
+	if err := r.registry.Reset(patterns...); err != nil {
+		return nil, fmt.Errorf("failed to reset component log levels: %w", err)
+	}
+	return "ok", nil
+}

admin/commands/common/component_log_level_set.go

@@ -0,0 +1,96 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/utils/logging"
+)
+
+const (
+	maxPatternLength = 1024
+)
+
+var _ commands.AdminCommand = (*SetComponentLogLevelCommand)(nil)
+
+// SetComponentLogLevelCommand sets the log level for one or more components identified by
+// exact or wildcard patterns. Input is a JSON object mapping pattern to level string.
+//
+// Example input:
+//
+//	{"hotstuff.voter": "debug", "network.*": "warn"}
+type SetComponentLogLevelCommand struct {
+	registry *logging.LogRegistry
+}
+
+// NewSetComponentLogLevelCommand constructs a SetComponentLogLevelCommand.
+func NewSetComponentLogLevelCommand(registry *logging.LogRegistry) *SetComponentLogLevelCommand {
+	return &SetComponentLogLevelCommand{registry: registry}
+}
+
+type parsedComponentLevel struct {
+	pattern string
+	level   zerolog.Level
+}
+
+// Validator validates that the input is a non-empty map of pattern → level string with
+// recognisable level values.
+//
+// Returns [admin.InvalidAdminReqError] for invalid or malformed requests.
+func (s *SetComponentLogLevelCommand) Validator(req *admin.CommandRequest) error {
+	raw, ok := req.Data.(map[string]interface{})
+	if !ok {
+		return admin.NewInvalidAdminReqFormatError("input must be a JSON object mapping component pattern to level string")
+	}
+	if len(raw) == 0 {
+		return admin.NewInvalidAdminReqFormatError("input must not be empty")
+	}
+
+	parsed := make([]parsedComponentLevel, 0, len(raw))
+	for pattern, val := range raw {
+		levelStr, ok := val.(string)
+		if !ok {
+			return admin.NewInvalidAdminReqErrorf("level for %q must be a string", pattern)
+		}
+		level, err := zerolog.ParseLevel(levelStr)
+		if err != nil {
+			return admin.NewInvalidAdminReqErrorf("invalid level %q for component %q: %w", levelStr, pattern, err)
+		}
+		if len(pattern) > maxPatternLength {
+			return admin.NewInvalidAdminReqErrorf("pattern %q is too long (max %d characters)", pattern, maxPatternLength)
+		}
+		pattern = logging.NormalizePattern(strings.TrimSpace(pattern))
+		if pattern == "*" {
+			return admin.NewInvalidAdminReqErrorf("global wildcard \"*\" is not a valid when setting component level logging. use set-log-level instead")
+		}
+		if err := logging.ValidatePattern(logging.NormalizePattern(pattern)); err != nil {
+			return admin.NewInvalidAdminReqErrorf("invalid pattern %q: %w", pattern, err)
+		}
+
+		parsed = append(parsed, parsedComponentLevel{pattern: pattern, level: level})
+	}
+
+	req.ValidatorData = parsed
+	return nil
+}
+
+// Handler applies the validated component level overrides and returns the updated patterns.
+//
+// No error returns are expected during normal operation.
+func (s *SetComponentLogLevelCommand) Handler(_ context.Context, req *admin.CommandRequest) (interface{}, error) {
+	entries := req.ValidatorData.([]parsedComponentLevel)
+
+	result := make(map[string]string, len(entries))
+	for _, e := range entries {
+		if err := s.registry.SetLevel(e.pattern, e.level); err != nil {
+			return nil, fmt.Errorf("failed to set level for pattern %q: %w", e.pattern, err)
+		}
+		result[e.pattern] = fmt.Sprintf("set to %s", e.level)
+	}
+	return result, nil
+}

admin/commands/common/component_log_levels_get.go

@@ -0,0 +1,66 @@
+package common
+
+import (
+	"context"
+
+	"github.com/rs/zerolog"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/utils/logging"
+)
+
+var _ commands.AdminCommand = (*GetComponentLogLevelsCommand)(nil)
+
+// GetComponentLogLevelsCommand returns the current log level for every registered component
+// and the global default.
+type GetComponentLogLevelsCommand struct {
+	registry *logging.LogRegistry
+}
+
+// NewGetComponentLogLevelsCommand constructs a GetComponentLogLevelsCommand.
+func NewGetComponentLogLevelsCommand(registry *logging.LogRegistry) *GetComponentLogLevelsCommand {
+	return &GetComponentLogLevelsCommand{registry: registry}
+}
+
+// Validator performs no validation — this command takes no input.
+//
+// No error returns are expected during normal operation.
+func (g *GetComponentLogLevelsCommand) Validator(_ *admin.CommandRequest) error {
+	return nil
+}
+
+// Handler returns a snapshot of all registered component log levels and the registry config.
+//
+// No error returns are expected during normal operation.
+func (g *GetComponentLogLevelsCommand) Handler(_ context.Context, _ *admin.CommandRequest) (interface{}, error) {
+	_, levels := g.registry.Levels()
+	cfg := g.registry.Config()
+
+	components := make(map[string]interface{}, len(levels))
+	for id, cl := range levels {
+		components[id] = map[string]string{
+			"level":  zerolog.Level(cl.Level).String(),
+			"source": string(cl.Source),
+		}
+	}
+
+	staticOverrides := make(map[string]string, len(cfg.StaticOverrides))
+	for pattern, level := range cfg.StaticOverrides {
+		staticOverrides[pattern] = level.String()
+	}
+
+	dynamicOverrides := make(map[string]string, len(cfg.DynamicOverrides))
+	for pattern, level := range cfg.DynamicOverrides {
+		dynamicOverrides[pattern] = level.String()
+	}
+
+	return map[string]interface{}{
+		"config": map[string]interface{}{
+			"default":           cfg.Default.String(),
+			"static_overrides":  staticOverrides,
+			"dynamic_overrides": dynamicOverrides,
+		},
+		"components": components,
+	}, nil
+}

admin/commands/common/set_log_level.go → admin/commands/common/log_level_set.go

@@ -8,22 +8,35 @@ import (
 
 	"github.com/onflow/flow-go/admin"
 	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/utils/logging"
 )
 
 var _ commands.AdminCommand = (*SetLogLevelCommand)(nil)
 
-type SetLogLevelCommand struct{}
+// SetLogLevelCommand sets the global default log level. Components with per-component overrides
+// (from the CLI flag or a prior set-component-log-level command) are unaffected.
+type SetLogLevelCommand struct {
+	registry *logging.LogRegistry
+}
+
+// NewSetLogLevelCommand constructs a SetLogLevelCommand.
+func NewSetLogLevelCommand(registry *logging.LogRegistry) *SetLogLevelCommand {
+	return &SetLogLevelCommand{registry: registry}
+}
 
+// Handler sets the global default level via the registry.
+//
+// No error returns are expected during normal operation.
 func (s *SetLogLevelCommand) Handler(_ context.Context, req *admin.CommandRequest) (interface{}, error) {
 	level := req.ValidatorData.(zerolog.Level)
-	zerolog.SetGlobalLevel(level)
-
-	log.Info().Msgf("changed log level to %v", level)
+	s.registry.SetDefaultLevel(level)
+	log.Info().Msgf("changed default log level to %v", level)
 	return "ok", nil
 }
 
 // Validator validates the request.
-// Returns admin.InvalidAdminReqError for invalid/malformed requests.
+//
+// Returns [admin.InvalidAdminReqError] for invalid or malformed requests.
 func (s *SetLogLevelCommand) Validator(req *admin.CommandRequest) error {
 	level, ok := req.Data.(string)
 	if !ok {

cmd/node_builder.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"io"
 	"time"
 
 	"github.com/dgraph-io/badger/v2"
@@ -29,6 +30,7 @@ import (
 	"github.com/onflow/flow-go/state/protocol/events"
 	"github.com/onflow/flow-go/storage"
 	bstorage "github.com/onflow/flow-go/storage/badger"
+	"github.com/onflow/flow-go/utils/logging"
 )
 
 const NotSet = "not set"
@@ -158,6 +160,7 @@ type BaseConfig struct {
 	secretsDBEnabled            bool
 	InsecureSecretsDB           bool
 	level                       string
+	componentLogLevels          string
 	debugLogLimit               uint32
 	metricsPort                 uint
 	BootstrapDir                string
@@ -195,6 +198,8 @@ type NodeConfig struct {
 	Cancel context.CancelFunc // cancel function for the context that is passed to the networking layer
 	BaseConfig
 	Logger            zerolog.Logger
+	LogWriter         io.Writer
+	LogRegistry       *logging.LogRegistry
 	NodeID            flow.Identifier
 	Me                module.Local
 	Tracer            module.Tracer

cmd/scaffold.go

@@ -176,6 +176,7 @@ func (fnb *FlowNodeBuilder) BaseFlags() {
 	fnb.flags.StringVar(&fnb.BaseConfig.pebbleCheckpointsDir, "pebble-checkpoints-dir", defaultConfig.pebbleCheckpointsDir, "directory to store the checkpoints for the public pebble database (protocol state)")
 	fnb.flags.StringVar(&fnb.BaseConfig.secretsdir, "secretsdir", defaultConfig.secretsdir, "directory to store private database (secrets)")
 	fnb.flags.StringVarP(&fnb.BaseConfig.level, "loglevel", "l", defaultConfig.level, "level for logging output")
+	fnb.flags.StringVar(&fnb.BaseConfig.componentLogLevels, "component-log-levels", "", `per-component log levels, format: "component:level,prefix.*:level", e.g. "hotstuff:debug,network.*:warn"`)
 	fnb.flags.Uint32Var(&fnb.BaseConfig.debugLogLimit, "debug-log-limit", defaultConfig.debugLogLimit, "max number of debug/trace log events per second")
 	fnb.flags.UintVarP(&fnb.BaseConfig.metricsPort, "metricport", "m", defaultConfig.metricsPort, "port for /metrics endpoint")
 	fnb.flags.BoolVar(&fnb.BaseConfig.profilerConfig.Enabled, "profiler-enabled", defaultConfig.profilerConfig.Enabled, "whether to enable the auto-profiler")
@@ -910,12 +911,20 @@ func (fnb *FlowNodeBuilder) initLogger() error {
 		return fmt.Errorf("invalid log level: %w", err)
 	}
 
-	// Minimum log level is set to trace, then overridden by SetGlobalLevel.
-	// this allows admin commands to modify the level to any value during runtime
+	staticConfig, err := logging.ParseComponentLogLevels(fnb.BaseConfig.componentLogLevels)
+	if err != nil {
+		return fmt.Errorf("invalid --component-log-levels: %w", err)
+	}
+
+	// Logger level is set to trace; all filtering is owned by LogRegistry.
 	log = log.Level(zerolog.TraceLevel)
-	zerolog.SetGlobalLevel(lvl)
 
-	fnb.Logger = log
+	fnb.LogRegistry = logging.NewLogRegistry(fnb.LogWriter, lvl, staticConfig)
+
+	// setup the main logger under a default component. this ensures any logs emitted under the default
+	// logger remain at the set global level, even if there are other components with lower levels.
+	// for details on why this is necessary, see the LogRegistry documentation.
+	fnb.Logger = fnb.LogRegistry.Logger(log, "default")
 
 	return nil
 }
@@ -1968,10 +1977,13 @@ func FlowNode(role string, opts ...Option) *FlowNodeBuilder {
 		opt(config)
 	}
 
+	logWriter := os.Stderr
+
 	builder := &FlowNodeBuilder{
 		NodeConfig: &NodeConfig{
 			BaseConfig:              *config,
-			Logger:                  zerolog.New(os.Stderr),
+			Logger:                  zerolog.New(logWriter),
+			LogWriter:               logWriter,
 			PeerManagerDependencies: NewDependencyList(),
 			ConfigManager:           updatable_configs.NewManager(),
 		},
@@ -2015,7 +2027,13 @@ func (fnb *FlowNodeBuilder) Initialize() error {
 
 func (fnb *FlowNodeBuilder) RegisterDefaultAdminCommands() {
 	fnb.AdminCommand("set-log-level", func(config *NodeConfig) commands.AdminCommand {
-		return &common.SetLogLevelCommand{}
+		return common.NewSetLogLevelCommand(config.LogRegistry)
+	}).AdminCommand("set-component-log-level", func(config *NodeConfig) commands.AdminCommand {
+		return common.NewSetComponentLogLevelCommand(config.LogRegistry)
+	}).AdminCommand("reset-component-log-level", func(config *NodeConfig) commands.AdminCommand {
+		return common.NewResetComponentLogLevelCommand(config.LogRegistry)
+	}).AdminCommand("get-component-log-levels", func(config *NodeConfig) commands.AdminCommand {
+		return common.NewGetComponentLogLevelsCommand(config.LogRegistry)
 	}).AdminCommand("set-golog-level", func(config *NodeConfig) commands.AdminCommand {
 		return &common.SetGologLevelCommand{}
 	}).AdminCommand("get-config", func(config *NodeConfig) commands.AdminCommand {