fix: remove implicit fallback behavior

ARCHITECTURE.md

@@ -1697,6 +1697,25 @@ ingestion (`ensureGapConsumedUtxos`, used while closing the range between the
 snapshot and the chain tip) is unconditionally strict already, since that range
 is always expected to be fully recoverable from the snapshot import.
 
+`StrictUtxoValidation` is one of a small family of opt-in fail-fast toggles
+(default off), each covering a specific point where the node otherwise
+continues on missing or unreadable runtime state. The tolerant default is
+appropriate for different reasons per toggle — sometimes bootstrap-only,
+sometimes an ordinarily-transient runtime failure — so they are individual
+flags rather than one global switch. `StrictLeaderEligibility`
+(`ledger/verify_header.go`) turns the warn-and-skip that fires when the total
+active stake is zero or the active slot coefficient is unavailable into a block
+rejection; here the tolerant path exists for genesis bootstrap, so enable it
+only on an established node where those inputs exist. `StrictSlotClock`
+(`ledger/state.go` `validateTxCore`) rejects a transaction when the slot clock
+cannot be read instead of falling back to the snapshot tip slot; here the
+tolerant path absorbs ordinarily-transient clock read failures during normal
+operation, so it may remain off when transient fallback is acceptable — but a
+*persistent* clock failure silently degrades validation to a stale
+`snapshotTipSlot`, so enable it to fail fast when clock failures persist. All
+three are wired identically: `internal/config` field + CLI flag →
+`dingo.WithX` option → the consuming component's config struct.
+
 The Mithril ledger-state snapshot slot normally lags the immutable-chunk tip, so
 `processPostLedgerStateBlocks`/`processGapBlocks` ingest the blocks in between
 (the "gap blocks") for their transaction effects, including consumed-input

DATABASE.md

@@ -41,6 +41,16 @@ flowchart LR
     Blob --> KV
 ```
 
+### Plugin Configuration Validation
+
+`plugin.ProcessConfig` (`database/plugin/register.go`) strictly validates the `database.blob.*` and `database.metadata.*` sections of the YAML config against the plugins actually compiled into the running binary:
+
+- For a plugin name that matches a compiled-in plugin (e.g. `badger`, always built; `sqlite`, always built), every key under that section must match one of that plugin's declared options. An unrecognized key (e.g. `database.blob.badger.buckit`, a typo for `bucket`) fails config load instead of being silently ignored.
+- For a plugin name that does not match any compiled-in plugin, config load fails with `unknown <blob|metadata> plugin "<name>" in config` (e.g. a typo'd section name such as `badgre`), **unless** the name is one of the build-tag-gated plugins (`gcs`, `s3` for blob; `mysql`, `postgres` for metadata) that are only compiled in under `-tags dingo_extra_plugins`. A default build tolerates a `database.blob.gcs:`/`database.blob.s3:`/`database.metadata.mysql:`/`database.metadata.postgres:` section without validating its keys, since the plugin (and its option list) isn't registered in that build to validate against; selecting one of these as the active `BlobPlugin`/`MetadataPlugin` still fails at startup with `MissingPluginError` regardless of build.
+- Building with `dingo_extra_plugins` registers `gcs`/`s3`/`mysql`/`postgres` like any other plugin, so their sections then get the same strict per-key validation as `badger`/`sqlite`.
+
+This means the same config file can behave differently across builds: a typo under an extra-plugin section (e.g. `database.blob.gcs.buckit`) is only caught in an extra-plugins build, since a default build has no registered option list for `gcs` to check against.
+
 ## SQL Conventions
 
 - Table and column names are snake_case GORM names unless a model has an explicit `gorm:"column:..."` tag.

README.md

@@ -428,15 +428,16 @@ Leave the mode-sensitive Badger settings unset if you want Dingo's storage-mode
 
 Google Cloud Storage Options:
 - `bucket` - GCS bucket name
-- `project-id` - Google Cloud project ID
-- `prefix` - Path prefix within bucket
+
+Project and credentials are always resolved via Application Default Credentials; there is no plugin config option for project ID or a bucket key prefix.
 
 AWS S3 Options:
+- `endpoint` - S3 endpoint
 - `bucket` - S3 bucket name
 - `region` - AWS region
 - `prefix` - Path prefix within bucket
-- `access-key-id` - AWS access key ID (optional - uses default credential chain if not provided)
-- `secret-access-key` - AWS secret access key (optional - uses default credential chain if not provided)
+
+Credentials are always resolved via the AWS SDK's default credential chain (env vars, shared config, IAM role, etc.); there is no plugin config option for static credentials.
 
 SQLite Options:
 - `data-dir` - Path to SQLite database file

api/utxorpc/submit.go

@@ -132,6 +132,7 @@ func (s *submitServiceServer) WaitForTx(
 			if !ok {
 				s.utxorpc.config.Logger.Warn(
 					"unexpected event data type in WaitForTx",
+					"type", fmt.Sprintf("%T", evt.Data),
 				)
 				return
 			}
@@ -398,6 +399,10 @@ func (s *submitServiceServer) WatchMempool(
 			}()
 			addEvt, ok := evt.Data.(mempool.AddTransactionEvent)
 			if !ok {
+				s.utxorpc.config.Logger.Warn(
+					"unexpected event data type in WatchMempool",
+					"type", fmt.Sprintf("%T", evt.Data),
+				)
 				return
 			}
 			txRawBytes := addEvt.Body

chainselection/selector.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"log/slog"
 	"math"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -1412,9 +1413,12 @@ func (cs *ChainSelector) evaluationLoop() {
 func (cs *ChainSelector) runEvaluationTick() {
 	defer func() {
 		if r := recover(); r != nil {
+			// recover() suppresses the runtime's own stack trace, so
+			// capture one here or the panic becomes undiagnosable.
 			cs.config.Logger.Error(
 				"panic in evaluation tick, continuing",
 				"panic", r,
+				"stack", string(debug.Stack()),
 			)
 		}
 	}()
@@ -1428,6 +1432,7 @@ func (cs *ChainSelector) runTriggeredEvaluation() {
 			cs.config.Logger.Error(
 				"panic in triggered evaluation, continuing",
 				"panic", r,
+				"stack", string(debug.Stack()),
 			)
 		}
 	}()

config.go

@@ -147,6 +147,8 @@ type Config struct {
 	peerSharing              bool
 	validateHistorical       bool
 	strictUtxoValidation     bool
+	strictLeaderEligibility  bool
+	strictSlotClock          bool
 	tracing                  bool
 	tracingStdout            bool
 	runMode                  string
@@ -719,6 +721,25 @@ func WithStrictUtxoValidation(strict bool) ConfigOptionFunc {
 	}
 }
 
+// WithStrictLeaderEligibility specifies whether a block is rejected (rather
+// than warned about and skipped) when the total active stake is zero or the
+// active slot coefficient is unavailable or non-positive during Praos leader
+// eligibility checking. See ledger.LedgerStateConfig.StrictLeaderEligibility.
+func WithStrictLeaderEligibility(strict bool) ConfigOptionFunc {
+	return func(c *Config) {
+		c.strictLeaderEligibility = strict
+	}
+}
+
+// WithStrictSlotClock specifies whether a transaction is rejected (rather than
+// validated against the snapshot tip slot) when the slot clock cannot be read
+// during validation. See ledger.LedgerStateConfig.StrictSlotClock.
+func WithStrictSlotClock(strict bool) ConfigOptionFunc {
+	return func(c *Config) {
+		c.strictSlotClock = strict
+	}
+}
+
 // WithDatabaseWorkerPoolConfig specifies the database worker pool configuration
 func WithDatabaseWorkerPoolConfig(
 	cfg ledger.DatabaseWorkerPoolConfig,

connmanager/connection_manager.go

@@ -17,6 +17,7 @@ package connmanager
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log/slog"
 	"net"
@@ -835,6 +836,13 @@ func (c *ConnectionManager) HandleConnectionRecycleRequestedEvent(
 ) {
 	e, ok := evt.Data.(ConnectionRecycleRequestedEvent)
 	if !ok {
+		// A mismatched event payload means a recycle request is being
+		// dropped silently; log it rather than swallowing, matching the
+		// event-type-assertion handling elsewhere (peergov, ouroboros).
+		c.config.Logger.Warn(
+			"unexpected event data type in HandleConnectionRecycleRequestedEvent",
+			"type", fmt.Sprintf("%T", evt.Data),
+		)
 		return
 	}
 	conn := c.GetConnectionById(e.ConnectionId)

connmanager/outbound.go

@@ -45,12 +45,30 @@ func (c *ConnectionManager) CreateOutboundConn(
 	if c.config.OutboundSourcePort > 0 {
 		// Setup connection to use our listening port as the source port
 		// This is required for peer sharing to be useful
-		clientAddr, _ = net.ResolveTCPAddr(
+		resolvedAddr, resolveErr := net.ResolveTCPAddr(
 			"tcp",
 			fmt.Sprintf(":%d", c.config.OutboundSourcePort),
 		)
-		dialer.LocalAddr = clientAddr
-		dialer.Control = socketControl
+		if resolveErr != nil {
+			// internal/config.ValidateRuntimeConfig rejects an
+			// out-of-range relayPort (which OutboundSourcePort is
+			// derived from) once CLI/env/YAML have all merged, so this
+			// should be unreachable via normal configuration. Kept as a
+			// backstop for direct API callers of this package that set
+			// OutboundSourcePort without going through internal/config
+			// at all.
+			c.config.Logger.Warn(
+				"outbound: failed to resolve source port, dialing without source-port reuse",
+				"role", "client",
+				"address", address,
+				"outbound_source_port", c.config.OutboundSourcePort,
+				"error", resolveErr,
+			)
+		} else {
+			clientAddr = resolvedAddr
+			dialer.LocalAddr = clientAddr
+			dialer.Control = socketControl
+		}
 	}
 	c.config.Logger.Debug(
 		"establishing TCP connection to: "+address,

connmanager/outbound_test.go

@@ -0,0 +1,55 @@
+// Copyright 2026 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connmanager
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestCreateOutboundConn_InvalidSourcePortLogsWarning verifies that an
+// out-of-range OutboundSourcePort (which makes net.ResolveTCPAddr fail)
+// is logged instead of silently disabling source-port reuse and dialing
+// without a source-port bind.
+func TestCreateOutboundConn_InvalidSourcePortLogsWarning(t *testing.T) {
+	var logBuf bytes.Buffer
+	cm := NewConnectionManager(ConnectionManagerConfig{
+		Logger: slog.New(
+			slog.NewTextHandler(&logBuf, nil),
+		),
+		// Out of the valid 16-bit TCP port range, so
+		// net.ResolveTCPAddr fails.
+		OutboundSourcePort: 100000,
+	})
+
+	ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+	defer cancel()
+
+	// The dial itself is expected to fail (nothing listening), but the
+	// resolve failure must be logged rather than silently ignored.
+	_, _ = cm.CreateOutboundConn(ctx, "127.0.0.1:1")
+
+	logOutput := logBuf.String()
+	if !strings.Contains(logOutput, "failed to resolve source port") {
+		t.Errorf(
+			"expected a source-port resolve warning in logs, got: %s",
+			logOutput,
+		)
+	}
+}

connmanager/recycle_event_test.go

@@ -0,0 +1,46 @@
+// Copyright 2026 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connmanager
+
+import (
+	"bytes"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/blinklabs-io/dingo/event"
+)
+
+// TestHandleConnectionRecycleRequestedEvent_UnexpectedTypeLogged verifies
+// that a mismatched event payload is logged rather than silently dropped:
+// a recycle request lost with no trace is exactly the kind of swallowed
+// failure this handler used to allow.
+func TestHandleConnectionRecycleRequestedEvent_UnexpectedTypeLogged(t *testing.T) {
+	var logBuf bytes.Buffer
+	cm := NewConnectionManager(ConnectionManagerConfig{
+		Logger: slog.New(slog.NewTextHandler(&logBuf, nil)),
+	})
+
+	// Wrong payload type: the handler asserts ConnectionRecycleRequestedEvent.
+	cm.HandleConnectionRecycleRequestedEvent(event.Event{Data: "not-a-recycle-event"})
+
+	logOutput := logBuf.String()
+	if !strings.Contains(logOutput, "unexpected event data type") {
+		t.Errorf(
+			"expected an unexpected-event-type warning in logs, got: %s",
+			logOutput,
+		)
+	}
+}

database/database.go

@@ -29,6 +29,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+// DefaultConfig is the source of truth for New's plugin defaulting below,
+// used by direct callers of this package (library usage, tests) that
+// don't go through internal/config. The CLI path resolves BlobPlugin/
+// MetadataPlugin earlier via the internal/config.DefaultBlobPlugin/
+// DefaultMetadataPlugin constants, which are kept in sync with these
+// values by TestDatabaseDefaultsMatchInternalConfigDefaults in
+// internal/config.
 var DefaultConfig = &Config{
 	BlobPlugin:     "badger",
 	DataDir:        ".dingo",
@@ -164,9 +171,21 @@ func New(
 	// Apply defaults for empty fields
 	if configCopy.BlobPlugin == "" {
 		configCopy.BlobPlugin = DefaultConfig.BlobPlugin
+		if configCopy.Logger != nil {
+			configCopy.Logger.Debug(
+				"no blob plugin configured, using default",
+				"plugin", configCopy.BlobPlugin,
+			)
+		}
 	}
 	if configCopy.MetadataPlugin == "" {
 		configCopy.MetadataPlugin = DefaultConfig.MetadataPlugin
+		if configCopy.Logger != nil {
+			configCopy.Logger.Debug(
+				"no metadata plugin configured, using default",
+				"plugin", configCopy.MetadataPlugin,
+			)
+		}
 	}
 	if configCopy.StorageMode == "" {
 		configCopy.StorageMode = types.StorageModeCore

database/default_plugin_log_test.go

@@ -0,0 +1,50 @@
+// Copyright 2026 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package database
+
+import (
+	"bytes"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestNew_DefaultPluginSelectionIsLogged verifies that silently defaulting
+// BlobPlugin/MetadataPlugin to badger/sqlite when unset is still observable
+// via a debug log, rather than happening with zero trace.
+func TestNew_DefaultPluginSelectionIsLogged(t *testing.T) {
+	var logBuf bytes.Buffer
+	config := &Config{
+		DataDir: "", // in-memory
+		Logger: slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{
+			Level: slog.LevelDebug,
+		})),
+	}
+	db, err := New(config)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, db.Close())
+	})
+
+	logOutput := logBuf.String()
+	if !strings.Contains(logOutput, "no blob plugin configured, using default") {
+		t.Errorf("expected default blob plugin log, got: %s", logOutput)
+	}
+	if !strings.Contains(logOutput, "no metadata plugin configured, using default") {
+		t.Errorf("expected default metadata plugin log, got: %s", logOutput)
+	}
+}