fix(cbor): tolerate duplicate map keys in pre-Conway MultiAsset decode (#1850)

Signed-off-by: Chris Gianelloni <wolf31o2@blinklabs.io>

Author: wolf31o2

cbor/decode.go

@@ -34,6 +34,10 @@ var (
 	cachedStrictDecMode     _cbor.DecMode
 	cachedStrictDecModeErr  error
 	cachedStrictDecModeOnce sync.Once
+
+	cachedLenientDecMode     _cbor.DecMode
+	cachedLenientDecModeErr  error
+	cachedLenientDecModeOnce sync.Once
 )
 
 const cborMaxNestedLevels = 256
@@ -73,6 +77,13 @@ func Decode(dataBytes []byte, dest any) (int, error) {
 	return dec.NumBytesRead(), err
 }
 
+// IsDuplicateMapKeyError returns true when err wraps fxamacker's duplicate map
+// key error.
+func IsDuplicateMapKeyError(err error) bool {
+	var dupErr *_cbor.DupMapKeyError
+	return errors.As(err, &dupErr)
+}
+
 // getStrictDecMode returns a cached DecMode with stricter limits for untrusted
 // network data. Uses sync.Once for thread-safe lazy initialization.
 // Returns the cached error if initialization failed.
@@ -109,6 +120,57 @@ func DecodeStrict(dataBytes []byte, dest any) (int, error) {
 	return dec.NumBytesRead(), err
 }
 
+// getLenientDecMode returns a cached DecMode that mirrors getDecMode() but keeps
+// DupMapKeyQuiet (last-wins) instead of DupMapKeyEnforcedAPF. This matches the
+// pre-Conway (protocol version < 9) cardano-ledger map decoding semantics, which
+// use Map.fromList (later duplicate key wins, earlier discarded) rather than
+// rejecting duplicates. Uses sync.Once for thread-safe lazy initialization.
+// Returns the cached error if initialization failed.
+func getLenientDecMode() (_cbor.DecMode, error) {
+	cachedLenientDecModeOnce.Do(func() {
+		decOptions := _cbor.DecOptions{
+			ExtraReturnErrors: _cbor.ExtraDecErrorUnknownField,
+			// DupMapKeyQuiet: keep the last value for a duplicate key without
+			// erroring. fxamacker decodes into a Go map via SetMapIndex per
+			// entry, so a repeated key overwrites, which is byte-for-byte the
+			// same resolution as cardano-ledger's Map.fromList for PV < 9.
+			DupMapKey:       _cbor.DupMapKeyQuiet,
+			MaxNestedLevels: cborMaxNestedLevels,
+			// Match getDecMode() limits: Cardano ledger state snapshots contain
+			// stake distribution maps that can exceed 1M entries on mainnet.
+			MaxMapPairs:      10_000_000,
+			MaxArrayElements: 10_000_000,
+		}
+		cachedLenientDecMode, cachedLenientDecModeErr = decOptions.DecModeWithTags(customTagSet)
+	})
+	return cachedLenientDecMode, cachedLenientDecModeErr
+}
+
+// DecodeLenient decodes CBOR data without rejecting duplicate map keys. How a
+// duplicate key is resolved is destination-type dependent, because it is
+// delegated to the underlying CBOR library's DupMapKeyQuiet mode, which "uses
+// faster of keep first or keep last depending on Go data type": decoding into a
+// Go map (MultiAsset's only current use) resolves last-wins, matching pre-Conway
+// (protocol version < 9) cardano-ledger Map.fromList semantics; decoding into a
+// Go struct may instead keep the first value written for a repeated field.
+// Callers that require last-wins semantics MUST therefore decode into a map, not
+// a struct. It is intended only for structures where Cardano consensus
+// historically permitted duplicate keys (e.g. pre-Conway MultiAsset value/mint
+// maps). Use Decode (strict, rejects duplicates) everywhere else.
+func DecodeLenient(dataBytes []byte, dest any) (int, error) {
+	data := bytes.NewReader(dataBytes)
+	decMode, err := getLenientDecMode()
+	if err != nil {
+		return 0, err
+	}
+	if decMode == nil {
+		return 0, errors.New("CBOR lenient decoder mode not initialized")
+	}
+	dec := decMode.NewDecoder(data)
+	err = dec.Decode(dest)
+	return dec.NumBytesRead(), err
+}
+
 // Extract the first item from a CBOR list. This will return the first item from the
 // provided list if it's numeric and an error otherwise
 func DecodeIdFromList(cborData []byte) (int, error) {

ledger/common/common.go

@@ -267,7 +267,8 @@ type (
 // MultiAsset represents a collection of policies, assets, and quantities. It's used for
 // TX outputs (uint64) and TX asset minting (int64 to allow for negative values for burning)
 type MultiAsset[T int64 | uint64 | *big.Int] struct {
-	data map[Blake2b224]map[cbor.ByteString]T
+	data             map[Blake2b224]map[cbor.ByteString]T
+	duplicateMapKeys bool
 }
 
 // NewMultiAsset creates a MultiAsset with the specified data
@@ -290,8 +291,27 @@ type multiAssetJson[T int64 | uint64 | *big.Int] struct {
 }
 
 func (m *MultiAsset[T]) UnmarshalCBOR(data []byte) error {
-	_, err := cbor.Decode(data, &(m.data))
-	return err
+	var decoded map[Blake2b224]map[cbor.ByteString]T
+	m.duplicateMapKeys = false
+	if _, err := cbor.Decode(data, &decoded); err == nil {
+		m.data = decoded
+		return nil
+	} else if !cbor.IsDuplicateMapKeyError(err) {
+		return err
+	}
+
+	// Pre-Conway (protocol version < 9) cardano-ledger decodes value/mint maps
+	// with Map.fromList, accepting duplicate PolicyID / AssetName keys and
+	// keeping the last one. Such transactions exist on canonical pre-Conway
+	// chains, so keep lenient last-wins decoding while recording that Conway+
+	// era decoders must reject this value.
+	m.duplicateMapKeys = true
+	decoded = nil
+	if _, err := cbor.DecodeLenient(data, &decoded); err != nil {
+		return err
+	}
+	m.data = decoded
+	return nil
 }
 
 func (m *MultiAsset[T]) MarshalCBOR() ([]byte, error) {
@@ -300,6 +320,13 @@ func (m *MultiAsset[T]) MarshalCBOR() ([]byte, error) {
 	return cbor.Encode(m.data)
 }
 
+func (m *MultiAsset[T]) CheckForDuplicateKeys() error {
+	if m != nil && m.duplicateMapKeys {
+		return errors.New("duplicate map key in multiasset")
+	}
+	return nil
+}
+
 func (m MultiAsset[T]) MarshalJSON() ([]byte, error) {
 	tmpAssets := make([]multiAssetJson[T], 0, len(m.data))
 	for policyId, policyData := range m.data {
@@ -325,6 +352,7 @@ func (m *MultiAsset[T]) UnmarshalJSON(data []byte) error {
 	if err := json.Unmarshal(data, &tmpAssets); err != nil {
 		return err
 	}
+	m.duplicateMapKeys = false
 	if m.data == nil {
 		m.data = make(map[Blake2b224]map[cbor.ByteString]T)
 	}

ledger/common/common_test.go

@@ -1406,3 +1406,96 @@ func TestAddInt64Checked(t *testing.T) {
 		})
 	}
 }
+
+// TestMultiAssetUnmarshalDuplicatePolicyLastWins verifies that decoding a
+// MultiAsset whose outer (PolicyID) map contains a duplicate key does not error
+// and resolves last-wins, matching pre-Conway (protocol version < 9)
+// cardano-ledger Map.fromList semantics. Such transactions exist on the
+// canonical preprod Babbage chain, so rejecting them would wedge sync.
+func TestMultiAssetUnmarshalDuplicatePolicyLastWins(t *testing.T) {
+	// 28-byte PolicyID, repeated as two outer map entries.
+	policyBytes := bytes.Repeat([]byte{0x11}, Blake2b224Size)
+	policyHex := hex.EncodeToString(policyBytes)
+	// a2                              outer map, 2 pairs
+	//   581c <28-byte policy>         key: PolicyID
+	//   a1 41 aa 01                   value: { 0xAA: 1 }
+	//   581c <28-byte policy>         key: same PolicyID (duplicate)
+	//   a1 41 bb 02                   value: { 0xBB: 2 }
+	cborHex := "a2" +
+		"581c" + policyHex + "a141aa01" +
+		"581c" + policyHex + "a141bb02"
+	cborData, err := hex.DecodeString(cborHex)
+	assert.NoError(t, err)
+
+	var ma MultiAsset[MultiAssetTypeMint]
+	if err := ma.UnmarshalCBOR(cborData); err != nil {
+		t.Fatalf("unexpected error decoding MultiAsset with duplicate PolicyID: %v", err)
+	}
+	assert.ErrorContains(t, ma.CheckForDuplicateKeys(), "duplicate map key")
+
+	policy := NewBlake2b224(policyBytes)
+	// Last-wins: only the second entry survives, so the policy maps to {0xBB: 2}
+	// and the first entry's {0xAA: 1} is discarded entirely.
+	if len(ma.data) != 1 {
+		t.Fatalf("expected 1 policy after last-wins merge, got %d", len(ma.data))
+	}
+	inner, ok := ma.data[policy]
+	if !ok {
+		t.Fatalf("expected surviving policy %x to be present", policyBytes)
+	}
+	if len(inner) != 1 {
+		t.Fatalf("expected surviving inner asset map to have 1 entry, got %d", len(inner))
+	}
+	last := ma.Asset(policy, []byte{0xBB})
+	if last == nil || last.Int64() != 2 {
+		t.Fatalf("expected last-wins asset 0xBB == 2, got %v", last)
+	}
+	if dropped := ma.Asset(policy, []byte{0xAA}); dropped != nil {
+		t.Fatalf("expected first duplicate asset 0xAA to be discarded, got %v", dropped)
+	}
+}
+
+// TestMultiAssetUnmarshalDuplicateAssetNameLastWins verifies that a duplicate
+// key in the inner (AssetName) map is also tolerated and resolved last-wins,
+// not just the outer (PolicyID) map. Both maps decode through the same lenient
+// path into Go maps, so both follow pre-Conway (protocol version < 9)
+// cardano-ledger Map.fromList semantics. This pins the behavior MultiAsset
+// relies on; a future change to strict decoding must not silently regress it.
+func TestMultiAssetUnmarshalDuplicateAssetNameLastWins(t *testing.T) {
+	policyBytes := bytes.Repeat([]byte{0x22}, Blake2b224Size)
+	policyHex := hex.EncodeToString(policyBytes)
+	// a1                              outer map, 1 pair
+	//   581c <28-byte policy>         key: PolicyID
+	//   a2                            value: inner map, 2 pairs
+	//     41 cc 01                    key: 0xCC -> 1
+	//     41 cc 09                    key: 0xCC (duplicate) -> 9
+	cborHex := "a1" +
+		"581c" + policyHex +
+		"a2" + "41cc01" + "41cc09"
+	cborData, err := hex.DecodeString(cborHex)
+	assert.NoError(t, err)
+
+	var ma MultiAsset[MultiAssetTypeMint]
+	if err := ma.UnmarshalCBOR(cborData); err != nil {
+		t.Fatalf("unexpected error decoding MultiAsset with duplicate AssetName: %v", err)
+	}
+	assert.ErrorContains(t, ma.CheckForDuplicateKeys(), "duplicate map key")
+
+	policy := NewBlake2b224(policyBytes)
+	if len(ma.data) != 1 {
+		t.Fatalf("expected 1 policy, got %d", len(ma.data))
+	}
+	inner, ok := ma.data[policy]
+	if !ok {
+		t.Fatalf("expected policy %x to be present", policyBytes)
+	}
+	// Last-wins: the duplicate AssetName collapses to a single entry with the
+	// second value.
+	if len(inner) != 1 {
+		t.Fatalf("expected 1 inner asset after last-wins merge, got %d", len(inner))
+	}
+	last := ma.Asset(policy, []byte{0xCC})
+	if last == nil || last.Int64() != 9 {
+		t.Fatalf("expected last-wins asset 0xCC == 9, got %v", last)
+	}
+}

ledger/conway/conway.go

@@ -620,11 +620,35 @@ func (b *ConwayTransactionBody) UnmarshalCBOR(cborData []byte) error {
 			return err
 		}
 	}
+	if err := checkMultiAssetDuplicateKeys(tmp.TxMint); err != nil {
+		return err
+	}
+	for idx := range tmp.TxOutputs {
+		if err := checkMultiAssetDuplicateKeys(tmp.TxOutputs[idx].Assets()); err != nil {
+			return fmt.Errorf("transaction output %d: %w", idx, err)
+		}
+	}
+	if tmp.TxCollateralReturn != nil {
+		if err := checkMultiAssetDuplicateKeys(
+			tmp.TxCollateralReturn.Assets(),
+		); err != nil {
+			return fmt.Errorf("collateral return: %w", err)
+		}
+	}
 	*b = ConwayTransactionBody(tmp)
 	b.SetCborReference(cborData)
 	return nil
 }
 
+func checkMultiAssetDuplicateKeys[T int64 | uint64 | *big.Int](
+	assets *common.MultiAsset[T],
+) error {
+	if assets == nil {
+		return nil
+	}
+	return assets.CheckForDuplicateKeys()
+}
+
 func (b *ConwayTransactionBody) Inputs() []common.TransactionInput {
 	ret := make([]common.TransactionInput, 0, len(b.TxInputs.Items()))
 	for _, input := range b.TxInputs.Items() {

ledger/conway/conway_test.go

@@ -222,6 +222,45 @@ func TestConwayTransactionBodyRejectsDuplicateTaggedInputs(t *testing.T) {
 	assert.ErrorContains(t, err, "duplicate member in set")
 }
 
+func TestConwayTransactionBodyRejectsDuplicateMultiAssetKeys(t *testing.T) {
+	tests := []struct {
+		name string
+		body []byte
+	}{
+		{
+			name: "mint duplicate policy",
+			body: append(
+				[]byte{0xa1, 0x09},
+				testDuplicatePolicyMultiAssetCbor(0x11)...,
+			),
+		},
+		{
+			name: "mint duplicate asset name",
+			body: append(
+				[]byte{0xa1, 0x09},
+				testDuplicateAssetNameMultiAssetCbor(0x22)...,
+			),
+		},
+		{
+			name: "output duplicate asset name",
+			body: append(
+				[]byte{0xa1, 0x01, 0x81},
+				testBabbageOutputWithAssetsCbor(
+					t,
+					testDuplicateAssetNameMultiAssetCbor(0x33),
+				)...,
+			),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var body ConwayTransactionBody
+			err := body.UnmarshalCBOR(tt.body)
+			assert.ErrorContains(t, err, "duplicate map key")
+		})
+	}
+}
+
 func TestConwayTransactionBodyRejectsDuplicateTaggedSetFields(t *testing.T) {
 	input := testConwayShelleyInput()
 	var signer common.Blake2b224
@@ -291,6 +330,42 @@ func testConwayShelleyInput() shelley.ShelleyTransactionInput {
 	}
 }
 
+func testDuplicatePolicyMultiAssetCbor(policyByte byte) []byte {
+	policy := bytes.Repeat([]byte{policyByte}, common.Blake2b224Size)
+	ret := []byte{0xa2, 0x58, 0x1c}
+	ret = append(ret, policy...)
+	ret = append(ret, 0xa1, 0x41, 0xaa, 0x01, 0x58, 0x1c)
+	ret = append(ret, policy...)
+	ret = append(ret, 0xa1, 0x41, 0xbb, 0x02)
+	return ret
+}
+
+func testDuplicateAssetNameMultiAssetCbor(policyByte byte) []byte {
+	policy := bytes.Repeat([]byte{policyByte}, common.Blake2b224Size)
+	ret := []byte{0xa1, 0x58, 0x1c}
+	ret = append(ret, policy...)
+	ret = append(ret, 0xa2, 0x41, 0xcc, 0x01, 0x41, 0xcc, 0x09)
+	return ret
+}
+
+func testBabbageOutputWithAssetsCbor(t *testing.T, assets []byte) []byte {
+	t.Helper()
+	addr, err := common.NewAddressFromBytes(
+		test.DecodeHexString(
+			"40000000000000000000000000000000000000000000000000000000008198bd431b03",
+		),
+	)
+	assert.NoError(t, err)
+	addrCbor, err := cbor.Encode(addr)
+	assert.NoError(t, err)
+
+	ret := []byte{0xa2, 0x00}
+	ret = append(ret, addrCbor...)
+	ret = append(ret, 0x01, 0x82, 0x01)
+	ret = append(ret, assets...)
+	return ret
+}
+
 func TestConwayTx_WithReferenceInputs_CborRoundTrip(t *testing.T) {
 	// Test CBOR round-trip for transactions with reference inputs (CIP-0031)