Serdes fuzz testing and some bug fixes/compatability work (#100)

* serde fuzz testing and some bug fixes/compatability work

* a

* some serdes error simplification

* tweaks to errors

* add ip serdes support

* more tests

* oops

Author: toptobes

astra/internal/untyped/row_test.go

@@ -121,7 +121,7 @@ func TestRow_UnmarshalAstraRaw(t *testing.T) {
 				{Name: "ip", Column: table.Column{Type: table.TypeInet}},
 				{Name: "blob", Column: table.Column{Type: table.TypeBlob}},
 			},
-			jsonData: `{"uuid": "550e8400-e29b-41d4-a716-446655440000", "ip": [192, 168, 1, 1], "blob": {"$binary": "aGVsbG8="}}`,
+			jsonData: `{"uuid": "550e8400-e29b-41d4-a716-446655440000", "ip": "192.168.1.1", "blob": {"$binary": "aGVsbG8="}}`,
 			validate: func(t *testing.T, row Row) {
 				data := row.ToMap()
 				if _, ok := data["uuid"].(datatypes.UUID); !ok {
@@ -233,6 +233,11 @@ func TestRow_UnmarshalAstraRaw_InvalidJSON(t *testing.T) {
 func TestProperty_DeserializeWithTypeHint_Varint(t *testing.T) {
 	rapid.Check(t, func(t *rapid.T) {
 		s := rapid.StringMatching(`-?[0-9]{1,50}`).Draw(t, "digits")
+
+		if s[0] == '0' || s[0] == '-' && s[1] == '0' {
+			return // invalid json
+		}
+
 		raw := json.RawMessage(s)
 		col := table.Column{Type: table.TypeVarint}
 
@@ -259,6 +264,11 @@ func TestProperty_DeserializeWithTypeHint_Varint(t *testing.T) {
 func TestProperty_DeserializeWithTypeHint_Decimal(t *testing.T) {
 	rapid.Check(t, func(t *rapid.T) {
 		s := rapid.StringMatching(`-?[0-9]{1,20}(\.[0-9]{1,20})?`).Draw(t, "digits")
+
+		if s[0] == '0' || s[0] == '-' && s[1] == '0' {
+			return // invalid json
+		}
+
 		raw := json.RawMessage(s)
 		col := table.Column{Type: table.TypeDecimal}
 
@@ -574,7 +584,10 @@ func genValueForColumn(col table.Column) *rapid.Generator[any] {
 		case table.TypeUUID, table.TypeTimeUUID:
 			return datatypes.MustParseUUID(rapid.StringMatching(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`).Draw(t, "uuid"))
 		case table.TypeInet:
-			return net.IP(rapid.SliceOfN(rapid.Byte(), 4, 16).Draw(t, "ip_bytes"))
+			if rapid.Bool().Draw(t, "is_ipv4") {
+				return net.IP(rapid.SliceOfN(rapid.Byte(), 4, 4).Draw(t, "ipv4_bytes"))
+			}
+			return net.IP(rapid.SliceOfN(rapid.Byte(), 16, 16).Draw(t, "ipv6_bytes"))
 		case table.TypeBlob:
 			return rapid.SliceOf(rapid.Byte()).Draw(t, "blob")
 		case table.TypeDecimal:

astra/options/api_options.go

@@ -111,11 +111,12 @@ type SerdesOptions struct {
 	UseJSONMarshal  *bool
 
 	// Deserialization flags
-	SparseRows           *bool
-	UseNumber            *bool
-	DesNoCache           *bool
-	ExtendedErrorContext *bool
-	UseJSONUnmarshal     *bool
+	SparseRows                   *bool
+	UseNumber                    *bool
+	DesNoCache                   *bool
+	ExtendedErrorSnippet         *bool
+	UseJSONUnmarshal             *bool
+	CaseInsensitiveFieldMatching *bool
 }
 
 // Merge implements shouldMerge for SerdesOptions.
@@ -163,8 +164,9 @@ func (o *SerdesOptions) walkDesFlags(fn func(serdes.DesFlags, **bool)) {
 	fn(serdes.SparseRows, &o.SparseRows)
 	fn(serdes.UseNumber, &o.UseNumber)
 	fn(serdes.DesNoCache, &o.DesNoCache)
-	fn(serdes.ExtendedErrorContext, &o.ExtendedErrorContext)
+	fn(serdes.ExtendedErrorSnippet, &o.ExtendedErrorSnippet)
 	fn(serdes.UseJSONUnmarshal, &o.UseJSONUnmarshal)
+	fn(serdes.CaseInsensitiveFieldMatching, &o.CaseInsensitiveFieldMatching)
 }
 
 // TimeoutOptions contains timeout configuration for API operations.

astra/options/builders_gen.go

@@ -43,30 +43,7 @@ type DecodeCtx struct {
 	Target    Target
 	TargetCtx TargetDecodeCtx
 	Flags     DesFlags
-}
-
-func (c DecodeCtx) syntaxError(src []byte, msg string) error {
-	return &SyntaxError{msg: msg, Snippet: errorSnippet(src, c.Flags)}
-}
-
-func (c DecodeCtx) syntaxErrorWrap(src []byte, msg string, err error) error {
-	return &SyntaxError{msg: msg, Snippet: errorSnippet(src, c.Flags), Err: err}
-}
-
-func (c DecodeCtx) unmarshalTypeError(src []byte, t reflect.Type) error {
-	return &UnmarshalTypeError{Value: nextJsonType(src), Type: t, Snippet: errorSnippet(src, c.Flags)}
-}
-
-func (c DecodeCtx) unmarshalValueTypeError(src []byte, t reflect.Type, value string) error {
-	return &UnmarshalTypeError{Value: value, Type: t, Snippet: errorSnippet(src, c.Flags)}
-}
-
-func (c DecodeCtx) unmarshalTypeErrorWrap(src []byte, t reflect.Type, err error) error {
-	return &UnmarshalTypeError{Value: nextJsonType(src), Type: t, Snippet: errorSnippet(src, c.Flags), Err: err}
-}
-
-func (c DecodeCtx) unmarshalValueTypeErrorWrap(src []byte, t reflect.Type, value string, err error) error {
-	return &UnmarshalTypeError{Value: value, Type: t, Snippet: errorSnippet(src, c.Flags), Err: err}
+	payload   *[]byte
 }
 
 type AstraMarshaler interface {
@@ -190,6 +167,8 @@ func resolveCodec(ctx codecCtx, t reflect.Type, seen seenStructs, canAddr bool)
 		return codec{durationEncoder, durationDecoder}
 	case timeType:
 		return codec{timeEncoder, timeDecoder}
+	case ipType:
+		return codec{ipEncoder, ipDecoder}
 	}
 
 	if c.encode != nil {
@@ -220,7 +199,7 @@ func resolveCodec(ctx codecCtx, t reflect.Type, seen seenStructs, canAddr bool)
 		c = mkSomeInterfaceCodec(t)
 	default:
 		if c.encode == nil {
-			c = mkErroredCodec(&UnsupportedTypeError{Type: t})
+			c = mkErroredCodec(&EncodeError{Action: EncodeActionUnsupportedType, Type: t})
 		}
 	}
 

astra/serdes/codecs.go

@@ -125,7 +125,7 @@ func TestCache_HeavyContention_Correctness(t *testing.T) {
 				// 4. Round-trip: Decode
 				valOut := reflect.New(typ).Elem()
 				ptrOut := unsafe.Pointer(valOut.UnsafeAddr())
-				_, err = c.decode(DecodeCtx{}, buf, ptrOut)
+				_, err = c.decode(DecodeCtx{payload: &buf}, buf, ptrOut)
 				if err != nil {
 					t.Errorf("worker %d: decode failed: %v", workerID, err)
 					continue

astra/serdes/codecs_test.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/big"
+	"net"
 	"reflect"
 	"strconv"
 	"time"
@@ -90,7 +91,7 @@ var oidTag = []byte("objectId")
 
 func objectIdEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target != TargetCollection {
-		return dst, &UnsupportedValueError{Msg: "ObjectId is only supported for collections"}
+		return dst, ctx.unsupportedValueError("ObjectId is only supported for collections")
 	}
 
 	return encodeDollarDatatype(dst, oidTag, func(dst []byte) ([]byte, error) {
@@ -103,7 +104,7 @@ func objectIdEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error
 
 func objectIdDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target != TargetCollection {
-		return src, &UnsupportedValueError{Msg: "ObjectId is only supported for collections"}
+		return src, ctx.unsupportedValueError(src, "ObjectId is only supported for collections")
 	}
 
 	src, oid, err := parseDollarDatatype(ctx, src, oidTag, func(ctx DecodeCtx, b []byte) ([]byte, datatypes.ObjectId, error) {
@@ -179,7 +180,7 @@ func timeDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 
 func dateOnlyEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return nil, &UnsupportedValueError{Msg: "DateOnly is not supported for collections"}
+		return nil, ctx.unsupportedValueError("DateOnly is not supported for collections")
 	}
 
 	d := (*datatypes.DateOnly)(p)
@@ -191,7 +192,7 @@ func dateOnlyEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error
 
 func dateOnlyDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return src, &UnsupportedValueError{Msg: "DateOnly is not supported for collections"}
+		return src, ctx.unsupportedValueError(src, "DateOnly is not supported for collections")
 	}
 
 	srcAfter, str, _, err := parseStringUnquote(ctx, src)
@@ -214,7 +215,7 @@ func dateOnlyDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error
 
 func timeOnlyEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return nil, &UnsupportedValueError{Msg: "TimeOnly is not supported for collections"}
+		return nil, ctx.unsupportedValueError("TimeOnly is not supported for collections")
 	}
 
 	t := (*datatypes.TimeOnly)(p)
@@ -226,7 +227,7 @@ func timeOnlyEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error
 
 func timeOnlyDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return src, &UnsupportedValueError{Msg: "TimeOnly is not supported for collections"}
+		return src, ctx.unsupportedValueError(src, "TimeOnly is not supported for collections")
 	}
 
 	srcAfter, str, _, err := parseStringUnquote(ctx, src)
@@ -355,27 +356,37 @@ func vectorDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error)
 }
 
 // ================================
-// | Binary data ([]byte) - encoded as {"$binary":"<base64>"} in all contexts,
+// | Binary data ([]byte) - encoded as {"$binary":"<base64>"} in tables and collections, and just "<base64>" otherwise
 // | but can be decoded from either that format or from a base64 string
+// |
+// | NOTE: There's technically no reason to ever encode a []byte outside of encoding to a collection/table, but the
+// | special case just exists to help the fuzz testing pass
 // ================================
 
-func binaryEncoder(_ EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
-	return encodeDollarDatatype(dst, []byte("binary"), func(dst []byte) ([]byte, error) {
-		return encodeBytesAsBase64(dst, *(*[]byte)(p)), nil
-	})
+func binaryEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
+	if ctx.Target == TargetCollection || ctx.Target == TargetTable {
+		return encodeDollarDatatype(dst, []byte("binary"), func(dst []byte) ([]byte, error) {
+			return encodeBytesAsBase64(dst, *(*[]byte)(p)), nil
+		})
+	}
+	return encodeBytesAsBase64(dst, *(*[]byte)(p)), nil
 }
 
 func binaryDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 	src = skipWS(src)
 
+	if b, ok := consumeNull(src); ok {
+		*(*[]byte)(p) = nil
+		return b, nil
+	}
+
 	if len(src) != 0 && src[0] == '"' {
-		src, str, _, err := parseStringUnquote(ctx, src)
-		if err != nil {
-			return src, err
-		}
+		src, data, err := decodeBytesFromBase64(ctx, src)
 
-		*(*[]byte)(p) = str
-		return src, nil
+		if err == nil {
+			*(*[]byte)(p) = data
+		}
+		return src, err
 	}
 
 	if len(src) != 0 && src[0] == '{' {
@@ -444,7 +455,7 @@ func decodeBytesFromBase64(ctx DecodeCtx, src []byte) ([]byte, []byte, error) {
 
 func durationEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return nil, &UnsupportedValueError{Msg: "Duration is not supported for collections"}
+		return nil, ctx.unsupportedValueError("Duration is not supported for collections")
 	}
 	d := (*datatypes.Duration)(p)
 	dst = append(dst, '"')
@@ -455,7 +466,7 @@ func durationEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error
 
 func durationDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
 	if ctx.Target == TargetCollection {
-		return src, &UnsupportedValueError{Msg: "Duration is not supported for collections"}
+		return src, ctx.unsupportedValueError(src, "Duration is not supported for collections")
 	}
 	src, str, _, err := parseStringUnquote(ctx, src)
 	if err != nil {
@@ -469,6 +480,37 @@ func durationDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error
 	return src, nil
 }
 
+// ================================
+// | net.IP - encoded as a plain quoted string in all contexts
+// ================================
+
+func ipEncoder(ctx EncodeCtx, dst []byte, p unsafe.Pointer) ([]byte, error) {
+	if ctx.Target == TargetCollection {
+		return nil, ctx.unsupportedValueError("net.IP is not supported for collections")
+	}
+	ip := (*net.IP)(p)
+	dst = append(dst, '"')
+	dst = append(dst, ip.String()...)
+	dst = append(dst, '"')
+	return dst, nil
+}
+
+func ipDecoder(ctx DecodeCtx, src []byte, p unsafe.Pointer) ([]byte, error) {
+	if ctx.Target == TargetCollection {
+		return src, ctx.unsupportedValueError(src, "net.IP is not supported for collections")
+	}
+	srcAfter, str, _, err := parseStringUnquote(ctx, src)
+	if err != nil {
+		return srcAfter, err
+	}
+	ip := net.ParseIP(unsafeString(str))
+	if ip == nil {
+		return srcAfter, ctx.syntaxError(src, "invalid IP string")
+	}
+	*(*net.IP)(p) = ip
+	return srcAfter, nil
+}
+
 // ================================
 // | Helpers
 // ================================