Allow inlinedtree.Build() to spill large lists into separate objects

inlinedtree.Build() can currently only store messages in separate
objects. This is problematic, because we also have many cases in which
we want to store lists of objects separately.

To achieve this, let's add a Marshalable type to model_core. We can use
this to encode all the different strategies of how we marshal data.
Right now there are only implementations for plain messages and lists of
messages. Maybe later we also want to provide an implementation for
blobs, so that we can store large Starlark strings/bytes in separate
objects.

When storing lists externally, we need to emit parent nodes. For this we
already have btree.ParentNodeComputer. Let's reuse this here. Provide a
btree.MaybeMergeNodes() function that acts as a bridge between
inlinedtree and btree. Because inlinedtree.ParentAppender does not allow
returning errors, change btree.ParentNodeComputer to also not allow it.

I'm not entirely happy with how friendly the resulting API is to use.
Let's see if we can improve it at some point in the future. For example,
I don't like how we need to pass the capturer, encoder, and parent node
computer around separately, even though I don't think you ever want
those to differ. We should really address that at some point in the
nearby future.

Author: EdSchouten

cmd/bonanza_builder/main.go

@@ -374,17 +374,12 @@ func (e *builderExecutor) Execute(ctx context.Context, action *model_build_pb.Ac
 		btree.NewObjectCreatingNodeMerger(
 			evaluationTreeEncoder,
 			namespace.ReferenceFormat,
-			/* parentNodeComputer = */ func(createdObject model_core.Decodable[model_core.CreatedObject[dag.ObjectContentsWalker]], childNodes []*model_evaluation_pb.Evaluation) (model_core.PatchedMessage[*model_evaluation_pb.Evaluation, dag.ObjectContentsWalker], error) {
+			/* parentNodeComputer = */ func(createdObject model_core.Decodable[model_core.CreatedObject[dag.ObjectContentsWalker]], childNodes []*model_evaluation_pb.Evaluation) model_core.PatchedMessage[*model_evaluation_pb.Evaluation, dag.ObjectContentsWalker] {
 				var firstKey []byte
 				switch firstEntry := childNodes[0].Level.(type) {
 				case *model_evaluation_pb.Evaluation_Leaf_:
-					flattenedAny, err := model_core.FlattenAny(model_core.NewMessage(firstEntry.Leaf.Key, createdObject.Value.Contents))
-					if err != nil {
-						return model_core.PatchedMessage[*model_evaluation_pb.Evaluation, dag.ObjectContentsWalker]{}, err
-					}
-					firstKey, err = model_core.MarshalTopLevelMessage(flattenedAny)
-					if err != nil {
-						return model_core.PatchedMessage[*model_evaluation_pb.Evaluation, dag.ObjectContentsWalker]{}, err
+					if flattenedAny, err := model_core.FlattenAny(model_core.NewMessage(firstEntry.Leaf.Key, createdObject.Value.Contents)); err == nil {
+						firstKey, _ = model_core.MarshalTopLevelMessage(flattenedAny)
 					}
 				case *model_evaluation_pb.Evaluation_Parent_:
 					firstKey = firstEntry.Parent.FirstKey
@@ -398,7 +393,7 @@ func (e *builderExecutor) Execute(ctx context.Context, action *model_build_pb.Ac
 							},
 						},
 					}
-				}), nil
+				})
 			},
 		),
 	)
@@ -446,7 +441,7 @@ func (e *builderExecutor) Execute(ctx context.Context, action *model_build_pb.Ac
 			btree.NewObjectCreatingNodeMerger(
 				evaluationTreeEncoder,
 				namespace.ReferenceFormat,
-				/* parentNodeComputer = */ func(createdObject model_core.Decodable[model_core.CreatedObject[dag.ObjectContentsWalker]], childNodes []*model_evaluation_pb.Dependency) (model_core.PatchedMessage[*model_evaluation_pb.Dependency, dag.ObjectContentsWalker], error) {
+				/* parentNodeComputer = */ func(createdObject model_core.Decodable[model_core.CreatedObject[dag.ObjectContentsWalker]], childNodes []*model_evaluation_pb.Dependency) model_core.PatchedMessage[*model_evaluation_pb.Dependency, dag.ObjectContentsWalker] {
 					return model_core.BuildPatchedMessage(func(patcher *model_core.ReferenceMessagePatcher[dag.ObjectContentsWalker]) *model_evaluation_pb.Dependency {
 						return &model_evaluation_pb.Dependency{
 							Level: &model_evaluation_pb.Dependency_Parent_{
@@ -455,7 +450,7 @@ func (e *builderExecutor) Execute(ctx context.Context, action *model_build_pb.Ac
 								},
 							},
 						}
-					}), nil
+					})
 				},
 			),
 		)
@@ -533,8 +528,8 @@ func (e *builderExecutor) Execute(ctx context.Context, action *model_build_pb.Ac
 	}
 	var evaluationsReference *model_core_pb.WeakDecodableReference
 	if len(evaluations.Message) > 0 {
-		createdEvaluations, err := model_core.MarshalAndEncodePatchedListMessage(
-			evaluations,
+		createdEvaluations, err := model_core.MarshalAndEncode(
+			model_core.MessageListToMarshalable(evaluations),
 			namespace.ReferenceFormat,
 			evaluationTreeEncoder,
 		)

pkg/bazelclient/commands/build/do_build.go

@@ -350,8 +350,8 @@ func DoBuild(args *arguments.BuildCommand, workspacePath path.Parser) {
 		if l := createdRootDirectory.MaximumSymlinkEscapementLevels; l == nil || l.Value != 0 {
 			logger.Fatal(formatted.Textf("Module %#v contains one or more symbolic links that potentially escape the module's root directory", moduleName.String()))
 		}
-		createdObject, err := model_core.MarshalAndEncodePatchedMessage(
-			createdModuleRootDirectories[i].Message,
+		createdObject, err := model_core.MarshalAndEncode(
+			model_core.MessageToMarshalable(createdModuleRootDirectories[i].Message),
 			referenceFormat,
 			directoryParameters.GetEncoder(),
 		)
@@ -392,8 +392,8 @@ func DoBuild(args *arguments.BuildCommand, workspacePath path.Parser) {
 		logger.Fatal(formatted.Textf("Failed to create build specification encoder: %s", err))
 	}
 
-	createdBuildSpecification, err := model_core.MarshalAndEncodePatchedMessage(
-		model_core.NewPatchedMessage(&buildSpecification, buildSpecificationPatcher),
+	createdBuildSpecification, err := model_core.MarshalAndEncode(
+		model_core.NewPatchedMessage(model_core.NewMessageMarshalable(&buildSpecification), buildSpecificationPatcher),
 		referenceFormat,
 		buildSpecificationEncoder,
 	)

pkg/model/analysis/action_result.go

@@ -109,22 +109,23 @@ func convertDictToEnvironmentVariableList[TMetadata model_core.ReferenceMetadata
 	commandEncoder model_encoding.BinaryEncoder,
 	referenceFormat object.ReferenceFormat,
 	capturer model_core.CreatedObjectCapturer[TMetadata],
-) (model_core.PatchedMessage[[]*model_command_pb.EnvironmentVariableList_Element, TMetadata], error) {
+) (model_core.PatchedMessage[[]*model_command_pb.EnvironmentVariableList_Element, TMetadata], btree.ParentNodeComputer[*model_command_pb.EnvironmentVariableList_Element, TMetadata], error) {
+	parentNodeComputer := func(createdObject model_core.Decodable[model_core.CreatedObject[TMetadata]], childNodes []*model_command_pb.EnvironmentVariableList_Element) model_core.PatchedMessage[*model_command_pb.EnvironmentVariableList_Element, TMetadata] {
+		return model_core.BuildPatchedMessage(func(patcher *model_core.ReferenceMessagePatcher[TMetadata]) *model_command_pb.EnvironmentVariableList_Element {
+			return &model_command_pb.EnvironmentVariableList_Element{
+				Level: &model_command_pb.EnvironmentVariableList_Element_Parent{
+					Parent: patcher.CaptureAndAddDecodableReference(createdObject, capturer),
+				},
+			}
+		})
+	}
 	environmentVariablesBuilder := btree.NewSplitProllyBuilder(
 		1<<16,
 		1<<18,
 		btree.NewObjectCreatingNodeMerger(
 			commandEncoder,
 			referenceFormat,
-			/* parentNodeComputer = */ func(createdObject model_core.Decodable[model_core.CreatedObject[TMetadata]], childNodes []*model_command_pb.EnvironmentVariableList_Element) (model_core.PatchedMessage[*model_command_pb.EnvironmentVariableList_Element, TMetadata], error) {
-				return model_core.BuildPatchedMessage(func(patcher *model_core.ReferenceMessagePatcher[TMetadata]) *model_command_pb.EnvironmentVariableList_Element {
-					return &model_command_pb.EnvironmentVariableList_Element{
-						Level: &model_command_pb.EnvironmentVariableList_Element_Parent{
-							Parent: patcher.CaptureAndAddDecodableReference(createdObject, capturer),
-						},
-					}
-				}), nil
-			},
+			parentNodeComputer,
 		),
 	)
 	for _, name := range slices.Sorted(maps.Keys(environment)) {
@@ -138,8 +139,9 @@ func convertDictToEnvironmentVariableList[TMetadata model_core.ReferenceMetadata
 				},
 			}),
 		); err != nil {
-			return model_core.PatchedMessage[[]*model_command_pb.EnvironmentVariableList_Element, TMetadata]{}, err
+			return model_core.PatchedMessage[[]*model_command_pb.EnvironmentVariableList_Element, TMetadata]{}, nil, err
 		}
 	}
-	return environmentVariablesBuilder.FinalizeList()
+	envList, err := environmentVariablesBuilder.FinalizeList()
+	return envList, parentNodeComputer, err
 }

pkg/model/analysis/base_computer_test.go

@@ -148,8 +148,8 @@ func attachMessageObject[TMessage proto.Message](
 	builder func(childPatcher *model_core.ReferenceMessagePatcher[model_core.CreatedObjectTree]) TMessage,
 ) *model_core_pb.DecodableReference {
 	createdObject := util.Must(
-		model_core.MarshalAndEncodePatchedMessage(
-			model_core.BuildPatchedMessage(builder),
+		model_core.MarshalAndEncode(
+			model_core.MessageToMarshalable(model_core.BuildPatchedMessage(builder)),
 			util.Must(object.NewReferenceFormat(object_pb.ReferenceFormat_SHA256_V1)),
 			model_encoding.NewChainedBinaryEncoder(nil),
 		),