feat: Track history in per layer metadata file

img/private/import.bzl

@@ -20,7 +20,22 @@ def _digest_to_file(ctx, digest):
         fail("invalid number of files for digest: {}".format(digest))
     return files[0]
 
-def _write_layer_info(ctx, manifest, config, layer_index, index_position = None):
+def _normalize_history_entry(entry):
+    """Copy only the known OCI history fields.
+
+    Imported image configs may contain non-standard keys in their history
+    entries. The layer metadata JSON is later decoded by the manifest tool with
+    DisallowUnknownFields (recursively), so forwarding unexpected keys would make
+    the build fail. Keeping only the spec'd fields avoids that.
+    See https://github.com/opencontainers/image-spec/blob/main/config.md#properties.
+    """
+    normalized = {}
+    for field in ("created", "created_by", "author", "comment", "empty_layer"):
+        if field in entry:
+            normalized[field] = entry[field]
+    return normalized
+
+def _write_layer_info(ctx, manifest, config, history, layer_index, index_position = None):
     """Write layer info to file and return SingleLayerInfo provider."""
     layers = manifest.get("layers", [])
     if layer_index >= len(layers):
@@ -52,13 +67,19 @@ def _write_layer_info(ctx, manifest, config, layer_index, index_position = None)
             config.get("architecture", "unknown"),
             layer_index,
         )
+
+    if history and layer_index < len(history):
+        layer_history = history[layer_index]
+    else:
+        layer_history = []
     metadata = dict(
         name = name,
         diff_id = diff_id,
         mediaType = media_type,
         digest = digest,
         size = size,
         annotations = layer.get("annotations", {}),
+        history = layer_history,
     )
     index_position_str = "" if index_position == None else str(index_position) + "_"
     layer_metadata = ctx.actions.declare_file(ctx.attr.name + "_{}{}_layer_metadata.json".format(index_position_str, layer_index))
@@ -115,8 +136,27 @@ def _build_manifest_info(ctx, digest, descriptor = None, index_position = None,
 
     missing_blobs = []
     layers = []
+
+    # Assign each history entry to an actual non-empty layer. This preserves the full image
+    # history across our layer splitting.
+    history_by_layer = []
+    current_layer = []
+    for hist_entry in config.get("history", []):
+        current_layer.append(_normalize_history_entry(hist_entry))
+        if not hist_entry.get("empty_layer", False):
+            history_by_layer.append(current_layer)
+            current_layer = []
+
+    # Bundle any trailing empty-layer entries with the last non-empty layer so they
+    # aren't lost. If the history had no non-empty entries at all, attach them to the
+    # first layer (when one exists) rather than dropping them.
+    if current_layer:
+        if history_by_layer:
+            history_by_layer[-1].extend(current_layer)
+        elif manifest.get("layers", []):
+            history_by_layer.append(current_layer)
     for (layer_index, layer) in enumerate(manifest.get("layers", [])):
-        layer_info = _write_layer_info(ctx, manifest, config, layer_index, index_position)
+        layer_info = _write_layer_info(ctx, manifest, config, history_by_layer, layer_index, index_position)
         if layer_info.blob == None:
             missing_blobs.append(layer["digest"].removeprefix("sha256:"))
         layers.append(layer_info)

img_tool/cmd/compress/compress.go

@@ -234,13 +234,20 @@ func writeMetadata(compressorState api.AppenderState, annotations map[string]str
 		mergedAnnotations[k] = v
 	}
 
+	// Preserve history from the source layer if it exists
+	history := []api.History{{CreatedBy: layerName}}
+	if sourceMetadata != nil && len(sourceMetadata.History) > 0 {
+		history = sourceMetadata.History
+	}
+
 	metadata := api.Descriptor{
 		Name:        layerName,
 		DiffID:      fmt.Sprintf("sha256:%x", compressorState.ContentHash),
 		MediaType:   mediaType,
 		Digest:      fmt.Sprintf("sha256:%x", compressorState.OuterHash),
 		Size:        compressorState.CompressedSize,
 		Annotations: mergedAnnotations,
+		History:     history,
 	}
 
 	json.NewEncoder(outputFile).SetIndent("", "  ")

img_tool/cmd/hash/hashfile.go

@@ -275,6 +275,7 @@ func writeLayerMetadata(compressedHash []byte, meta *layerMetadata, req *hashReq
 		Digest:      fmt.Sprintf("sha256:%x", compressedHash),
 		Size:        meta.compressedSize,
 		Annotations: mergedAnnotations,
+		History:     []api.History{{CreatedBy: layerName}},
 	}
 
 	// Write JSON output

img_tool/cmd/indexfromocilayout/indexfromocilayout.go

@@ -329,6 +329,10 @@ func convertManifest(manifestDesc *specv1.Descriptor, manifestIdx int, arch, ope
 		return specv1.Descriptor{}, fmt.Errorf("layer count mismatch: OCI layout has %d layers, but %d layer media types specified", len(manifest.Layers), layerCount)
 	}
 
+	// Distribute the source config's history across layers so it is preserved
+	// when this converted image is used as a base (mirrors the image_import rule).
+	perLayerHistory := metadata.SplitHistoryPerLayer(config.History, len(manifest.Layers))
+
 	// Copy/hardlink each layer blob and create metadata JSONs
 	for i := range manifest.Layers {
 		targetMediaType, ok := layerMediaTypes.Get(manifestIdx, i)
@@ -374,6 +378,7 @@ func convertManifest(manifestDesc *specv1.Descriptor, manifestIdx int, arch, ope
 			manifest.Layers[i].Digest.String(),
 			manifest.Layers[i].Size,
 			manifest.Layers[i].Annotations,
+			perLayerHistory[i],
 			metadataFile,
 		); err != nil {
 			return specv1.Descriptor{}, fmt.Errorf("writing metadata for layer %d: %w", i, err)

img_tool/cmd/layer/layer.go

@@ -450,6 +450,7 @@ func writeMetadata(name string, compressionAlgorithm api.CompressionAlgorithm, u
 		fmt.Sprintf("sha256:%x", compressorState.OuterHash),
 		compressorState.CompressedSize,
 		mergedAnnotations,
+		nil,
 		outputFile,
 	)
 }

img_tool/cmd/manifest/manifest.go

@@ -338,6 +338,17 @@ func prepareConfig(layers []api.Descriptor, templatesData *ConfigTemplates, crea
 	if createdTime != nil {
 		config.Created = createdTime
 	}
+	for _, layer := range layers {
+		for _, historyEntry := range layer.History {
+			config.History = append(config.History, specv1.History{
+				Created:    historyEntry.Created,
+				CreatedBy:  historyEntry.CreatedBy,
+				Author:     historyEntry.Author,
+				Comment:    historyEntry.Comment,
+				EmptyLayer: historyEntry.EmptyLayer,
+			})
+		}
+	}
 
 	return config, nil
 }
@@ -386,9 +397,9 @@ func overlayConfigFromFile(config *specv1.Image, filePath string, isBase bool) e
 	if configFragment.Architecture != "" {
 		config.Architecture = configFragment.Architecture
 	}
-	if len(configFragment.History) > 0 {
-		config.History = append(config.History, configFragment.History...)
-	}
+	// History is reconstructed from per-layer metadata in prepareConfig; the base
+	// config's redundant copy is intentionally not merged here, otherwise base
+	// layers would be counted twice (history longer than rootfs.diff_ids).
 
 	// merge config.Config
 	if configFragment.Config.User != "" {

img_tool/cmd/manifestfromocilayout/manifestfromocilayout.go

@@ -214,6 +214,10 @@ func processOCILayout() error {
 		return fmt.Errorf("layer count mismatch: OCI layout has %d layers, but %d layer media types specified", len(manifest.Layers), len(layerMediaTypes.values))
 	}
 
+	// Distribute the source config's history across layers so it is preserved
+	// when this converted image is used as a base (mirrors the image_import rule).
+	perLayerHistory := metadata.SplitHistoryPerLayer(config.History, len(manifest.Layers))
+
 	// Copy/hardlink each layer blob and create metadata JSONs
 	for i := range manifest.Layers {
 		targetMediaType, _ := layerMediaTypes.Get(i)
@@ -248,6 +252,7 @@ func processOCILayout() error {
 			manifest.Layers[i].Digest.String(),
 			manifest.Layers[i].Size,
 			manifest.Layers[i].Annotations,
+			perLayerHistory[i],
 			metadataFile,
 		); err != nil {
 			return fmt.Errorf("writing metadata for layer %d: %w", i, err)

img_tool/pkg/api/api.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/fs"
 	"iter"
+	"time"
 )
 
 type (
@@ -65,13 +66,33 @@ var (
 	Symlink     = FileType{"l"}
 )
 
+// History describes the history of a layer.
+// This is a re-export of the oci spec v1 History structure to avoid taking a dep on it
+type History struct {
+	// Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6.
+	Created *time.Time `json:"created,omitempty"`
+
+	// CreatedBy is the command which created the layer.
+	CreatedBy string `json:"created_by,omitempty"`
+
+	// Author is the author of the build point.
+	Author string `json:"author,omitempty"`
+
+	// Comment is a custom message set when creating the layer.
+	Comment string `json:"comment,omitempty"`
+
+	// EmptyLayer is used to mark if the history item created a filesystem diff.
+	EmptyLayer bool `json:"empty_layer,omitempty"`
+}
+
 type Descriptor struct {
 	Name        string            `json:"name,omitempty"`
 	DiffID      string            `json:"diff_id,omitempty"`
 	MediaType   string            `json:"mediaType"`
 	Digest      string            `json:"digest"`
 	Size        int64             `json:"size"`
 	Annotations map[string]string `json:"annotations,omitempty"`
+	History     []History         `json:"history,omitempty"`
 }
 
 type AppenderState struct {

img_tool/pkg/metadata/BUILD.bazel

@@ -5,5 +5,8 @@ go_library(
     srcs = ["metadata.go"],
     importpath = "github.com/bazel-contrib/rules_img/img_tool/pkg/metadata",
     visibility = ["//visibility:public"],
-    deps = ["//pkg/api"],
+    deps = [
+        "//pkg/api",
+        "@com_github_opencontainers_image_spec//specs-go/v1:specs-go",
+    ],
 )

img_tool/pkg/metadata/metadata.go

@@ -6,18 +6,24 @@ import (
 	"io"
 	"slices"
 
+	specv1 "github.com/opencontainers/image-spec/specs-go/v1"
+
 	"github.com/bazel-contrib/rules_img/img_tool/pkg/api"
 )
 
 // WriteLayerMetadata writes layer metadata in the format expected by SingleLayerInfo provider.
-// The format includes: name, diff_id, mediaType, digest, size, and annotations.
+// The format includes: name, diff_id, mediaType, digest, size, annotations, and history.
+//
+// When history is empty, a synthetic single entry {created_by: name} is written so every
+// layer carries at least a created_by marker.
 func WriteLayerMetadata(
 	name string,
 	diffID string,
 	mediaType string,
 	digest string,
 	size int64,
 	annotations map[string]string,
+	history []api.History,
 	outputFile io.Writer,
 ) error {
 	// Merge and sort annotations for determinism
@@ -33,13 +39,18 @@ func WriteLayerMetadata(
 		}
 	}
 
+	if len(history) == 0 {
+		history = []api.History{{CreatedBy: name}}
+	}
+
 	metadata := api.Descriptor{
 		Name:        name,
 		DiffID:      diffID,
 		MediaType:   mediaType,
 		Digest:      digest,
 		Size:        size,
 		Annotations: mergedAnnotations,
+		History:     history,
 	}
 
 	encoder := json.NewEncoder(outputFile)
@@ -74,3 +85,51 @@ func MergeAnnotations(userAnnotations map[string]string, layerAnnotations map[st
 
 	return merged
 }
+
+// SplitHistoryPerLayer distributes an OCI image config's history entries across
+// the image's layers, mirroring the layout used by the image_import rule.
+//
+// Empty-layer history entries (metadata-only operations such as ENV/CMD) are
+// bundled with the next non-empty layer; any trailing empty-layer entries are
+// attached to the last non-empty layer so no history is lost. If the config
+// history contains no non-empty entries at all, they are attached to the first
+// layer (when one exists). The returned slice has one element per layer (index i
+// corresponds to layer i); layers that receive no history get a nil slice, which
+// callers may replace with a fallback (see WriteLayerMetadata).
+func SplitHistoryPerLayer(history []specv1.History, numLayers int) [][]api.History {
+	perLayer := make([][]api.History, numLayers)
+	if numLayers == 0 {
+		return perLayer
+	}
+
+	var current []api.History
+	layerIndex := 0
+	for _, entry := range history {
+		current = append(current, historyFromOCI(entry))
+		if !entry.EmptyLayer {
+			if layerIndex < numLayers {
+				perLayer[layerIndex] = current
+				layerIndex++
+			}
+			current = nil
+		}
+	}
+	if len(current) > 0 {
+		if layerIndex > 0 {
+			perLayer[layerIndex-1] = append(perLayer[layerIndex-1], current...)
+		} else {
+			perLayer[0] = current
+		}
+	}
+	return perLayer
+}
+
+func historyFromOCI(entry specv1.History) api.History {
+	return api.History{
+		Created:    entry.Created,
+		CreatedBy:  entry.CreatedBy,
+		Author:     entry.Author,
+		Comment:    entry.Comment,
+		EmptyLayer: entry.EmptyLayer,
+	}
+}

tests/img_toolchain/testcases/compress_preserves_source_metadata.ini

@@ -17,7 +17,8 @@ name = source_metadata.json
     "org.example.keep": "yes",
     "containerd.io/snapshot/stargz/toc.digest": "sha256:old-toc",
     "io.containers.estargz.uncompressed-size": "123"
-  }
+  },
+  "history": [{"created_by": "RUN original-base-step"}]
 }
 
 [command]
@@ -37,3 +38,4 @@ file_contains = out_metadata.json, "org.example.override"
 file_contains = out_metadata.json, "new"
 file_not_contains = out_metadata.json, "sha256:old-toc"
 file_not_contains = out_metadata.json, "io.containers.estargz.uncompressed-size"
+file_contains = out_metadata.json, "RUN original-base-step"

tests/img_toolchain/testcases/compress_with_annotations.ini

@@ -28,4 +28,4 @@ file_contains = compress-meta.json, "team"
 file_contains = compress-meta.json, "platform"
 file_contains = compress-meta.json, "project"
 file_contains = compress-meta.json, "rules_img"
-file_sha256 = compress-meta.json, "7b492556ebc6c8791f42b36d67adc861fd3751a18ddde88cf150538435aab80c"
+file_sha256 = compress-meta.json, "3add29c12e02079cda2896749e77fbee698c7a32a6f5a03ba89390eee6faafb2"

tests/img_toolchain/testcases/layer_runfiles.ini

@@ -42,5 +42,6 @@ tar_entry_exists = layer.tar.gz, foo/bar/baz.exe.runfiles/_main/data/3.txt
 tar_entry_type = layer.tar.gz, foo/bar/baz.exe.runfiles/_main/data/3.txt, regular
 tar_entry_size = layer.tar.gz, foo/bar/baz.exe.runfiles/_main/data/3.txt, 14
 layer_invariants_intact = layer.tar.gz
-file_sha256 = layer_meta.json, "047d674cf55d5d5c80e917fa5847eff506614b06328e8e8d67cecaea415a4caf"
+file_contains = layer_meta.json, "history"
+file_sha256 = layer_meta.json, "0e5615c3fb2f0b83abbed6b228bace14b94c16089f91ebd8f36c0119fadd07f7"
 file_sha256 = layer.tar.gz, "1fce6f5736c0594d6a2273251e123babff5645f3dace6a06142d4a24a290dec7"