feat: add reuseBytes option to deserialize

packages/ssz/package.json

@@ -21,7 +21,8 @@
     "check-types": "tsc --noEmit",
     "clean": "rm -rf lib && rm -rf dist && rm -f tsconfig.tsbuildinfo",
     "prepublishOnly": "yarn build",
-    "benchmark": "node --loader=ts-node/esm --max-old-space-size=4096 --expose-gc ../../node_modules/.bin/benchmark 'test/perf/*.test.ts'",
+    "benchmark": "yarn benchmark:files 'test/perf/*.test.ts'",
+    "benchmark:files": "node --loader=ts-node/esm --max-old-space-size=4096 --expose-gc ../../node_modules/.bin/benchmark",
     "benchmark:local": "yarn benchmark --local",
     "test:unit": "vitest run --dir test/unit",
     "test:spec": "yarn test:spec-generic && yarn test:spec-static test:spec-eip-4881",

packages/ssz/src/type/abstract.ts

@@ -81,7 +81,7 @@ export abstract class Type<V> {
   /** INTERNAL METHOD: Serialize value to existing output ArrayBuffer views */
   abstract value_serializeToBytes(output: ByteViews, offset: number, value: V): number;
   /** INTERNAL METHOD: Deserialize value from a section of ArrayBuffer views */
-  abstract value_deserializeFromBytes(data: ByteViews, start: number, end: number): V;
+  abstract value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): V;
   /** INTERNAL METHOD: Return serialized size of a tree */
   abstract tree_serializedSize(node: Node): number;
   /** INTERNAL METHOD: Serialize tree to existing output ArrayBuffer views  */
@@ -121,16 +121,9 @@ export abstract class Type<V> {
   }
 
   /** Deserialize binary data to value */
-  deserialize(uint8Array: Uint8Array): V {
-    // Buffer.prototype.slice does not copy memory, force use Uint8Array.prototype.slice https://github.com/nodejs/node/issues/28087
-    // - Uint8Array.prototype.slice: Copy memory, safe to mutate
-    // - Buffer.prototype.slice: Does NOT copy memory, mutation affects both views
-    // We could ensure that all Buffer instances are converted to Uint8Array before calling value_deserializeFromBytes
-    // However doing that in a browser friendly way is not easy. Downstream code uses `Uint8Array.prototype.slice.call`
-    // to ensure Buffer.prototype.slice is never used. Unit tests also test non-mutability.
-
+  deserialize(uint8Array: Uint8Array, opts?: {reuseBytes?: boolean}): V {
     const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
-    return this.value_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length);
+    return this.value_deserializeFromBytes({uint8Array, dataView}, 0, uint8Array.length, opts?.reuseBytes);
   }
 
   // Merkleization

packages/ssz/src/type/arrayComposite.ts

@@ -90,7 +90,8 @@ export function value_deserializeFromBytesArrayComposite<
   data: ByteViews,
   start: number,
   end: number,
-  arrayProps: ArrayProps
+  arrayProps: ArrayProps,
+  reuseBytes?: boolean
 ): ValueOf<ElementType>[] {
   const offsets = readOffsetsArrayComposite(elementType.fixedSize, data.dataView, start, end, arrayProps);
   const length = offsets.length; // Capture length before pushing end offset
@@ -102,7 +103,7 @@ export function value_deserializeFromBytesArrayComposite<
     // The offsets are relative to the start
     const startEl = start + offsets[i];
     const endEl = i === length - 1 ? end : start + offsets[i + 1];
-    values[i] = elementType.value_deserializeFromBytes(data, startEl, endEl);
+    values[i] = elementType.value_deserializeFromBytes(data, startEl, endEl, reuseBytes);
   }
 
   return values;

packages/ssz/src/type/bitList.ts

@@ -6,6 +6,7 @@ import {
   packedNodeRootsToBytes,
   packedRootsBytesToNode,
 } from "@chainsafe/persistent-merkle-tree";
+import {slice} from "../util/byteArray.ts";
 import {maxChunksToDepth} from "../util/merkleize.ts";
 import {namedClass} from "../util/named.ts";
 import {Require} from "../util/types.ts";
@@ -79,8 +80,8 @@ export class BitListType extends BitArrayType {
     return applyPaddingBit(output.uint8Array, offset, value.bitLen);
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): BitArray {
-    const {uint8Array, bitLen} = this.deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end);
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): BitArray {
+    const {uint8Array, bitLen} = this.deserializeUint8ArrayBitListFromBytes(data.uint8Array, start, end, reuseBytes);
     return new BitArray(uint8Array, bitLen);
   }
 
@@ -135,8 +136,13 @@ export class BitListType extends BitArrayType {
 
   // Deserializer helpers
 
-  private deserializeUint8ArrayBitListFromBytes(data: Uint8Array, start: number, end: number): BitArrayDeserialized {
-    const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data, start, end);
+  private deserializeUint8ArrayBitListFromBytes(
+    data: Uint8Array,
+    start: number,
+    end: number,
+    reuseBytes?: boolean
+  ): BitArrayDeserialized {
+    const {uint8Array, bitLen} = deserializeUint8ArrayBitListFromBytes(data, start, end, reuseBytes);
     if (bitLen > this.limitBits) {
       throw Error(`bitLen over limit ${bitLen} > ${this.limitBits}`);
     }
@@ -149,7 +155,8 @@ type BitArrayDeserialized = {uint8Array: Uint8Array; bitLen: number};
 export function deserializeUint8ArrayBitListFromBytes(
   data: Uint8Array,
   start: number,
-  end: number
+  end: number,
+  reuseBytes?: boolean
 ): BitArrayDeserialized {
   if (end > data.length) {
     throw Error(`BitList attempting to read byte ${end} of data length ${data.length}`);
@@ -163,15 +170,13 @@ export function deserializeUint8ArrayBitListFromBytes(
   }
 
   if (lastByte === 1) {
-    // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087
-    const uint8Array = Uint8Array.prototype.slice.call(data, start, end - 1);
+    const uint8Array = slice(data, start, end - 1, reuseBytes);
     const bitLen = (size - 1) * 8;
     return {uint8Array, bitLen};
   }
 
   // the last byte is > 1, so a padding bit will exist in the last byte and need to be removed
-  // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087
-  const uint8Array = Uint8Array.prototype.slice.call(data, start, end);
+  const uint8Array = slice(data, start, end, reuseBytes);
   // mask lastChunkByte
   const lastByteBitLength = lastByte.toString(2).length - 1;
   const bitLen = (size - 1) * 8 + lastByteBitLength;

packages/ssz/src/type/bitVector.ts

@@ -1,4 +1,5 @@
 import {Node, getNodesAtDepth, packedNodeRootsToBytes, packedRootsBytesToNode} from "@chainsafe/persistent-merkle-tree";
+import {slice} from "../util/byteArray.ts";
 import {maxChunksToDepth} from "../util/merkleize.ts";
 import {namedClass} from "../util/named.ts";
 import {Require} from "../util/types.ts";
@@ -78,10 +79,9 @@ export class BitVectorType extends BitArrayType {
     return offset + this.fixedSize;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): BitArray {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): BitArray {
     this.assertValidLength(data.uint8Array, start, end);
-    // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087
-    return new BitArray(Uint8Array.prototype.slice.call(data.uint8Array, start, end), this.lengthBits);
+    return new BitArray(slice(data.uint8Array, start, end, reuseBytes), this.lengthBits);
   }
 
   tree_serializedSize(): number {

packages/ssz/src/type/byteArray.ts

@@ -7,7 +7,7 @@ import {
   getHashComputations,
   toGindex,
 } from "@chainsafe/persistent-merkle-tree";
-import {byteArrayEquals, fromHexString, toHexString} from "../util/byteArray.ts";
+import {byteArrayEquals, fromHexString, slice, toHexString} from "../util/byteArray.ts";
 import {ByteViews} from "./abstract.ts";
 import {CompositeType, LENGTH_GINDEX} from "./composite.ts";
 
@@ -74,9 +74,9 @@ export abstract class ByteArrayType extends CompositeType<ByteArray, ByteArray,
     return offset + value.length;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ByteArray {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ByteArray {
     this.assertValidSize(end - start);
-    return Uint8Array.prototype.slice.call(data.uint8Array, start, end);
+    return slice(data.uint8Array, start, end, reuseBytes);
   }
 
   value_toTree(value: ByteArray): Node {

packages/ssz/src/type/container.ts

@@ -214,14 +214,19 @@ export class ContainerType<Fields extends Record<string, Type<unknown>>> extends
     return variableIndex;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields<Fields> {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields<Fields> {
     const fieldRanges = this.getFieldRanges(data.dataView, start, end);
     const value = {} as {[K in keyof Fields]: unknown};
 
     for (let i = 0; i < this.fieldsEntries.length; i++) {
       const {fieldName, fieldType} = this.fieldsEntries[i];
       const fieldRange = fieldRanges[i];
-      value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end);
+      value[fieldName] = fieldType.value_deserializeFromBytes(
+        data,
+        start + fieldRange.start,
+        start + fieldRange.end,
+        reuseBytes
+      );
     }
 
     return value as ValueOfFields<Fields>;

packages/ssz/src/type/listComposite.ts

@@ -135,8 +135,13 @@ export class ListCompositeType<
     return value_serializeToBytesArrayComposite(this.elementType, value.length, output, offset, value);
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOf<ElementType>[] {
-    return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this);
+  value_deserializeFromBytes(
+    data: ByteViews,
+    start: number,
+    end: number,
+    reuseBytes?: boolean
+  ): ValueOf<ElementType>[] {
+    return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this, reuseBytes);
   }
 
   tree_serializedSize(node: Node): number {

packages/ssz/src/type/optional.ts

@@ -124,14 +124,19 @@ export class OptionalType<ElementType extends Type<unknown>> extends CompositeTy
     return offset;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfType<ElementType> {
+  value_deserializeFromBytes(
+    data: ByteViews,
+    start: number,
+    end: number,
+    reuseBytes?: boolean
+  ): ValueOfType<ElementType> {
     if (start === end) return null as ValueOfType<ElementType>;
 
     const selector = data.uint8Array[start];
     if (selector !== 1) {
       throw new Error(`Invalid selector for Optional type: ${selector}`);
     }
-    return this.elementType.value_deserializeFromBytes(data, start + 1, end) as ValueOfType<ElementType>;
+    return this.elementType.value_deserializeFromBytes(data, start + 1, end, reuseBytes) as ValueOfType<ElementType>;
   }
 
   tree_serializedSize(node: Node): number {

packages/ssz/src/type/profile.ts

@@ -266,7 +266,7 @@ export class ProfileType<Fields extends Record<string, Type<unknown>>> extends C
     return variableIndex;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields<Fields> {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields<Fields> {
     const {optionalFields, fieldRanges} = this.getFieldRanges(data, start, end);
     const value = {} as {[K in keyof Fields]: unknown};
     const optionalFieldsLen = optionalFields.uint8Array.length;
@@ -280,7 +280,12 @@ export class ProfileType<Fields extends Record<string, Type<unknown>>> extends C
         continue;
       }
       const fieldRange = fieldRanges[i];
-      value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end);
+      value[fieldName] = fieldType.value_deserializeFromBytes(
+        data,
+        start + fieldRange.start,
+        start + fieldRange.end,
+        reuseBytes
+      );
     }
 
     return value as ValueOfFields<Fields>;

packages/ssz/src/type/stableContainer.ts

@@ -258,7 +258,7 @@ export class StableContainerType<Fields extends Record<string, Type<unknown>>> e
     return variableIndex;
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfFields<Fields> {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfFields<Fields> {
     const {activeFields, fieldRanges} = this.getFieldRanges(data, start, end);
     const value = {} as {[K in keyof Fields]: unknown};
 
@@ -270,7 +270,12 @@ export class StableContainerType<Fields extends Record<string, Type<unknown>>> e
       }
 
       const fieldRange = fieldRanges[rangesIx++];
-      value[fieldName] = fieldType.value_deserializeFromBytes(data, start + fieldRange.start, start + fieldRange.end);
+      value[fieldName] = fieldType.value_deserializeFromBytes(
+        data,
+        start + fieldRange.start,
+        start + fieldRange.end,
+        reuseBytes
+      );
     }
 
     return value as ValueOfFields<Fields>;

packages/ssz/src/type/union.ts

@@ -141,15 +141,15 @@ export class UnionType<Types extends Type<unknown>[]> extends CompositeType<
     return this.types[value.selector].value_serializeToBytes(output, offset + 1, value.value);
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOfTypes<Types> {
+  value_deserializeFromBytes(data: ByteViews, start: number, end: number, reuseBytes?: boolean): ValueOfTypes<Types> {
     const selector = data.uint8Array[start];
     if (selector > this.maxSelector) {
       throw Error(`Invalid selector ${selector}`);
     }
 
     return {
       selector,
-      value: this.types[selector].value_deserializeFromBytes(data, start + 1, end) as unknown,
+      value: this.types[selector].value_deserializeFromBytes(data, start + 1, end, reuseBytes) as unknown,
     } as ValueOfTypes<Types>;
   }
 

packages/ssz/src/type/vectorComposite.ts

@@ -116,8 +116,13 @@ export class VectorCompositeType<
     return value_serializeToBytesArrayComposite(this.elementType, this.length, output, offset, value);
   }
 
-  value_deserializeFromBytes(data: ByteViews, start: number, end: number): ValueOf<ElementType>[] {
-    return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this);
+  value_deserializeFromBytes(
+    data: ByteViews,
+    start: number,
+    end: number,
+    reuseBytes?: boolean
+  ): ValueOf<ElementType>[] {
+    return value_deserializeFromBytesArrayComposite(this.elementType, data, start, end, this, reuseBytes);
   }
 
   tree_serializedSize(node: Node): number {

packages/ssz/src/util/byteArray.ts

@@ -3,6 +3,19 @@ import {ByteVector} from "../interface.ts";
 // Caching this info costs about ~1000 bytes and speeds up toHexString() by x6
 const hexByByte = new Array<string>(256);
 
+/** Wrapper to select Uint8Array.slice or Uint8Array.subarray */
+export function slice(data: Uint8Array, start?: number, end?: number, reuseBytes?: boolean): Uint8Array {
+  // Buffer.prototype.slice does not copy memory, force use Uint8Array.prototype.slice https://github.com/nodejs/node/issues/28087
+  // - Uint8Array.prototype.slice: Copy memory, safe to mutate
+  // - Buffer.prototype.slice: Does NOT copy memory, mutation affects both views
+  // We could ensure that all Buffer instances are converted to Uint8Array before calling value_deserializeFromBytes
+  // However doing that in a browser friendly way is not easy. Downstream code uses `Uint8Array.prototype.slice.call`
+  // to ensure Buffer.prototype.slice is never used. Unit tests also test non-mutability.
+  return reuseBytes
+    ? Uint8Array.prototype.subarray.call(data, start, end)
+    : Uint8Array.prototype.slice.call(data, start, end);
+}
+
 export function toHexString(bytes: Uint8Array | ByteVector): string {
   let hex = "0x";
   for (const byte of bytes) {

packages/ssz/test/perf/eth2/deserialize.test.ts

@@ -38,6 +38,15 @@ describe("Deserialize frequent eth2 objects", () => {
         type.deserialize(bytes);
       },
     });
+
+    bench<Uint8Array, Uint8Array>({
+      id: `deserialize ${type.typeName} - struct (reuse bytes)`,
+      before: () => type.serialize(value),
+      beforeEach: (bytes) => bytes,
+      fn: (bytes) => {
+        type.deserialize(bytes, {reuseBytes: true});
+      },
+    });
   }
 
   for (const validatorCount of [300_000]) {

packages/ssz/test/unit/uint8Array.test.ts

@@ -1,27 +1,26 @@
 import {describe, expect, it} from "vitest";
+import {slice} from "../../src/util/byteArray.ts";
 
-describe("Mutability Buffer, Uint8Array", () => {
-  it("Ensure Uint8Array.slice copies memory", () => {
+describe("slice", () => {
+  it("Ensure slice copies memory with reuseBytes == false", () => {
     const len = 64;
     const index = 54;
     const newValue = 1;
 
     const u1 = new Uint8Array(len);
-    const u2 = u1.slice(0);
+    const u2 = slice(u1, 0, u1.length, false);
     u2[index] = newValue;
 
     expect(u1[index]).to.equal(0, "u1 should have original value");
     expect(u2[index]).to.equal(newValue, "u2 should have new value");
   });
-
-  // Buffer.prototype.slice does not copy memory, Enforce Uint8Array usage https://github.com/nodejs/node/issues/28087
-  it("Ensure Buffer does not copy memory", () => {
+  it("Ensure slice does not copy memory with reuseBytes == true", () => {
     const len = 64;
     const index = 54;
     const newValue = 1;
 
-    const u1 = Buffer.alloc(len, 0);
-    const u2 = u1.slice(0);
+    const u1 = new Uint8Array(len);
+    const u2 = slice(u1, 0, u1.length, true);
     u2[index] = newValue;
 
     expect(u1[index]).to.equal(newValue, "u1 should have new value");