Implement benchmark comparison of different feture flag configurations (#52925)

Summary:
Pull Request resolved: #52925

# Changelog:
[Internal] -

This adds an extra "ranking" report when running Fantom benchmark vs different React Feature flag configurations.

It can be very useful when implementing some particular optimization, to streamline the before/after comparison wit this optimization enabled/disabled.

Reviewed By: andrewdacenko

Differential Revision: D79269601

fbshipit-source-id: f29e761e313d6857e5b3ac65faf2a387a84be9df

Author: rshest

private/react-native-fantom/runner/benchmarkUtils.js

@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict-local
+ * @format
+ */
+
+type TestTaskTiming = {
+  name: string,
+  latency: {
+    mean: number,
+    min: number,
+    max: number,
+    p50: number,
+    p75: number,
+    p99: number,
+  },
+};
+
+export type BenchmarkTestArtifact = {
+  type: string,
+  timings: $ReadOnlyArray<TestTaskTiming>,
+};
+
+export const printBenchmarkResultsRanking = (
+  testResults: Array<{
+    title: string,
+    testArtifact: mixed,
+  }>,
+) => {
+  const numberOfTests = testResults.length;
+  if (numberOfTests <= 1 || testResults[0].testArtifact === undefined) {
+    // No test variations to compare between, or there are no benchmak results present, just don't print anything
+    return;
+  }
+
+  const testTaskTimings: {[string]: Array<[string, number]>} = {};
+  for (let i = 0; i < numberOfTests; i++) {
+    if (testResults[i] == null) {
+      continue;
+    }
+    // $FlowExpectedError[incompatible-cast]
+    const testArtifact = testResults[i].testArtifact as ?BenchmarkTestArtifact;
+    const testVariationName = testResults[i].title;
+    if (
+      testArtifact == null ||
+      testArtifact.type !== 'benchmark' ||
+      testVariationName == null
+    ) {
+      continue;
+    }
+    for (const taskTiming of testArtifact.timings) {
+      const taskName = taskTiming.name;
+      if (testTaskTimings[taskName] === undefined) {
+        testTaskTimings[taskName] = [];
+      }
+      testTaskTimings[taskName].push([
+        testVariationName,
+        taskTiming.latency.p50,
+      ]);
+    }
+  }
+
+  // Sort each task's executions by latency
+  for (const taskName in testTaskTimings) {
+    testTaskTimings[taskName].sort((a, b) => a[1] - b[1]);
+  }
+
+  // Print the rankings
+  console.log('### Benchmark Results Ranking ###');
+  for (const taskName in testTaskTimings) {
+    console.log(`> ${taskName}:`);
+    let lastTiming;
+    for (const [i, [testVariationName, latency]] of testTaskTimings[
+      taskName
+    ].entries()) {
+      console.log(
+        `  ${i + 1}. ${testVariationName}: ${latency.toFixed(2)}ms ${getTimingDelta(lastTiming, latency)}`,
+      );
+      lastTiming = latency;
+    }
+    console.log('\n');
+  }
+};
+
+function getTimingDelta(lastTiming: ?number, currentTiming: ?number): string {
+  if (lastTiming != null && currentTiming != null) {
+    const deltaPercent = ((currentTiming - lastTiming) / lastTiming) * 100;
+    return `(${deltaPercent.toFixed(2)}% ${deltaPercent > 0 ? 'slower' : 'faster'})`;
+  } else {
+    return '';
+  }
+}

private/react-native-fantom/runner/runner.js

@@ -14,13 +14,15 @@ import type {
   TestSuiteResult,
 } from '../runtime/setup';
 import type {TestSnapshotResults} from '../runtime/snapshotContext';
+import type {BenchmarkTestArtifact} from './benchmarkUtils';
 import type {FantomTestConfig} from './getFantomTestConfigs';
 import type {
   AsyncCommandResult,
   ConsoleLogMessage,
   HermesVariant,
 } from './utils';
 
+import {printBenchmarkResultsRanking} from './benchmarkUtils';
 import {createBundle, createSourceMap} from './bundling';
 import entrypointTemplate from './entrypoint-template';
 import * as EnvironmentOptions from './EnvironmentOptions';
@@ -233,6 +235,7 @@ module.exports = async function runTest(
       snapshotResults: {} as TestSnapshotResults,
       status: 'pending' as TestCaseResult['status'],
       testFilePath: testPath,
+      testArtifact: {} as mixed,
       title,
     },
   ];
@@ -431,6 +434,18 @@ module.exports = async function runTest(
     snapshotResults,
   );
 
+  printBenchmarkResultsRanking(
+    testResults.map(testResult => {
+      // $FlowExpectedError[incompatible-cast]
+      const testArtifact = testResult.testArtifact as ?BenchmarkTestArtifact;
+      const title = testResult.ancestorTitles[0];
+      return {
+        title,
+        testArtifact,
+      };
+    }),
+  );
+
   return {
     testFilePath: testPath,
     failureMessage: formatResultsErrors(

private/react-native-fantom/runtime/setup.js

@@ -26,6 +26,7 @@ export type TestCaseResult = {
   failureDetails: Array<FailureDetail>,
   numPassingAsserts: number,
   snapshotResults: TestSnapshotResults,
+  testArtifact?: mixed,
   // location: string,
 };
 
@@ -314,6 +315,7 @@ function runSpec(spec: Spec): TestCaseResult {
     failureDetails: [],
     numPassingAsserts: 0,
     snapshotResults: {},
+    testArtifact: null,
   };
 
   if (!shouldRunSuite(spec)) {
@@ -328,7 +330,7 @@ function runSpec(spec: Spec): TestCaseResult {
 
   try {
     invokeHooks(spec.parentContext, 'beforeEachHooks');
-    spec.implementation();
+    result.testArtifact = spec.implementation();
     invokeHooks(spec.parentContext, 'afterEachHooks');
 
     status = 'passed';

private/react-native-fantom/src/Benchmark.js

@@ -176,6 +176,7 @@ export function suite(
         'Failing focused test to prevent it from being committed',
       );
     }
+    return createBenchmarkTestArtifact(bench, tasks);
   });
 
   const test = (
@@ -253,3 +254,17 @@ function printBenchmarkResults(bench: Bench) {
   console.table(nullthrows(bench.table()));
   console.log('');
 }
+
+function createBenchmarkTestArtifact(bench: Bench, tasks: Array<TestTask>) {
+  return {
+    type: 'benchmark',
+    timings: tasks.map((task, i) => {
+      const result = bench.results[i];
+      const {min, max, mean, p50, p75, p99} = result.latency;
+      return {
+        name: task.name,
+        latency: {min, max, mean, p50, p75, p99},
+      };
+    }),
+  };
+}