Improve binary search for apply-load.

The binary search uses t-statistic to ensure the necessary confidence at each step. This results in needing less samples far away from the true value and more samples around it. We still need at least 30 samples though to keep math simple. This works reasonably well, but still doesn't avoid some fundamental variance issues that we have that result in very different and statistically significant performance difference between different runs. That concern should hopefully be addressed separately.

Also fix some SAC max TPS benchmark issues, the math got messed up after I've updated it from 1s granularity to 50ms.

Author: dmkozh

Builds/VisualStudio/stellar-core.vcxproj

@@ -577,6 +577,7 @@ exit /b 0
     <ClCompile Include="..\..\src\invariant\AccountSubEntriesCountIsValid.cpp" />
     <ClCompile Include="..\..\src\invariant\ArchivedStateConsistency.cpp" />
     <ClCompile Include="..\..\src\invariant\BucketListIsConsistentWithDatabase.cpp" />
+    <ClCompile Include="..\..\src\invariant\BucketListStateConsistency.cpp" />
     <ClCompile Include="..\..\src\invariant\ConservationOfLumens.cpp" />
     <ClCompile Include="..\..\src\invariant\ConstantProductInvariant.cpp" />
     <ClCompile Include="..\..\src\invariant\EventsAreConsistentWithEntryDiffs.cpp" />
@@ -1044,6 +1045,7 @@ exit /b 0
     <ClInclude Include="..\..\src\invariant\AccountSubEntriesCountIsValid.h" />
     <ClInclude Include="..\..\src\invariant\ArchivedStateConsistency.h" />
     <ClInclude Include="..\..\src\invariant\BucketListIsConsistentWithDatabase.h" />
+    <ClInclude Include="..\..\src\invariant\BucketListStateConsistency.h" />
     <ClInclude Include="..\..\src\invariant\ConservationOfLumens.h" />
     <ClInclude Include="..\..\src\invariant\ConstantProductInvariant.h" />
     <ClInclude Include="..\..\src\invariant\EventsAreConsistentWithEntryDiffs.h" />

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -1420,6 +1420,12 @@
     <ClCompile Include="..\..\src\invariant\ArchivedStateConsistency.cpp">
       <Filter>invariant</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\ledger\LedgerEntryScope.cpp" />
+    <ClCompile Include="..\..\src\util\MetricsRegistry.cpp" />
+    <ClCompile Include="..\..\src\util\SimpleTimer.cpp" />
+    <ClCompile Include="..\..\src\invariant\BucketListStateConsistency.cpp">
+      <Filter>invariant</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\lib\util\cpptoml.h">
@@ -2518,6 +2524,12 @@
     <ClInclude Include="..\..\src\invariant\ArchivedStateConsistency.h">
       <Filter>invariant</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\ledger\LedgerEntryScope.h" />
+    <ClInclude Include="..\..\src\util\MetricsRegistry.h" />
+    <ClInclude Include="..\..\src\util\SimpleTimer.h" />
+    <ClInclude Include="..\..\src\invariant\BucketListStateConsistency.h">
+      <Filter>invariant</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\..\AUTHORS" />

src/main/CommandLine.cpp

@@ -1897,19 +1897,20 @@ runApplyLoad(CommandLineArgs const& args)
             config.TESTING_UPGRADE_MAX_TX_SET_SIZE = 1000;
             config.LEDGER_PROTOCOL_VERSION =
                 Config::CURRENT_LEDGER_PROTOCOL_VERSION;
-            if (config.APPLY_LOAD_NUM_LEDGERS == 0)
+            if (config.APPLY_LOAD_NUM_LEDGERS < 30)
             {
                 throw std::runtime_error(
-                    "APPLY_LOAD_NUM_LEDGERS must be greater than 0");
+                    "APPLY_LOAD_NUM_LEDGERS must be at least 30");
             }
             if (mode == ApplyLoadMode::MAX_SAC_TPS)
             {
-                if (config.APPLY_LOAD_MAX_SAC_TPS_MIN_TPS >=
+                if (config.APPLY_LOAD_MAX_SAC_TPS_MIN_TPS >
                     config.APPLY_LOAD_MAX_SAC_TPS_MAX_TPS)
                 {
                     throw std::runtime_error(
-                        "APPLY_LOAD_MAX_SAC_TPS_MIN_TPS must be less than "
-                        "APPLY_LOAD_MAX_SAC_TPS_MAX_TPS for max_sac_tps mode");
+                        "APPLY_LOAD_MAX_SAC_TPS_MIN_TPS must not be greater "
+                        "than APPLY_LOAD_MAX_SAC_TPS_MAX_TPS for max_sac_tps "
+                        "mode");
                 }
 
                 // For now, metrics are expensive at high, parallel load. We

src/main/Config.cpp

@@ -1752,15 +1752,6 @@ Config::processConfig(std::shared_ptr<cpptoml::table> t)
                  [&]() {
                      APPLY_LOAD_TARGET_CLOSE_TIME_MS =
                          readInt<uint32_t>(item, 1);
-                     if (APPLY_LOAD_TARGET_CLOSE_TIME_MS %
-                             ApplyLoad::TARGET_CLOSE_TIME_STEP_MS !=
-                         0)
-                     {
-                         throw std::invalid_argument(fmt::format(
-                             FMT_STRING("APPLY_LOAD_TARGET_CLOSE_TIME_MS "
-                                        "must be a multiple of {}."),
-                             ApplyLoad::TARGET_CLOSE_TIME_STEP_MS));
-                     }
                  }},
                 {"APPLY_LOAD_MAX_SAC_TPS_MIN_TPS",
                  [&]() {