Add option to run the large network test on bare metal

AkkaDotNet.LargeNetworkTests.sln

@@ -27,6 +27,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "backend", "backend", "{BE2E
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AkkaDotNet.Messages", "src\shared\AkkaDotNet.Messages\AkkaDotNet.Messages.csproj", "{8F96042B-A435-4789-A92A-67B9C433E293}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BareMetal.Local", "src\bare-metal\BareMetal.Local\BareMetal.Local.csproj", "{09A7128A-599C-4ED5-ACB0-BBFA4A8A4CC7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -55,6 +57,10 @@ Global
 		{8F96042B-A435-4789-A92A-67B9C433E293}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8F96042B-A435-4789-A92A-67B9C433E293}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8F96042B-A435-4789-A92A-67B9C433E293}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09A7128A-599C-4ED5-ACB0-BBFA4A8A4CC7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{09A7128A-599C-4ED5-ACB0-BBFA4A8A4CC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{09A7128A-599C-4ED5-ACB0-BBFA4A8A4CC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{09A7128A-599C-4ED5-ACB0-BBFA4A8A4CC7}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/Directory.Packages.props

@@ -14,6 +14,7 @@
     <PackageVersion Include="Akka.Logger.Serilog" Version="1.5.12" />
     <PackageVersion Include="Akka.Management" Version="$(AkkaManagementVersion)" />
     <PackageVersion Include="Akka.Persistence.Azure" Version="1.5.1" />
+    <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>
   <!-- Phobos -->
   <ItemGroup>

src/backend/AkkaDotNet.BackEnd/Program.cs

@@ -24,7 +24,7 @@
 builder.Services.AddAkka(ActorSystemConstants.ActorSystemName, configurationBuilder =>
 {
     configurationBuilder.WithStressCluster(akkaConfiguration,
-        new[] { ActorSystemConstants.BackendRole, ActorSystemConstants.DistributedPubSubRole });
+        new[] { ActorSystemConstants.BackendRole, ActorSystemConstants.DistributedPubSubRole }, builder.Configuration);
     configurationBuilder.WithSerilog(akkaConfiguration.SerilogOptions);
     configurationBuilder.WithReadyCheckActors();
     if (akkaConfiguration.ShardingOptions.Enabled)
@@ -53,13 +53,20 @@
 
     }
 });
-builder.Services.WithOpenTelemetry();
+
+if (akkaConfiguration.EnableOpenTelemetry)
+{
+    builder.Services.WithOpenTelemetry();
+}
 
 var app = builder.Build();
 
-// per https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.Prometheus/README.md
 app.UseRouting();
-app.UseOpenTelemetryPrometheusScrapingEndpoint();
+if (akkaConfiguration.EnableOpenTelemetry)
+{
+    // per https://github.com/open-telemetry/opentelemetry-dotnet/blob/fe78453c03feb8dbe506b2a0284312bdfa1367c5/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md
+    app.UseOpenTelemetryPrometheusScrapingEndpoint();
+}
 app.UseEndpoints(endpoints =>
 {
     endpoints.MapControllers();

src/backend/AkkaDotNet.BackEnd/appsettings.json

@@ -34,7 +34,8 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "Akka": "Debug"
     }
   },
   "AllowedHosts": "*"

src/bare-metal/BareMetal.Local/BareMetal.Local.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\..\backend\AkkaDotNet.BackEnd\AkkaDotNet.BackEnd.csproj" />
+      <ProjectReference Include="..\..\frontend\AkkaDotNet.FrontEnd\AkkaDotNet.FrontEnd.csproj" />
+    </ItemGroup>
+
+
+    <ItemGroup>
+      <PackageReference Include="System.CommandLine" />
+    </ItemGroup>
+
+</Project>

src/bare-metal/BareMetal.Local/Program.cs

@@ -0,0 +1,103 @@
+using System.CommandLine;
+using System.Diagnostics;
+
+namespace BareMetal.Local;
+
+public static class Program
+{
+    private const int StartPort = 9221;
+    private static readonly List<Process> RunningProcesses = new ();
+    private static string _workingDirectory;
+
+    static Program()
+    {
+        var currentProcess = Process.GetCurrentProcess();
+        var processPath = currentProcess.MainModule?.FileName ?? Environment.CurrentDirectory;
+        _workingDirectory = Path.GetDirectoryName(processPath) ?? Environment.CurrentDirectory;
+    }
+
+    public static async Task<int> Main(string[] args)
+    {
+        var backendOption = new Option<int>(
+            aliases: new[] { "--backend", "-b" }, 
+            description:"Number of backend roles to start.", getDefaultValue: () => 3);
+
+        var rootCommand = new RootCommand("Run AkkaDotNet.LargeNetworkTests on bare metal")
+        {
+            backendOption
+        };
+
+        rootCommand.SetHandler(async backend =>
+        {
+            await Startup(backend);
+        }, backendOption);
+
+        return await rootCommand.InvokeAsync(args);
+    }
+
+    private static async Task Startup(int nodeCount)
+    {
+        foreach (var i in Enumerable.Range(0, nodeCount))
+        {
+            StartBackEnd(StartPort + i * 4);
+        }
+
+        StartFrontEnd(5000);
+        Console.WriteLine("Press any key to stop.");
+        Console.ReadKey();
+
+        var tasks = RunningProcesses.Select(p =>
+        {
+            p.Kill();
+            return p.WaitForExitAsync();
+        });
+        await Task.WhenAll(tasks);
+    }
+
+    private static Process StartBackEnd(int port)
+    {
+        var p = new Process
+        {
+            StartInfo = new ProcessStartInfo("dotnet")
+            {
+                Arguments = "AkkaDotNet.BackEnd.dll",
+                Environment = 
+                {
+                    ["EnableOpenTelemetry"] = "false",
+                    ["StressOptions__AkkaClusterOptions__Port"] = (port + 0).ToString(),
+                    ["StressOptions__AkkaClusterOptions__ManagementPort"] = (port + 1).ToString(),
+                    ["petabridge__cmd__port"] = (port + 2).ToString(),
+                    ["ASPNETCORE_URLS"] = $"http://localhost:{port + 3}",
+                }, 
+                WorkingDirectory = _workingDirectory
+            }
+        };
+        RunningProcesses.Add(p);
+        p.Start();
+        return p;
+    }
+
+    private static Process StartFrontEnd(int port)
+    {
+        var p = new Process
+        {
+            StartInfo = new ProcessStartInfo("dotnet")
+            {
+                Arguments = "AkkaDotNet.FrontEnd.dll",
+                Environment = 
+                {
+                    ["EnableOpenTelemetry"] = "false",
+                    ["ASPNETCORE_URLS"] = $"http://localhost:{port}",
+                    ["petabridge__cmd__port"] = (port + 1).ToString(),
+                    ["StressOptions__AkkaClusterOptions__Port"] = (port + 2).ToString(),
+                    ["StressOptions__AkkaClusterOptions__ManagementPort"] = (port + 3).ToString(),
+                }, 
+                WorkingDirectory = _workingDirectory 
+            }
+        };
+        RunningProcesses.Add(p);
+        p.Start();
+        return p;
+    }
+}
+

src/frontend/AkkaDotNet.FrontEnd/Program.cs

@@ -18,7 +18,7 @@
 builder.Services.AddAkka(ActorSystemConstants.ActorSystemName, configurationBuilder =>
 {
     configurationBuilder.WithStressCluster(akkaConfiguration,
-        new[] { ActorSystemConstants.FrontendRole, ActorSystemConstants.DistributedPubSubRole });
+        new[] { ActorSystemConstants.FrontendRole, ActorSystemConstants.DistributedPubSubRole }, builder.Configuration);
     configurationBuilder.WithSerilog(akkaConfiguration.SerilogOptions);
     configurationBuilder.WithReadyCheckActors();
     if (akkaConfiguration.DistributedPubSubOptions.Enabled)
@@ -34,13 +34,19 @@
     }
 });
 
-builder.Services.WithOpenTelemetry();
+if (akkaConfiguration.EnableOpenTelemetry)
+{
+    builder.Services.WithOpenTelemetry();    
+}
 
 var app = builder.Build();
 
-// per https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.Prometheus/README.md
 app.UseRouting();
-app.UseOpenTelemetryPrometheusScrapingEndpoint();
+if (akkaConfiguration.EnableOpenTelemetry)
+{
+    // per https://github.com/open-telemetry/opentelemetry-dotnet/blob/fe78453c03feb8dbe506b2a0284312bdfa1367c5/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md
+    app.UseOpenTelemetryPrometheusScrapingEndpoint();
+}
 app.UseEndpoints(endpoints =>
 {
     endpoints.MapControllers();

src/frontend/AkkaDotNet.FrontEnd/appsettings.json

@@ -30,7 +30,8 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
+      "Microsoft.Hosting.Lifetime": "Information",
+      "Akka": "Debug"
     }
   },
   "AllowedHosts": "*"

src/shared/AkkaDotNet.Infrastructure/Configuration/StressHostingExtensions.cs

@@ -11,6 +11,7 @@
 using AkkaDotNet.Infrastructure.Actors;
 using AkkaDotNet.Infrastructure.Persistence;
 using AkkaDotNet.Messages;
+using Microsoft.Extensions.Configuration;
 using Petabridge.Cmd.Cluster;
 using Petabridge.Cmd.Cluster.Sharding;
 using Petabridge.Cmd.Host;
@@ -103,7 +104,7 @@ public static class StressHostingExtensions
         akka.remote.dot-netty.tcp.maximum-frame-size = 1m
     ";
 
-    public static AkkaConfigurationBuilder WithStressCluster(this AkkaConfigurationBuilder builder, StressOptions options, IEnumerable<string> roles)
+    public static AkkaConfigurationBuilder WithStressCluster(this AkkaConfigurationBuilder builder, StressOptions options, IEnumerable<string> roles, IConfiguration configuration)
     {
         var clusterOptions = new ClusterOptions() { Roles = roles.ToArray() };
 
@@ -167,11 +168,8 @@ public static AkkaConfigurationBuilder WithStressCluster(this AkkaConfigurationB
         }
 
         builder = builder
-            .ConfigureLoggers(configBuilder =>
-            {
-                configBuilder.LogConfigOnStart = true;
-            })
             .AddHocon(MaxFrameSize, HoconAddMode.Prepend)
+            .AddHocon(configuration.GetSection("petabridge"), HoconAddMode.Prepend)
             .WithRemoting("0.0.0.0", options.AkkaClusterOptions.Port, options.AkkaClusterOptions.Hostname)
             .WithClustering(clusterOptions)
             .AddPersistence(options.PersistenceOptions)

src/shared/AkkaDotNet.Infrastructure/Configuration/StressOptions.cs

@@ -16,6 +16,8 @@ public class StressOptions
     public DispatcherConfig DispatcherConfig { get; set; } = DispatcherConfig.Defaults;
 
     public PersistenceOptions PersistenceOptions { get; set; } = new PersistenceOptions();
+
+    public bool EnableOpenTelemetry { get; set; } = true;
 }
 
 public class DistributedPubSubOptions

src/shared/AkkaDotNet.Infrastructure/OpenTelemetry/OpenTelemetryConfigurationExtensions.cs

@@ -45,16 +45,17 @@ public static IServiceCollection WithOpenTelemetry(this IServiceCollection servi
                     {
                         options.Filter = context => !context.Request.Path.StartsWithSegments("/metrics");
                     })
-                    .SetSampler(new TraceIdRatioBasedSampler(1.0d))
-                    .AddOtlpExporter(options =>
+                    .SetSampler(new TraceIdRatioBasedSampler(1.0d));
+
+                var oTelEndpoint = Environment.GetEnvironmentVariable(JaegerAgentHostEnvironmentVar);
+                if (!string.IsNullOrWhiteSpace(oTelEndpoint))
+                {
+                    builder.AddOtlpExporter(options =>
                     {
-                        var endpoint = Environment.GetEnvironmentVariable(JaegerAgentHostEnvironmentVar);
-                        if (endpoint is not null)
-                        {
-                            options.Endpoint = new Uri($"http://{endpoint}:4317");
-                            options.Protocol = OtlpExportProtocol.Grpc;
-                        }
+                        options.Endpoint = new Uri($"http://{oTelEndpoint}:4317");
+                        options.Protocol = OtlpExportProtocol.Grpc;
                     });
+                }
             })
             .WithMetrics(builder =>
             {
@@ -65,7 +66,7 @@ public static IServiceCollection WithOpenTelemetry(this IServiceCollection servi
                     .AddHttpClientInstrumentation()
                     .AddAspNetCoreInstrumentation()
                     .AddMeter(AkkaStressSource)
-                    .AddPrometheusExporter(_ => { });
+                    .AddPrometheusExporter();
             });
 
         return services;