Add simulator metrics subsystem with MQTT support

docs/sim-metrics-mqtt.md

@@ -0,0 +1,62 @@
+# Simulator MQTT Metrics
+
+This feature publishes low-overhead MNA simulator metrics to MQTT.
+
+The design goal is minimal simulator impact:
+
+- Metrics are captured with primitive counters in the MNA code path.
+- Publishing is batched by `integrations.mqtt.simMetrics.publishIntervalTicks`.
+- Network writes happen on a dedicated async thread (`eln-sim-metrics`), not on the simulator thread.
+
+## Configuration
+
+Configure in the main mod config under `integrations.mqtt`:
+
+- `enabled`: enables MQTT support globally.
+- `simMetrics.enabled`: enables simulator metrics publishing.
+- `simMetrics.server`: MQTT server name from `config/eln/mqtt.json`.
+- `simMetrics.id`: stream ID used in topic paths (default `server`).
+- `simMetrics.publishIntervalTicks`: number of simulator ticks per publish batch (default `20`).
+
+`config/eln/mqtt.json` still defines broker connection info (name/uri/credentials/prefix) as used by MQTT meters/controllers.
+
+## Topics
+
+`$id` below is `integrations.mqtt.simMetrics.id`.
+`$prefix` below is the optional per-server MQTT prefix from `config/eln/mqtt.json`.
+
+Base topic:
+
+- `$prefix/eln/sim/$id`
+
+Stat topics:
+
+- `.../stat/subsystems`
+- `.../stat/inversions`
+- `.../stat/singular_matrices`
+- `.../stat/inversion_avg_ns`
+- `.../stat/inversion_max_ns`
+- `.../stat/tick_us`
+- `.../stat/electrical_us`
+- `.../stat/thermal_fast_us`
+- `.../stat/thermal_slow_us`
+- `.../stat/slow_us`
+- `.../stat/subsystems_current`
+- `.../stat/electrical_processes`
+- `.../stat/thermal_fast_loads`
+- `.../stat/thermal_fast_connections`
+- `.../stat/thermal_fast_processes`
+- `.../stat/thermal_slow_loads`
+- `.../stat/thermal_slow_connections`
+- `.../stat/thermal_slow_processes`
+- `.../stat/slow_processes`
+
+Info topics (retained):
+
+- `.../info/source` (`simulator`)
+- `.../info/id` (stream ID)
+
+## Notes
+
+- If `integrations.mqtt.simMetrics.enabled=true` and `integrations.mqtt.simMetrics.server` is blank, metrics are not published and a warning is logged.
+- This replaces Prometheus/CloudWatch ingestion for simulator metrics with MQTT outputs only.

src/main/java/mods/eln/Eln.java

@@ -41,6 +41,7 @@
 import mods.eln.item.electricalitem.PortableOreScannerItem.RenderStorage.OreScannerConfigElement;
 import mods.eln.misc.*;
 import mods.eln.mqtt.MqttManager;
+import mods.eln.metrics.MetricsSubsystem;
 import mods.eln.node.NodeBlockEntity;
 import mods.eln.node.NodeManager;
 import mods.eln.node.NodeManagerNbt;
@@ -196,6 +197,12 @@ public class Eln {
     public static OreItem oreItem;
     public static PortableNaNDescriptor portableNaNDescriptor = null;
     public static CableRenderDescriptor stdPortableNaN = null;
+    public static boolean mqttEnabled = false;
+    public static boolean simMetricsEnabled = false;
+    public static String simMetricsMqttServer = "";
+    public static String simMetricsId = "server";
+    public static int simMetricsPublishIntervalTicks = 20;
+    public static boolean debugEnabled = false;
     public static SiliconWafer siliconWafer;
     public static Transistor transistor;
     public static Thermistor thermistor;
@@ -329,6 +336,7 @@ public void preInit(FMLPreInitializationEvent event) {
         config.writeExampleFile();
         FuelRegistry.init(event.getSuggestedConfigurationFile());
         MqttManager.init();
+        MetricsSubsystem.refreshFromConfig();
 
         eventChannel = NetworkRegistry.INSTANCE.newEventDrivenChannel(channelName);
 
@@ -539,6 +547,7 @@ public void onServerStopped(FMLServerStoppedEvent ev) {
         LampSupplyElement.channelMap.clear();
         WirelessSignalTxElement.channelMap.clear();
         MqttManager.shutdown();
+        MetricsSubsystem.shutdown();
     }
 
     @EventHandler

src/main/java/mods/eln/sim/Simulator.java

@@ -4,7 +4,9 @@
 import cpw.mods.fml.common.eventhandler.SubscribeEvent;
 import cpw.mods.fml.common.gameevent.TickEvent.Phase;
 import cpw.mods.fml.common.gameevent.TickEvent.ServerTickEvent;
+import mods.eln.Eln;
 import mods.eln.environment.RoomThermalManager;
+import mods.eln.metrics.MetricsSubsystem;
 import mods.eln.misc.Utils;
 import mods.eln.sim.mna.RootSystem;
 import mods.eln.sim.mna.component.Component;
@@ -429,6 +431,11 @@ public void tick(ServerTickEvent event) {
             thermalFastNsStack /= 20;
             thermalSlowNsStack /= 20;
             slowNsStack /= 20;
+            double avgTickMicroseconds = avgTickTime;
+            double electricalMicroseconds = electricalNsStack / 1000.0;
+            double thermalFastMicroseconds = thermalFastNsStack / 1000.0;
+            double thermalSlowMicroseconds = thermalSlowNsStack / 1000.0;
+            double slowMicroseconds = slowNsStack / 1000.0;
 
             Utils.println("ticks " + new DecimalFormat("#").format((int) avgTickTime) + " us" + "  E " + electricalNsStack / 1000 + "  TF " + thermalFastNsStack / 1000 + "  TS " + thermalSlowNsStack / 1000 + "  S " + slowNsStack / 1000
 
@@ -442,6 +449,24 @@ public void tick(ServerTickEvent event) {
                 + "    " + thermalSlowProcessList.size() + " TSP"
                 + "    " + slowProcessList.size() + " SP"
             );
+            if (Eln.simMetricsEnabled) {
+                MetricsSubsystem.publishSimulatorRuntimeMetrics(
+                    avgTickMicroseconds,
+                    electricalMicroseconds,
+                    thermalFastMicroseconds,
+                    thermalSlowMicroseconds,
+                    slowMicroseconds,
+                    mna.getSubSystemCount(),
+                    electricalProcessList.size(),
+                    thermalFastLoadList.size(),
+                    thermalFastConnectionList.size(),
+                    thermalFastProcessList.size(),
+                    thermalSlowLoadList.size(),
+                    thermalSlowConnectionList.size(),
+                    thermalSlowProcessList.size(),
+                    slowProcessList.size()
+                );
+            }
 
             avgTickTime = 0;
 

src/main/java/mods/eln/sim/mna/MnaStepMetricsAccumulator.java

@@ -0,0 +1,76 @@
+package mods.eln.sim.mna;
+
+public class MnaStepMetricsAccumulator {
+    private int subSystemCount;
+    private int inversionCount;
+    private int singularMatrixCount;
+    private long inversionTotalNanoseconds;
+    private long inversionMaxNanoseconds;
+
+    public void reset() {
+        subSystemCount = 0;
+        inversionCount = 0;
+        singularMatrixCount = 0;
+        inversionTotalNanoseconds = 0L;
+        inversionMaxNanoseconds = 0L;
+    }
+
+    public void setSubSystemCount(int value) {
+        subSystemCount = value;
+    }
+
+    public void addInversion(long nanoseconds) {
+        inversionCount++;
+        inversionTotalNanoseconds += nanoseconds;
+        if (nanoseconds > inversionMaxNanoseconds) {
+            inversionMaxNanoseconds = nanoseconds;
+        }
+    }
+
+    public void addInversions(int count, long totalNanoseconds, long maxNanoseconds) {
+        if (count <= 0) {
+            return;
+        }
+        inversionCount += count;
+        inversionTotalNanoseconds += totalNanoseconds;
+        if (maxNanoseconds > inversionMaxNanoseconds) {
+            inversionMaxNanoseconds = maxNanoseconds;
+        }
+    }
+
+    public void addSingular(int count) {
+        singularMatrixCount += count;
+    }
+
+    public void add(MnaStepMetricsAccumulator other) {
+        inversionCount += other.inversionCount;
+        singularMatrixCount += other.singularMatrixCount;
+        inversionTotalNanoseconds += other.inversionTotalNanoseconds;
+        if (other.inversionMaxNanoseconds > inversionMaxNanoseconds) {
+            inversionMaxNanoseconds = other.inversionMaxNanoseconds;
+        }
+    }
+
+    public int getSubSystemCount() {
+        return subSystemCount;
+    }
+
+    public int getInversionCount() {
+        return inversionCount;
+    }
+
+    public int getSingularMatrixCount() {
+        return singularMatrixCount;
+    }
+
+    public long getInversionMaxNanoseconds() {
+        return inversionMaxNanoseconds;
+    }
+
+    public double getInversionAverageNanoseconds() {
+        if (inversionCount <= 0) {
+            return 0.0;
+        }
+        return (double) inversionTotalNanoseconds / inversionCount;
+    }
+}

src/main/java/mods/eln/sim/mna/RootSystem.java

@@ -1,5 +1,7 @@
 package mods.eln.sim.mna;
 
+import mods.eln.Eln;
+import mods.eln.metrics.MetricsSubsystem;
 import mods.eln.misc.Profiler;
 import mods.eln.misc.Utils;
 import mods.eln.sim.ElectricalLoad;
@@ -12,6 +14,9 @@
 import java.util.*;
 
 public class RootSystem {
+    private final MnaStepMetricsAccumulator stepMetricsAccumulator = new MnaStepMetricsAccumulator();
+    private final MnaStepMetricsAccumulator publishMetricsAccumulator = new MnaStepMetricsAccumulator();
+    private int publishTickCounter = 0;
 
     double dt;
     int interSystemOverSampling;
@@ -240,6 +245,32 @@ public void step() {
         }
 
         profiler.stop();
+
+        if (Eln.simMetricsEnabled) {
+            collectAndPublishSimMetrics();
+        }
+    }
+
+    private void collectAndPublishSimMetrics() {
+        stepMetricsAccumulator.reset();
+        stepMetricsAccumulator.setSubSystemCount(systems.size());
+        for (SubSystem s : systems) {
+            s.drainMnaMetrics(stepMetricsAccumulator);
+        }
+        publishMetricsAccumulator.setSubSystemCount(stepMetricsAccumulator.getSubSystemCount());
+        publishMetricsAccumulator.add(stepMetricsAccumulator);
+        publishTickCounter++;
+        if (publishTickCounter >= Eln.simMetricsPublishIntervalTicks) {
+            MetricsSubsystem.publishMnaMetrics(
+                    publishMetricsAccumulator.getSubSystemCount(),
+                    publishMetricsAccumulator.getInversionCount(),
+                    publishMetricsAccumulator.getSingularMatrixCount(),
+                    publishMetricsAccumulator.getInversionAverageNanoseconds(),
+                    publishMetricsAccumulator.getInversionMaxNanoseconds()
+            );
+            publishMetricsAccumulator.reset();
+            publishTickCounter = 0;
+        }
     }
 
     private void buildSubSystem(State root) {

src/main/java/mods/eln/sim/mna/SubSystem.java

@@ -32,6 +32,10 @@ public class SubSystem {
     int stateCount;
     double[][] A;
     boolean singularMatrix;
+    private int singularMatrixCountSinceLastDrain = 0;
+    private int inversionCountSinceLastDrain = 0;
+    private long inversionTotalNanosecondsSinceLastDrain = 0L;
+    private long inversionMaximumNanosecondsSinceLastDrain = 0L;
 
     DD[][] AInvdata;
     double[] Idata;
@@ -122,15 +126,23 @@ public void generateMatrix() {
 
         //	org.apache.commons.math3.linear.
 
+        long inversionStartNanoseconds = Eln.simMetricsEnabled ? System.nanoTime() : 0L;
         try {
             AInvdata = invertMatrix(A);
             singularMatrix = false;
+            if (Eln.simMetricsEnabled) {
+                long inversionTimeNanoseconds = System.nanoTime() - inversionStartNanoseconds;
+                inversionCountSinceLastDrain++;
+                inversionTotalNanosecondsSinceLastDrain += inversionTimeNanoseconds;
+                if (inversionTimeNanoseconds > inversionMaximumNanosecondsSinceLastDrain) {
+                    inversionMaximumNanosecondsSinceLastDrain = inversionTimeNanoseconds;
+                }
+            }
         } catch (Exception e) {
             singularMatrix = true;
             AInvdata = null;
             if (stateCount > 1) {
-                int idx = 0;
-                idx++;
+                singularMatrixCountSinceLastDrain++;
                 Utils.println("//////////SingularMatrix////////////");
             }
         }
@@ -236,6 +248,23 @@ public void step() {
         stepFlush();
     }
 
+    public void drainMnaMetrics(MnaStepMetricsAccumulator accumulator) {
+        if (singularMatrixCountSinceLastDrain != 0) {
+            accumulator.addSingular(singularMatrixCountSinceLastDrain);
+            singularMatrixCountSinceLastDrain = 0;
+        }
+        if (inversionCountSinceLastDrain != 0) {
+            accumulator.addInversions(
+                    inversionCountSinceLastDrain,
+                    inversionTotalNanosecondsSinceLastDrain,
+                    inversionMaximumNanosecondsSinceLastDrain
+            );
+            inversionCountSinceLastDrain = 0;
+            inversionTotalNanosecondsSinceLastDrain = 0L;
+            inversionMaximumNanosecondsSinceLastDrain = 0L;
+        }
+    }
+
     public void stepCalc() {
         if (!matrixValid) {
             generateMatrix();

src/main/kotlin/mods/eln/config/JsonConfig.kt

@@ -4,6 +4,7 @@ import com.google.gson.GsonBuilder
 import com.google.gson.JsonElement
 import com.google.gson.JsonObject
 import com.google.gson.JsonParser
+import mods.eln.Eln
 import mods.eln.Other
 import mods.eln.item.LampLists
 import mods.eln.item.TurbineBladeLists
@@ -78,6 +79,10 @@ class JsonConfig @JvmOverloads constructor(
         spec(path = "integrations.modbus.enabled", defaultValue = false, comment = "Enable Modbus RTU."),
         spec(path = "integrations.modbus.port", defaultValue = 1502, comment = "TCP port for Modbus RTU."),
         spec(path = "integrations.mqtt.enabled", defaultValue = false, comment = "Enable MQTT devices. Server endpoints live in config/eln/mqtt.json."),
+        spec(path = "integrations.mqtt.simMetrics.enabled", defaultValue = false, comment = "Publish simulator MNA metrics to MQTT."),
+        spec(path = "integrations.mqtt.simMetrics.server", defaultValue = "", comment = "MQTT server name in config/eln/mqtt.json used for simulator metrics."),
+        spec(path = "integrations.mqtt.simMetrics.id", defaultValue = "server", comment = "Topic ID used under eln/sim/<id>/... for simulator metrics."),
+        spec(path = "integrations.mqtt.simMetrics.publishIntervalTicks", defaultValue = 20, comment = "Simulator ticks to aggregate before MQTT publish. Higher values reduce overhead."),
         spec(path = "integrations.computerProbe.enabled", defaultValue = true, comment = "Enable the OC/CC to Eln computer probe."),
         spec(path = "integrations.energyExporter.enabled", defaultValue = true, comment = "Enable the Eln energy exporter."),
         spec(path = "integrations.oredict.tungstenEnabled", defaultValue = false, comment = "Use shared ore dictionary entries for tungsten."),
@@ -225,6 +230,12 @@ class JsonConfig @JvmOverloads constructor(
         Other.wattsToEu = getDoubleOrElse("balance.integrationConversion.wattsToEu", 1.0 / 3.0)
         Other.wattsToOC = getDoubleOrElse("balance.integrationConversion.wattsToOc", 1.0 / 3.0 / 2.5)
         Other.wattsToRf = getDoubleOrElse("balance.integrationConversion.wattsToRf", 1.0 / 3.0 * 4)
+        Eln.mqttEnabled = getBooleanOrElse("integrations.mqtt.enabled", false)
+        Eln.simMetricsEnabled = getBooleanOrElse("integrations.mqtt.simMetrics.enabled", false)
+        Eln.simMetricsMqttServer = getStringOrElse("integrations.mqtt.simMetrics.server", "")
+        Eln.simMetricsId = getStringOrElse("integrations.mqtt.simMetrics.id", "server")
+        Eln.simMetricsPublishIntervalTicks = max(1, getIntOrElse("integrations.mqtt.simMetrics.publishIntervalTicks", 20))
+        Eln.debugEnabled = getBooleanOrElse("debug.logging.enabled", false)
 
         setRuntimeValue(
             "runtime.items.batteries.standardHalfLifeTicks",
@@ -781,6 +792,10 @@ class JsonConfig @JvmOverloads constructor(
         "integrations.modbus.enabled" -> listOf(LegacyKey("modbus", "enable"))
         "integrations.modbus.port" -> listOf(LegacyKey("modbus", "port"))
         "integrations.mqtt.enabled" -> listOf(LegacyKey("mqtt", "enable"))
+        "integrations.mqtt.simMetrics.enabled" -> listOf(LegacyKey("mqtt", "simMetricsEnable"))
+        "integrations.mqtt.simMetrics.server" -> listOf(LegacyKey("mqtt", "simMetricsServer"))
+        "integrations.mqtt.simMetrics.id" -> listOf(LegacyKey("mqtt", "simMetricsId"))
+        "integrations.mqtt.simMetrics.publishIntervalTicks" -> listOf(LegacyKey("mqtt", "simMetricsPublishIntervalTicks"))
         "integrations.computerProbe.enabled" -> listOf(LegacyKey("compatibility", "ComputerProbeEnable"))
         "integrations.energyExporter.enabled" -> listOf(LegacyKey("compatibility", "ElnToOtherEnergyConverterEnable"))
         "integrations.oredict.tungstenEnabled" -> listOf(LegacyKey("dictionary", "tungsten"))