fix(actor): address PR #22 review — TD3 signature, tuple skip, docs, reset test

- learn_TD3 now accepts recurrent=False to match the DDPG/RDDPG signatures;
  the learn_gsp dispatch was passing 3 positional args to a 2-arg method.
  Latent bug today (GSP networks are built as DDPG/attention, not TD3) but
  removes the footgun before the diagnostic batch exercises TD3 variants.
- TD3's non-actor-update step returns (0, 0); previously we unwrapped to
  0.0 and logged it. That produces false collapse signals every
  update_actor_iter-1 ticks. Now we skip the entry entirely — leave
  last_gsp_loss at None as if no GSP step ran.
- Doc the semantic: last_gsp_loss is the GSP learner's training loss,
  which is actor loss (policy-gradient signal) for DDPG/RDDPG/TD3 and
  genuine MSE only for attention. For prediction-collapse detection
  consumers should rely on gsp_squared_error and the HDF5Logger
  episode-level gsp_output_std / gsp_pred_target_corr attrs.
- Add reset-between-ticks test covering the load-bearing invariant that
  last_gsp_loss returns to None when a learn() call runs but no GSP
  learning step fires.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Joshua Bloom

gsp_rl/src/actors/actor.py

@@ -113,7 +113,16 @@ def __init__(
                 self.build_gsp_network('attention')
             self.build_gsp_network('DDPG')
 
-        # Information-collapse diagnostic: last GSP prediction network training loss.
+        # Information-collapse diagnostic: last GSP learner training loss.
+        # NOTE: this is the loss returned by the GSP learner's inner learn step, which means:
+        #   - For DDPG/RDDPG/TD3 GSP schemes: actor loss (a critic-derived policy-gradient
+        #     signal), NOT the prediction MSE against delta-theta. A collapsed predictor may
+        #     not produce an anomalous value here, since the critic's value landscape can
+        #     support multiple policy solutions.
+        #   - For the attention GSP scheme: genuine prediction MSE against the label.
+        # For prediction-collapse detection, prefer the raw per-step squared error captured
+        # in RL-CollectiveTransport as `gsp_squared_error` plus the episode-level
+        # `gsp_output_std` / `gsp_pred_target_corr` attrs computed in the Stelaris HDF5Logger.
         # Populated by learn_gsp() whenever a GSP learning step fires; reset to None at the
         # start of each learn() call so callers can distinguish "no GSP step this tick" from
         # "GSP step ran".
@@ -452,9 +461,12 @@ def learn_gsp(self):
         elif self.gsp_networks['learning_scheme'] == 'attention':
             loss = self.learn_attention(self.gsp_networks)
         if loss is not None:
-            # TD3's edge-case path returns (0, 0); normalize to a scalar for logging.
+            # TD3's non-actor-update steps return (0, 0) (critic stepped, actor did not).
+            # Recording a legitimate 0.0 there would produce false collapse signals every
+            # `update_actor_iter - 1` ticks, so skip those entries entirely — leave
+            # last_gsp_loss at None as if no GSP step ran this tick.
             if isinstance(loss, tuple):
-                loss = loss[0]
+                return
             self.last_gsp_loss = float(loss)
 
     def store_agent_transition(self, s, a, r, s_, d):

gsp_rl/src/actors/learning_aids.py

@@ -405,7 +405,7 @@ def learn_RDDPG(self, networks, gsp = False, recurrent = False):
 
         return actor_loss.item()
 
-    def learn_TD3(self, networks, gsp = False):
+    def learn_TD3(self, networks, gsp = False, recurrent = False):
         states, actions, rewards, states_, dones = self.sample_memory(networks)
 
         with T.no_grad():

tests/test_actor/test_gsp_loss_exposure.py

@@ -85,6 +85,29 @@ def test_last_gsp_loss_populated_after_learn_with_gsp(self):
         assert actor.last_gsp_loss is not None
         assert isinstance(actor.last_gsp_loss, float)
 
+    def test_last_gsp_loss_resets_between_ticks(self):
+        """Each learn() call starts by resetting last_gsp_loss to None.
+
+        This is the load-bearing invariant of the field: consumers must be able to read
+        it after learn() and distinguish "no GSP step ran this tick" (None) from
+        "GSP step ran and returned a value" (float). If the reset fails, a stale value
+        from a previous tick bleeds into the current tick's reading.
+        """
+        actor = make_gsp_actor()
+        fill_primary_and_gsp_buffers(actor)
+        actor.learn()
+        assert actor.last_gsp_loss is not None  # populated after first learn
+
+        # Drain the GSP replay buffer below the batch size so the next learn_gsp early-returns.
+        # Simulate by swapping in an empty gsp replay buffer. This is a white-box probe of the
+        # reset invariant rather than a full end-to-end run.
+        actor.gsp_networks['replay'].mem_ctr = 0
+
+        actor.learn()
+        assert actor.last_gsp_loss is None, (
+            "last_gsp_loss should reset to None when learn() runs but no GSP step fires"
+        )
+
     def test_last_gsp_loss_remains_none_when_gsp_disabled(self):
         """Non-GSP actor never populates last_gsp_loss."""
         actor = Actor(