feat(hpc): guard SLURM submit mode inside submitted jobs

Author: harryswift01

CodeEntropy/config/runtime.py

@@ -258,6 +258,16 @@ def run_entropy_workflow(self) -> None:
 
                 args = self._config_manager.resolve(args, run_config)
 
+                if getattr(args, "submit", False):
+                    if os.environ.get("CODEENTROPY_SUBMITTED_JOB") == "1":
+                        run_logger.info(
+                            "Already running inside submitted SLURM job; "
+                            "continuing workflow."
+                        )
+                    else:
+                        HPCDaskManager(args).submit_master()
+                        return
+
                 log_level = (
                     logging.DEBUG if getattr(args, "verbose", False) else logging.INFO
                 )
@@ -268,11 +278,6 @@ def run_entropy_workflow(self) -> None:
 
                 self._validate_required_args(args)
 
-                if getattr(args, "submit", False):
-                    self._config_manager._check_parallel_frame_options(args)
-                    HPCDaskManager(args).submit_master()
-                    return
-
                 self.print_args_table(args)
 
                 universe_operations = UniverseOperations()
@@ -305,6 +310,7 @@ def run_entropy_workflow(self) -> None:
                 except Exception:
                     logger.error("Run arguments at failure could not be serialized")
 
+            logger.exception("Fatal error during entropy calculation")
             raise RuntimeError("CodeEntropyRunner encountered an error") from exc
 
     @staticmethod