Reset retry counter when there is progress (#296)

* Job completed_fraction API
* write completed_fraction into submit log
* Task resumable, retry count reset when there is progress

Fix #295

Author: albertz

sisyphus/engine.py

@@ -187,6 +187,9 @@ def submit(self, task):
             submit_info = rqmt.copy()
             submit_info["engine_info"] = engine_info
             submit_info["engine_name"] = engine_name
+            completed_fraction = task._job.completed_fraction()
+            assert completed_fraction is None or 0 <= completed_fraction <= 1, f"{task._job} {completed_fraction=!r}"
+            submit_info["completed_fraction"] = completed_fraction
             with open(submit_log, "a") as submit_file:
                 submit_file.write("%s\n" % str((task_ids, submit_info)))
 

sisyphus/job.py

@@ -1219,6 +1219,12 @@ def info(self):
         """
         pass
 
+    def completed_fraction(self) -> Optional[float]:
+        """
+        :return: fraction between 0 and 1 or None if not available
+        """
+        return None
+
     @classmethod
     def hash(cls, parsed_args: Dict[str, Any]) -> str:
         """

sisyphus/task.py

@@ -397,8 +397,12 @@ def _get_state_helper(self, engine, task_id=None):
                         # used to avoid wrongly marking jobs as interrupted do to slow filesystem updates
                         elif self.running(task_id):
                             return gs.STATE_RUNNING
-                        history = [] if engine is None else engine.get_submit_history(self)
-                        if history and len(history[task_id]) > gs.MAX_SUBMIT_RETRIES:
+                        history = [] if engine is None else engine.get_submit_history(self)[task_id]
+                        # Filter entries with less progress than the most recent entry.
+                        # https://github.com/rwth-i6/sisyphus/issues/295
+                        completed_fraction = (history[-1].get("completed_fraction") or 0) if history else 0
+                        history = [d for d in history if (d.get("completed_fraction") or 0) >= completed_fraction]
+                        if len(history) > gs.MAX_SUBMIT_RETRIES:
                             # More then three tries to run this task, something is wrong
                             return gs.STATE_RETRY_ERROR
                         else: