SISGraph.{jobs,find,jobs_sorted} update_graph option (#293)

Author: albertz

sisyphus/graph.py

@@ -355,28 +355,32 @@ def job_directory_structure(self):
     def job_by_id(self, sis_id):
         return self.id_to_job_dict().get(sis_id)
 
-    def jobs(self) -> List[Job]:
+    def jobs(self, *, update_graph: bool = True) -> List[Job]:
         """
-        :return ([Job, ...]): List with all jobs in grpah
+        :param update_graph: if True, update all nodes/jobs (calling :func:`Job._sis_runnable`)
+             while running through the graph to get the most current dependency graph.
+        :return ([Job, ...]): List with all jobs in graph
         """
         job_list = []
 
         def f(job):
             job_list.append(job)
             return True
 
-        self.for_all_nodes(f)
+        self.for_all_nodes(f, update_graph=update_graph)
         return job_list
 
-    def find(self, pattern, mode="all"):
+    def find(self, pattern: str, mode: str = "all", *, update_graph: bool = True):
         """Returns a list with all jobs and paths that partly match the pattern
 
-        :param pattern(str): Pattern to match
-        :param mode(str): Select if jobs, paths or both should be returned. Possible values: all, path, job
+        :param pattern: Pattern to match
+        :param mode: Select if jobs, paths or both should be returned. Possible values: all, path, job
+        :param update_graph: if True, update all nodes/jobs (calling :func:`Job._sis_runnable`)
+             while running through the graph to get the most current dependency graph.
         :return ([Job/Path, ...]): List with all matching jobs/paths
         """
         out = set()
-        for j in self.jobs():
+        for j in self.jobs(update_graph=update_graph):
             if mode in ("all", "job"):
                 vis_name = j.get_vis_name()
                 aliases = j._sis_aliases if j._sis_aliases is not None else set()
@@ -392,10 +396,12 @@ def find(self, pattern, mode="all"):
                         out.add(p)
         return list(out)
 
-    def jobs_sorted(self):
+    def jobs_sorted(self, *, update_graph: bool = True):
         """Yields jobs in a order so that for each jop all jobs
         it depends on are already finished
 
+        :param update_graph: if True, update all nodes/jobs (calling :func:`Job._sis_runnable`)
+             while running through the graph to get the most current dependency graph.
         :return (generator Node): jobs sorted by dependency
         """
         id_to_job = {}
@@ -406,7 +412,7 @@ def get_job(sis_id):
             return id_to_job[sis_id]
 
         stack = []
-        for job in self.jobs():
+        for job in self.jobs(update_graph=update_graph):
             node = get_job(job._sis_id())
             node.job = job
             for i in job._sis_inputs:

sisyphus/toolkit.py

@@ -50,19 +50,19 @@
 import subprocess
 import importlib
 
-from sisyphus.tools import sh, extract_paths
-import sisyphus.block
-from sisyphus.block import block, sub_block, set_root_block
-from sisyphus.async_workflow import async_run, async_gather, async_context
-from sisyphus.delayed_ops import Delayed
-from sisyphus import cleaner
-
-from sisyphus.job_path import AbstractPath, Path, Variable
+from sisyphus.tools import sh, extract_paths  # noqa: F401
+import sisyphus.block  # noqa: F401
+from sisyphus.block import block, sub_block, set_root_block  # noqa: F401
+from sisyphus.async_workflow import async_run, async_gather, async_context  # noqa: F401
+from sisyphus.delayed_ops import Delayed  # noqa: F401
+from sisyphus import cleaner  # noqa: F401
+
+from sisyphus.job_path import AbstractPath, Path, Variable  # noqa: F401
 from sisyphus.job import Job
 from sisyphus import graph
 import sisyphus.global_settings as gs
 
-from sisyphus.loader import config_manager
+from sisyphus.loader import config_manager  # noqa: F401
 
 
 class BlockedWorkflow(Exception):
@@ -478,8 +478,8 @@ def import_directory(job):
     # run once before to unsure inputs are updated at least once
     sis_graph.for_all_nodes(import_directory, bottom_up=True)
     # run until no new jobs are added. This could be solved more efficient, but this is works...
-    while number_of_jobs != len(sis_graph.jobs()):
-        number_of_jobs = len(sis_graph.jobs())
+    while number_of_jobs != len(sis_graph.jobs(update_graph=False)):
+        number_of_jobs = len(sis_graph.jobs(update_graph=False))
         sis_graph.for_all_nodes(import_directory, bottom_up=True)