Merge pull request #15 from MiraGeoscience/release/0.1.4

Release/0.1.4

Author: andrewg-mira

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ ci:
 
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.8.0
+    rev: 22.12.0
     hooks:
     -   id: black
         types: [text]
@@ -22,7 +22,7 @@ repos:
         types: [text]
         types_or: [python, pyi]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
     -   id: isort
         additional_dependencies: [toml] # to read config from pyproject.toml
@@ -35,20 +35,20 @@ repos:
         types: [text]
         types_or: [python, pyi]
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         types: [text]
         types_or: [python, pyi]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.37.3
+    rev: v3.3.1
     hooks:
     -   id: pyupgrade
         args: [--py37-plus]
         types: [text]
         types_or: [python, pyi]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.971
+    rev: v0.991
     hooks:
     -   id: mypy
         additional_dependencies: [types-toml]
@@ -72,14 +72,14 @@ repos:
         types_or: [python, pyi]
         exclude: ^(devtools/|docs/|setup.py)
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.1
+    rev: v2.2.2
     hooks:
     -   id: codespell
         exclude: (\.ipynb$|^\.github/workflows/issue_to_jira.yml$)
         types: [text]
         types_or: [python, pyi]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     -   id: trailing-whitespace
         exclude: \.mdj$
@@ -95,12 +95,12 @@ repos:
     -   id: mixed-line-ending
     -   id: name-tests-test
 -   repo: https://github.com/rstcheck/rstcheck
-    rev: v6.1.0
+    rev: v6.1.1
     hooks:
     -   id: rstcheck
         additional_dependencies: [sphinx]
 -   repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
     -   id: rst-backticks
         exclude: ^THIRD_PARTY_SOFTWARE.rst$

devtools/check-copyright.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-#  Copyright (c) 2022 Mira Geoscience Ltd.
+#  Copyright (c) 2023 Mira Geoscience Ltd.
 #
 #  This file is part of param-sweeps.
 #

param_sweeps/__init__.py

@@ -1,8 +1,9 @@
-#  Copyright (c) 2022 Mira Geoscience Ltd.
+#  Copyright (c) 2023 Mira Geoscience Ltd.
 #
 #  This file is part of param-sweeps.
 #
 #  param-sweeps is distributed under the terms and conditions of the MIT License
 #  (see LICENSE file at the root of this source code package).
 
-__version__ = "0.1.3"
+
+__version__ = "0.1.4"

param_sweeps/constants.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022 Mira Geoscience Ltd.
+#  Copyright (c) 2023 Mira Geoscience Ltd.
 #
 #  This file is part of param-sweeps.
 #

param_sweeps/driver.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022 Mira Geoscience Ltd.
+#  Copyright (c) 2023 Mira Geoscience Ltd.
 #
 #  This file is part of param-sweeps.
 #
@@ -8,10 +8,11 @@
 from __future__ import annotations
 
 import argparse
+import importlib
+import inspect
 import itertools
 import json
 import os
-import subprocess
 import uuid
 from dataclasses import dataclass
 from inspect import signature
@@ -92,6 +93,10 @@ class SweepDriver:
 
     def __init__(self, params):
         self.params: SweepParams = params
+        self.workspace = params.geoh5
+        self.working_directory = os.path.dirname(self.workspace.h5file)
+        lookup = self.get_lookup()
+        self.write_files(lookup)
 
     @staticmethod
     def uuid_from_params(params: tuple) -> str:
@@ -104,75 +109,97 @@ def uuid_from_params(params: tuple) -> str:
         """
         return str(uuid.uuid5(uuid.NAMESPACE_DNS, str(hash(params))))
 
-    def run(self, files_only=False):
-        """Execute a sweep."""
+    def get_lookup(self):
+        """Generate lookup table for sweep trials."""
+
+        lookup = {}
+        sets = self.params.parameter_sets()
+        iterations = list(itertools.product(*sets.values()))
+        for iteration in iterations:
+            param_uuid = SweepDriver.uuid_from_params(iteration)
+            lookup[param_uuid] = dict(zip(sets.keys(), iteration))
+            lookup[param_uuid]["status"] = "pending"
+
+        lookup = self.update_lookup(lookup, gather_first=True)
+        return lookup
+
+    def update_lookup(self, lookup: dict, gather_first: bool = False):
+        """Updates lookup with new entries. Ensures any previous runs are incorporated."""
+        lookup_path = os.path.join(self.working_directory, "lookup.json")
+        if os.path.exists(lookup_path) and gather_first:  # In case restarting
+            with open(lookup_path, encoding="utf8") as file:
+                lookup.update(json.load(file))
+
+        with open(lookup_path, "w", encoding="utf8") as file:
+            json.dump(lookup, file, indent=4)
+
+        return lookup
+
+    def write_files(self, lookup):
+        """Write ui.geoh5 and ui.json files for sweep trials."""
 
         ifile = InputFile.read_ui_json(self.params.worker_uijson)
         with ifile.data["geoh5"].open(mode="r") as workspace:
-            sets = self.params.parameter_sets()
-            iterations = list(itertools.product(*sets.values()))
-            print(
-                f"Running parameter sweep for {len(iterations)} "
-                f"trials of the {ifile.data['title']} driver."
-            )
 
-            param_lookup = {}
-            for count, iteration in enumerate(iterations):
-                param_uuid = SweepDriver.uuid_from_params(iteration)
-                filepath = os.path.join(
-                    os.path.dirname(workspace.h5file), f"{param_uuid}.ui.geoh5"
-                )
-                param_lookup[param_uuid] = dict(zip(sets.keys(), iteration))
+            for name, trial in lookup.items():
 
-                if os.path.exists(filepath):
-                    print(
-                        f"{count}: Skipping trial: {param_uuid}. "
-                        f"Already computed and saved to file."
-                    )
+                if trial["status"] != "pending":
                     continue
 
-                print(
-                    f"{count}: Running trial: {param_uuid}. "
-                    f"Use lookup.json to map uuid to parameter set."
+                filepath = os.path.join(
+                    os.path.dirname(workspace.h5file), f"{name}.ui.geoh5"
                 )
                 with Workspace(filepath) as iter_workspace:
                     ifile.data.update(
-                        dict(param_lookup[param_uuid], **{"geoh5": iter_workspace})
+                        dict(
+                            {key: val for key, val in trial.items() if key != "status"},
+                            **{"geoh5": iter_workspace},
+                        )
                     )
                     objects = [v for v in ifile.data.values() if hasattr(v, "uid")]
                     for obj in objects:
                         if not isinstance(obj, Data):
                             obj.copy(parent=iter_workspace, copy_children=True)
 
-                update_lookup(param_lookup, workspace)
-
-                ifile.name = f"{param_uuid}.ui.json"
+                ifile.name = f"{name}.ui.json"
                 ifile.path = os.path.dirname(workspace.h5file)
                 ifile.write_ui_json()
+                lookup[name]["status"] = "written"
 
-                if not files_only:
-                    call_worker_subprocess(ifile)
+        _ = self.update_lookup(lookup)
 
+    def run(self):
+        """Execute a sweep."""
 
-def call_worker_subprocess(ifile: InputFile):
-    """Runs the worker for the sweep parameters contained in 'ifile'."""
-    subprocess.run(
-        ["python", "-m", ifile.data["run_command"], ifile.path_name],
-        check=True,
-    )
+        lookup_path = os.path.join(self.working_directory, "lookup.json")
+        with open(lookup_path, encoding="utf8") as file:
+            lookup = json.load(file)
 
+        for name, trial in lookup.items():
+            ifile = InputFile.read_ui_json(
+                os.path.join(self.working_directory, f"{name}.ui.json")
+            )
+            status = trial.pop("status")
+            if status != "complete":
+                lookup[name]["status"] = "processing"
+                self.update_lookup(lookup)
+                call_worker(ifile)
+                lookup[name]["status"] = "complete"
+                self.update_lookup(lookup)
 
-def update_lookup(lookup: dict, workspace: Workspace):
-    """Updates lookup with new entries. Ensures any previous runs are incorporated."""
-    lookup_path = os.path.join(os.path.dirname(workspace.h5file), "lookup.json")
-    if os.path.exists(lookup_path):  # In case restarting
-        with open(lookup_path, encoding="utf8") as file:
-            lookup.update(json.load(file))
 
-    with open(lookup_path, "w", encoding="utf8") as file:
-        json.dump(lookup, file, indent=4)
+def call_worker(ifile: InputFile):
+    """Runs the worker for the sweep parameters contained in 'ifile'."""
 
-    return lookup
+    run_cmd = ifile.data["run_command"]
+    module = importlib.import_module(run_cmd)
+    filt = (
+        lambda member: inspect.isclass(member)
+        and member.__module__ == run_cmd
+        and hasattr(member, "run")
+    )
+    driver = inspect.getmembers(module, filt)[0][1]
+    driver.start(ifile.path_name)
 
 
 def file_validation(filepath):
@@ -188,14 +215,14 @@ def file_validation(filepath):
         raise OSError(f"File argument {filepath} must have extension 'ui.json'.")
 
 
-def main(file_path, files_only=False):
+def main(file_path):
     """Run the program."""
 
     file_validation(file_path)
     print("Reading parameters and workspace...")
     input_file = InputFile.read_ui_json(file_path)
     sweep_params = SweepParams.from_input_file(input_file)
-    SweepDriver(sweep_params).run(files_only)
+    SweepDriver(sweep_params).run()
 
 
 if __name__ == "__main__":
@@ -206,4 +233,4 @@ def main(file_path, files_only=False):
     parser.add_argument("file", help="File with ui.json format.")
 
     args = parser.parse_args()
-    main(args.file)
+    main(os.path.abspath(args.file))

param_sweeps/generate.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022 Mira Geoscience Ltd.
+#  Copyright (c) 2023 Mira Geoscience Ltd.
 #
 #  This file is part of param-sweeps.
 #
@@ -9,14 +9,19 @@
 
 import argparse
 import os
+import re
 from copy import deepcopy
 
 from geoh5py.ui_json import InputFile
 
 from param_sweeps.constants import default_ui_json
 
 
-def generate(worker: str, parameters: list[str] = None, update_values: dict = None):
+def generate(
+    worker: str,
+    parameters: list[str] | None = None,
+    update_values: dict | None = None,
+):
     """
     Generate an *_sweep.ui.json file to sweep parameters of the driver associated with 'file'.
 
@@ -47,7 +52,8 @@ def generate(worker: str, parameters: list[str] = None, update_values: dict = No
     dirname = os.path.dirname(file)
     filename = os.path.basename(file)
     filename = filename.rstrip("ui.json")
-    filename = filename.rstrip("_sweep")
+    filename = re.sub(r"\._sweep$", "", filename)
+    # filename = filename.rstrip("_sweep")
     filename = f"{filename}_sweep.ui.json"
 
     print(f"Writing sweep file to: {os.path.join(dirname, filename)}")

param_sweeps/sample_driver.py

@@ -0,0 +1,39 @@
+#  Copyright (c) 2023 Mira Geoscience Ltd.
+#
+#  This file is part of param-sweeps.
+#
+#  param-sweeps is distributed under the terms and conditions of the MIT License
+#  (see LICENSE file at the root of this source code package).
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from geoh5py.ui_json import InputFile
+
+
+@dataclass
+class SampleParams:
+
+    data_object: str | None = None
+    data: str | None = None
+    param: int = 1
+
+    def __init__(self, input_file):
+        for key, value in input_file.data.items():
+            setattr(self, key, value)
+
+
+class SampleDriver:
+    def __init__(self, params):
+        self.params = params
+
+    def run(self):
+        print(self.params.param)
+
+    @classmethod
+    def start(cls, filepath, driver_class=None):
+        _ = driver_class
+        ifile = InputFile.read_ui_json(filepath)
+        params = SampleParams(ifile)
+        SampleDriver(params).run()