Merge pull request #18 from PyFixate/config_file_updates

Config file updates

Author: pazzarpj

README.md

@@ -46,7 +46,7 @@ Contributions are welcome. Get in touch or create a new pull request.
 
 * **Ryan Parry-Jones** - *Original Developer* - [pazzarpj](https://github.com/pazzarpj)
 
-See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
+See also the list of [contributors](https://github.com/PyFixate/Fixate/contributors) who participated in this project.
 
 ## License
 

requirements-test

@@ -0,0 +1 @@
+pytest

src/fixate/__main__.py

@@ -36,7 +36,8 @@
 mutex_group.add_argument('-z', '--zip',
                          help="""Path to zip file of test scripts. Mutually exclusive with --path""", )
 parser.add_argument('-l', '--local_log', '--local-log',
-                    help="""Overrides the logging path to the current working directory""",
+                    help="""Deprecated. Use the -c config to set up reporting to a different directory
+                    Overrides the logging path to the current working directory""",
                     action="store_true")
 parser.add_argument('-q', '--qtgui',
                     help="""Argument to select the qt gui mode. This is still in early development""",
@@ -45,7 +46,14 @@
                     help="""Activate Dev Mode for more debug information""",
                     action="store_true")
 parser.add_argument('-c', '--config',
-                    help="""Specify config file""")
+                    help="""Specify the path to a configuration file.
+                    Configuration files are in yaml format. 
+                    Values in this file take precedence over those in a global config file. 
+                    This argument can be used multiple times with later config files taking precedence over earlier ones
+                    """,
+                    action='append',
+                    default=[]
+                    )
 parser.add_argument('-n', '--n_loops', '--n-loops',
                     help="""Loop the test. Use -1 for infinite loops""",
                     action="store")
@@ -115,11 +123,10 @@ class FixateController:
     It may be subclassed for different execution environments
     """
 
-    def __init__(self, sequencer, test_script_path, csv_output_path, args, loop):
+    def __init__(self, sequencer, test_script_path, args, loop):
         register_cmd_line()
         # self.register()
-        self.worker = FixateWorker(sequencer=sequencer, test_script_path=test_script_path,
-                                   csv_output_path=csv_output_path, args=args, loop=loop)
+        self.worker = FixateWorker(sequencer=sequencer, test_script_path=test_script_path, args=args, loop=loop)
 
     def fixate_exec(self):
         exit_code = self.worker.ui_run()
@@ -129,26 +136,26 @@ def fixate_exec(self):
 
 
 class FixateSupervisor:
-    def __init__(self, test_script_path, csv_output_path, args):
+    def __init__(self, test_script_path, args):
 
         # General setup
         self.test_script_path = test_script_path
-        self.csv_output_path = csv_output_path
         self.args = args
         self.loop = asyncio.get_event_loop()
         self.sequencer = RESOURCES["SEQUENCER"]
 
         # Environment specific setup
+        # TODO remove this to plugin architecture
         if self.args.qtgui:  # Run with the QT GUI
             self.loop = asyncio.new_event_loop()
 
             class QTController(FixateController):
-                def __init__(self, sequencer, test_script_path, csv_output_path, args, loop):
+                def __init__(self, sequencer, test_script_path, args, loop):
                     from PyQt5 import QtWidgets, QtCore
                     import fixate.ui_gui_qt as gui
 
-                    self.worker = FixateWorker(test_script_path=test_script_path, csv_output_path=csv_output_path,
-                                               args=args, loop=loop, sequencer=sequencer)
+                    self.worker = FixateWorker(test_script_path=test_script_path, args=args, loop=loop,
+                                               sequencer=sequencer)
 
                     QtWidgets.QApplication.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling)
                     self.fixateApp = QtWidgets.QApplication(sys.argv)
@@ -166,21 +173,20 @@ def fixate_exec(self):
                     self.fixateApp.closeAllWindows()
                     return exit_code
 
-            self.controller = QTController(sequencer=self.sequencer, test_script_path=test_script_path,
-                                           csv_output_path=csv_output_path, args=args, loop=self.loop)
+            self.controller = QTController(sequencer=self.sequencer, test_script_path=test_script_path, args=args,
+                                           loop=self.loop)
         else:  # Command line execution
             self.controller = FixateController(sequencer=self.sequencer, test_script_path=test_script_path,
-                                               csv_output_path=csv_output_path, args=args, loop=self.loop)
+                                               args=args, loop=self.loop)
 
     def run_fixate(self):
         return self.controller.fixate_exec()
 
 
 class FixateWorker:
-    def __init__(self, sequencer, test_script_path, csv_output_path, args, loop):
+    def __init__(self, sequencer, test_script_path, args, loop):
         self.sequencer = sequencer
         self.test_script_path = test_script_path
-        self.csv_output_path = csv_output_path
         self.args = args
         self.loop = loop
         self.start = False
@@ -214,23 +220,6 @@ def stop(self):
 
         return 11
 
-    def read_config(self, path=None):
-        """
-        Loads yaml config file, and reads in parameters
-        :param path:
-        :return:
-        """
-
-        if path is None:
-            return {}
-
-        try:
-            with open(os.path.join(path), 'rb') as f:
-                yaml = ruamel.yaml.YAML(typ="safe", pure=True)
-                return yaml.load(f)
-        except (IOError, OSError) as e:
-            raise e("Error opening config file")
-
     def ui_run(self):
 
         asyncio.set_event_loop(self.loop)
@@ -266,13 +255,13 @@ def ui_run(self):
             test_suite = load_test_suite(self.args.path, self.args.zip, self.args.zip_selector)
             test_data = retrieve_test_data(test_suite, self.args.index)
             self.sequencer.load(test_data)
+
             if self.args.local_log:
-                self.csv_output_path = os.path.join(os.path.dirname(self.test_script_path))
-            if self.csv_output_path is None:
-                self.config = self.read_config(self.args.config)
-                self.csv_output_path = os.path.join(self.config.get('BASE_CSV_PATH', ""), self.sequencer.context_data.get("part_number", ""),
-                                                    self.sequencer.context_data.get("module", ""))
-            register_csv(self.csv_output_path)
+                try:
+                    fixate.config.plg_csv["tpl_csv_path"] = ["{tpl_time_stamp}-{index}.csv"]
+                except (AttributeError, KeyError):
+                    pass
+            register_csv()
             self.sequencer.status = 'Running'
 
             def init_tasks():
@@ -332,12 +321,26 @@ def retrieve_test_data(test_suite, index):
     return sequence
 
 
-def run_main_program(test_script_path=None, csv_output_path=None):
+def run_main_program(test_script_path=None):
     args, unknown = parser.parse_known_args()
-    supervisor = FixateSupervisor(test_script_path, csv_output_path, args)
+    load_config(args.config)
+    supervisor = FixateSupervisor(test_script_path, args)
     exit(supervisor.run_fixate())
 
 
+def load_config(config: list = None):
+    # Load python environment fixate config
+    env_config = os.path.join(sys.prefix, "fixate.yml")
+    if os.path.exists(env_config):
+        fixate.config.load_yaml_config(env_config)
+    # TODO Load script config
+
+    # Load a list of config files
+    if config is not None:
+        for conf in config:
+            fixate.config.load_yaml_config(conf)
+
+
 # Setup configuration
 if __name__ == "__main__":
     run_main_program()

src/fixate/config/__init__.py

@@ -5,7 +5,10 @@
 Must ensure driver imports are infallible to prevent program crash on start
 """
 import importlib
+from fixate.config.helper import load_dict_config, load_json_config, load_yaml_config, get_plugin_data, get_plugins, \
+    get_config_dict, render_template
 
+# TODO review all these values to determine if they should exist in this module
 DRIVER_LIST = {"DMM": {"dmm.fluke_8846a.Fluke8846A"},
                "FUNC_GEN": {"funcgen.rigol_dg1022.RigolDG1022", "funcgen.keysight_33500b.Keysight33500B"},
                "DAQ": {"daq.daqmx.DaqMx"},
@@ -23,6 +26,26 @@
 ASYNC_TASKS = []
 DEBUG = False
 importer = None
+# Begin default "plugins"
+# Use plg_ prefix with dictionary of values to indicate to fixate to install this at startup
+# Default settings for csv reporting. Can be configured via yaml either removing or overriding with plg_csv
+tpl_csv_path = ["{start_date_time}-{index}.csv"]
+tpl_time_stamp = "{0:%Y}{0:%m}{0:%d}-{0:%H}{0:%M}{0:%S}"
+
+plg_csv = {
+    "import_name": "fixate.reporting.csv",
+    "REPORT_FORMAT_VERSION": 3,
+    "tpl_first_line": [
+        "0",
+        'Sequence',
+        "started={start_date_time}",
+        "fixate-version={fixate_version}",
+        "test-script-name={test_script_name}",
+        "report-format={REPORT_FORMAT_VERSION}",
+        "index_string={index}"]
+}
+index = None
+# TODO Remove this and do this as part of the plugin initialisation routine
 # Import the drivers from the DRIVER_LIST
 for key, value in DRIVER_LIST.items():
     for drv in value:
@@ -35,4 +58,4 @@
             DRIVERS[key].append((cls, getattr(importlib.import_module('fixate.drivers.' + imp_path), cls)))
         except Exception as e:
             # print(repr(e))
-            pass
+            pass

src/fixate/config/fixate_config.yml

@@ -1,8 +1,12 @@
 import json
+import ruamel.yaml
+import pathlib
 import fixate.config
+import os
+from string import Formatter
 
 
-def load_dict_config(in_dict, config_name=None):
+def load_dict_config(in_dict: dict, config_name: str = '') -> None:
     """
     :param in_dict:
      dictionary type storing configuration parameters
@@ -13,12 +17,12 @@ def load_dict_config(in_dict, config_name=None):
      config_name = None
      >>> import fixate.config
      >>> load_dict_config(my_config_dict)
-     >>> print(config.HI)
+     >>> print(fixate.config.HI)
      "WORLD"
      config_name = "My_Dict"
      >>> import fixate.config
      >>> load_dict_config(my_config_dict, "My_Dict")
-     >>> print(config.My_Dict)
+     >>> print(fixate.config.My_Dict)
      {"HI": "WORLD"}
     """
     if config_name:
@@ -27,7 +31,7 @@ def load_dict_config(in_dict, config_name=None):
         fixate.config.__dict__.update(in_dict)
 
 
-def load_json_config(in_file, config_name=None):
+def load_json_config(in_file, config_name=None) -> None:
     """
     :param in_file:
      valid open file like such as
@@ -45,13 +49,13 @@ def load_json_config(in_file, config_name=None):
      >>> import fixate.config
      >>> with open("my_json_file") as f:
      >>>    load_json_config("my_json_file")
-     >>> print(config.HI)
+     >>> print(fixate.config.HI)
      "WORLD"
      config_name = "My_Json"
      >>> import fixate.config
      >>> with open("my_json_file") as f:
      >>>    load_json_config("my_json_file", "My_Json")
-     >>> print(config.My_Json)
+     >>> print(fixate.config.My_Json)
      {"HI": "WORLD"}
     """
     if config_name:
@@ -60,3 +64,62 @@ def load_json_config(in_file, config_name=None):
         fixate.config.__dict__.update(json.load(in_file))
 
 
+def load_yaml_config(yaml_in: str) -> None:
+    """
+    :param in_file:
+     string representing a valid file path to the yaml file
+     >>> import fixate.config
+     >>> with open("my_yaml_file.yml") as f:
+     >>>    load_yaml_config("my_yaml_file.yml")
+     >>> print(fixate.config.HI)
+     "WORLD"
+    """
+    if not os.path.exists(yaml_in):
+        raise FileNotFoundError("Config file {} not found".format(yaml_in))
+    yaml = ruamel.yaml.YAML(typ="safe", pure=True)
+    yaml.default_flow_style = False
+    yaml_path = pathlib.Path(yaml_in)
+    fixate.config.__dict__.update(yaml.load(yaml_path))
+
+
+def get_plugins() -> dict:
+    return {k: v for k, v in fixate.config.__dict__.items() if k.startswith('plg_')}
+
+
+def get_plugin_data(plugin: str) -> dict:
+    return get_plugins().get(plugin, {})
+
+
+def get_config_dict() -> dict:
+    return fixate.config.__dict__
+
+
+class _UnseenFormatter(Formatter):
+    """
+    Renders string formats with invalid keys rendered as the key name
+    """
+    def get_value(self, key, args, kwargs):
+        if isinstance(key, str):
+            try:
+                return kwargs[key]
+            except KeyError:
+                return "None"
+        else:
+            return Formatter.get_value(key, args, kwargs)
+
+
+def render_template(_template: [str, list, tuple], *args, **kwargs):
+    """
+    :param template: Template string or list
+    :param kwargs:
+    :return:
+    """
+    if isinstance(_template, str):
+        return _render.format(*args, **kwargs)
+    renders = []
+    for itm in _template:
+        renders.append(_render.format(itm, *args, **kwargs))
+    return renders
+
+
+_render = _UnseenFormatter()