Merge remote-tracking branch 'origin/SpiNNcloud_merge' into SpiNNcloud_merge

Author: Christian-B

README.md

@@ -7,7 +7,6 @@ SpiNNUtils
 This provides basic utility functions and classes to other parts of SpiNNaker's
 tooling. Nothing in here knows anything about SpiNNaker functionality.
 
-
 `spinn_utilities.abstract_base`
 -------------------------------
 Provides a simplified (and faster) version of the standard Python Abstract

spinn_utilities/data/utils_data_view.py

@@ -24,6 +24,7 @@
     SimulatorShutdownException, UnexpectedStateChange)
 from spinn_utilities.executable_finder import ExecutableFinder
 from spinn_utilities.log import FormatAdapter
+from spinn_utilities import logger_utils
 from spinn_utilities.make_tools.log_sqllite_database import LogSqlLiteDatabase
 from .data_status import DataStatus
 from .reset_status import ResetStatus
@@ -783,7 +784,8 @@ def get_log_database_path(cls, database_key: str) -> Optional[str]:
             if 'RUNNER_ENVIRONMENT' in os.environ:
                 raise ValueError(f"No logs database found for {database_key=}")
             else:
-                logger.error(f"No logs database found for {database_key=}")
+                logger_utils.error_once(
+                    logger, f"No logs database found for {database_key=}")
                 return None
         return cls.__data._log_database_paths[database_key]
 

spinn_utilities/make_tools/file_converter.py

@@ -118,6 +118,10 @@ def convert(self, src_dir: str, dest_dir: str, file_name: str) -> None:
         if not os.path.exists(dest_dir):
             os.makedirs(dest_dir)
         destination = os.path.join(dest_dir, file_name)
+        if os.path.exists(destination):
+            if os.path.getmtime(destination) > os.path.getmtime(self._src):
+                # Destination is newer than source so skip
+                return
         directory_id = self._log_database.get_directory_id(src_dir, dest_dir)
         self._log_file_id = self._log_database.get_file_id(
             directory_id, file_name)

spinn_utilities/ranged/abstract_list.py

@@ -518,11 +518,16 @@ def __add__(self, other: Union[float, AbstractList[float]]
         :raises TypeError:
         """
         if isinstance(other, AbstractList):
-            d_operation: Callable[[Any, float], float] = lambda x, y: x + y
-            return DualList(
-                left=self, right=other, operation=d_operation)
+
+            def d_operation(x: Any, y: float) -> float:
+                return x + y
+
+            return DualList(left=self, right=other, operation=d_operation)
         if is_number(other):
-            s_operation: Callable[[Any], float] = lambda x: x + other
+
+            def s_operation(x: Any) -> float:
+                return x + other
+
             return SingleList(a_list=self, operation=s_operation)
         raise TypeError("__add__ operation only supported for other "
                         "RangedLists and numerical Values")
@@ -540,11 +545,16 @@ def __sub__(self, other: Union[float, AbstractList[float]]
         :raises TypeError:
         """
         if isinstance(other, AbstractList):
-            d_operation: Callable[[Any, float], float] = lambda x, y: x - y
-            return DualList(
-                left=self, right=other, operation=d_operation)
+
+            def d_operation(x: Any, y: float) -> float:
+                return x - y
+
+            return DualList(left=self, right=other, operation=d_operation)
         if is_number(other):
-            s_operation: Callable[[Any], float] = lambda x: x - other
+
+            def s_operation(x: Any) -> float:
+                return x - other
+
             return SingleList(a_list=self, operation=s_operation)
         raise TypeError("__sub__ operation only supported for other "
                         "RangedLists and numerical Values")
@@ -562,11 +572,16 @@ def __mul__(self, other: Union[float, AbstractList[float]]
         :raises TypeError:
         """
         if isinstance(other, AbstractList):
-            d_operation: Callable[[Any, float], float] = lambda x, y: x * y
-            return DualList(
-                left=self, right=other, operation=d_operation)
+
+            def d_operation(x: Any, y: float) -> float:
+                return x * y
+
+            return DualList(left=self, right=other, operation=d_operation)
         if is_number(other):
-            s_operation: Callable[[Any], float] = lambda x: x * other
+
+            def s_operation(x: Any) -> float:
+                return x * other
+
             return SingleList(a_list=self, operation=s_operation)
         raise TypeError("__mul__ operation only supported for other "
                         "RangedLists and numerical Values")
@@ -585,13 +600,18 @@ def __truediv__(self, other: Union[float, AbstractList[float]]
         :raises TypeError:
         """
         if isinstance(other, AbstractList):
-            d_operation: Callable[[Any, float], float] = lambda x, y: x / y
-            return DualList(
-                left=self, right=other, operation=d_operation)
+
+            def d_operation(x: Any, y: float) -> float:
+                return x / y
+
+            return DualList(left=self, right=other, operation=d_operation)
         if is_number(other):
             if _is_zero(other):
                 raise ZeroDivisionError()
-            s_operation: Callable[[Any], float] = lambda x: x / other
+
+            def s_operation(x: Any) -> float:
+                return x / other
+
             return SingleList(a_list=self, operation=s_operation)
         raise TypeError("__truediv__ operation only supported for other "
                         "RangedLists and numerical Values")
@@ -607,19 +627,25 @@ def __floordiv__(self, other: Union[float, AbstractList[float]]
         :raises TypeError:
         """
         if isinstance(other, AbstractList):
-            d_operation: Callable[[Any, float], int] = lambda x, y: int(x // y)
-            return DualList(
-                left=self, right=other, operation=d_operation)
+
+            def d_operation(x: Any, y: float) -> int:
+                return int(x // y)
+
+            return DualList(left=self, right=other, operation=d_operation)
         if is_number(other):
             if _is_zero(other):
                 raise ZeroDivisionError()
-            s_operation: Callable[[Any], int] = lambda x: int(x / other)
+
+            def s_operation(x: Any) -> int:
+                return int(x / other)
+
             return SingleList(a_list=self, operation=s_operation)
-        raise TypeError("__floordiv__ operation only supported for other "
-                        "RangedLists and numerical Values")
+        raise TypeError(
+            "__floordiv__ operation only supported for other "
+            "RangedLists and numerical Values"
+        )
 
-    def apply_operation(
-            self, operation: Callable[[T], U]) -> AbstractList[U]:
+    def apply_operation(self, operation: Callable[[T], U]) -> AbstractList[U]:
         """
         Applies a function on the list to create a new one.
         The values of the new list are created on the fly so any changes

unittests/make_tools/test_file_convert.py

@@ -14,6 +14,7 @@
 
 import os
 import pytest
+from pathlib import Path
 import sys
 import tempfile
 import unittest
@@ -39,9 +40,10 @@ def test_convert(self) -> None:
         file_name = "weird,file.c"
         src = os.path.join(path, "mock_src")
         dest = os.path.join(path, "modified_src")
-        file_converter.convert(src, dest, file_name)
         src_f = os.path.join(src, file_name)
         dest_f = os.path.join(dest, file_name)
+        Path(src_f).touch()
+        file_converter.convert(src, dest, file_name)
         src_lines = sum(1 for line in open(src_f))
         modified_lines = sum(1 for line in open(dest_f))
         self.assertEqual(src_lines, modified_lines)
@@ -99,6 +101,7 @@ def test_split_fail(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "bad_comma.c").touch()
                 file_converter.convert(src, dest, "bad_comma.c")
                 assert False
             except Exception as ex1:
@@ -116,6 +119,7 @@ def test_format_fail(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "bad_format.c").touch()
                 file_converter.convert(src, dest, "bad_format.c")
                 assert False
             except Exception as ex1:
@@ -131,6 +135,7 @@ def test_unclosed_log(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "unclosed.c").touch()
                 file_converter.convert(src, dest, "unclosed.c")
                 assert False
             except Exception as ex1:
@@ -149,6 +154,7 @@ def test_semi(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "semi.c").touch()
                 file_converter.convert(src, dest, "semi.c")
                 assert False
             except Exception as ex1:
@@ -167,6 +173,7 @@ def test_open(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "open.c").touch()
                 file_converter.convert(src, dest, "open.c")
                 assert False
             except Exception as ex1:
@@ -184,6 +191,7 @@ def test_too_few(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "too_few.c").touch()
                 file_converter.convert(src, dest, "too_few.c")
                 assert False
             except Exception as ex1:
@@ -203,6 +211,7 @@ def test_too_many(self) -> None:
             src = os.path.join(path, "mistakes")
             dest = os.path.join(path, "modified_src")
             try:
+                Path(src, "too_many.c").touch()
                 file_converter.convert(src, dest, "too_many.c")
                 assert False
             except Exception as ex1:

unittests/make_tools/test_replacer.py

@@ -29,6 +29,7 @@
 import math
 import os
 import pytest
+import shutil
 import tempfile
 import unittest
 
@@ -49,6 +50,8 @@ def setUpClass(cls) -> None:
         global logs_database
         src = os.path.join(PATH, "mock_src")
         dest = os.path.join(PATH, "modified_src")
+        if os.path.exists(dest):
+            shutil.rmtree(dest)
         logs_database = convert(src, dest, PATH, "R")
 
     @pytest.mark.xdist_group(name="mock_src")