data_store: optimise rtconfig serialisation

Author: oliver-sanders

cylc/flow/data_store_mgr.py

@@ -107,6 +107,7 @@
 from cylc.flow.network import API
 from cylc.flow.parsec.util import (
     listjoin,
+    fast_listjoin as listjoin,
     pdeepcopy,
     poverride,
 )

cylc/flow/parsec/util.py

@@ -113,13 +113,31 @@ def listjoin(lst, none_str=''):
     for item in lst:
         if item is None:
             items.append(none_str)
-        elif any(char in str(item) for char in ',#"\''):
-            items.append(repr(item))  # will be quoted
         else:
-            items.append(str(item))
+            string = str(item)
+            if any(char in string for char in ',#"\''):
+                items.append(repr(item))  # will be quoted
+            else:
+                items.append(string)
     return ', '.join(items)
 
 
+def fast_listjoin(lst, none_str=''):
+    """Faster variant of listjoin suitable in select cases.
+
+    Compared to listjoin, this does *not*:
+    * Rationalise and pretty-format integer lists.
+    * Intelligently quote arguments which need quoting.
+
+    Suitable for use in situations where you know the data type of the field
+    being joined and the above is of no concern.
+    """
+    return ', '.join((
+        none_str if item is None else str(item)
+        for item in lst or [None]
+    ))
+
+
 def printcfg(cfg, level=0, indent=0, prefix='', none_str='',
              handle=None):
     """Pretty-print a parsec config item or section (nested dict).

cylc/flow/parsec/validate.py

@@ -22,6 +22,7 @@
 Also provides default values from the spec as a nested dict.
 """
 
+from functools import lru_cache
 import re
 import shlex
 from collections import deque
@@ -655,12 +656,18 @@ def parsec_validate(cfg_root, spec_root):
     return ParsecValidator().validate(cfg_root, spec_root)
 
 
-class DurationFloat(float):
-    """Duration in floating point seconds, but stringify as ISO8601 format."""
-
-    def __str__(self):
+class _DurationFloat(float):
+    def _str(self):
         return str(Duration(seconds=self, standardize=True))
 
+    # NOTE: This object is immutable so we cache the value of __str__.
+    __str__ = lru_cache(1)(_str)
+
+
+# NOTE: Prevent duplicate DurationFloat objects being created for the same
+# duration value. This allows __str__ caching to be effective.
+DurationFloat = lru_cache(None)(_DurationFloat.__call__)
+
 
 class Range(tuple):
 

tests/unit/parsec/test_util.py

@@ -22,6 +22,7 @@
 from cylc.flow.parsec.util import (
     SECTION_EXPAND_PATTERN,
     expand_many_section,
+    fast_listjoin,
     itemstr,
     listjoin,
     m_override,
@@ -31,15 +32,44 @@
     replicate,
     un_many,
 )
-
-
-def test_listjoin():
-    assert listjoin(None) == ''
-    assert listjoin(None, 'test') == 'test'
-    assert listjoin([], 'test') == 'test'
-    assert listjoin([None], 'test') == 'test'
-    assert listjoin(['test', 'test']) == 'test, test'
-    assert listjoin(['test,', 'test']) == '\'test,\', test'
+from cylc.flow.parsec.validate import DurationFloat
+
+
+@pytest.mark.parametrize('method', (listjoin, fast_listjoin))
+def test_listjoin(method):
+    assert method(None) == ''
+    assert method(None, 'test') == 'test'
+    assert method([]) == ''
+    assert method([], 'test') == 'test'
+    assert method([None], 'test') == 'test'
+    assert method(['test', 'test']) == 'test, test'
+    if method == listjoin:
+        # intelligent quoting
+        assert method(['a#', 'b,', "c'"]) == "'a#', 'b,', \"c'\""
+        assert method(['test,', 'test']) == '\'test,\', test'
+
+        # integer list rationalisation
+        assert method([10, 11, 12]) == '10..12'
+        assert method([10, 10, 10]) == '3*10'
+        assert method([10, 10, 10, 42]) == '3*10, 42'
+
+
+def test_duration_float():
+    """It should prevent the creation of duplicate instances."""
+    # these calls should only create two discrete instances of DurationFloat
+    a = DurationFloat(1.0)
+    b = DurationFloat(1.0)
+    c = DurationFloat(42.0)
+
+    # a and b should be no only equal but also the same instance
+    assert a == b
+    assert id(a) == id(b)
+    assert str(a) == 'PT1S'
+
+    # a and c should be unequal and different instances
+    assert a != c
+    assert id(a) != id(c)
+    assert str(c) == 'PT42S'
 
 
 # --- printcfg