jinja2: remove legacy support for zero-prefixed integers

* Reverts #140, #146
* Closes #374
* Integers with leading zeros were supported by Jinja2 v2 when running
  under Python 2, but not when using Jinja2 v3 under Python 3.
* In order to permit Cylc 7 workflows to be inter-operable with Cylc 8,
  a hack was put in place to extend support for zero-prefixed integers into
  Cylc 8 (Jinja2 v3, Python 3).
* Cylc 7 compatibility mode has been removed in the 8.7 meta-release, so
  the hack is no longer required.

Author: oliver-sanders

changes.d/429.break.md

@@ -0,0 +1 @@
+Legacy support for zero-prefixed integers (e.g, `01`, `02`, `03` as opposed to `1`, `2`, `3`) has been removed. This support was dropped in Jinja2 v3.0, but support was extended in cylc-rose past this date in order to facilitate Cylc 7/8 interoperability.

cylc/rose/jinja2_parser.py

@@ -16,12 +16,8 @@
 """Utility for parsing Jinja2 expressions."""
 
 from ast import literal_eval as python_literal_eval
-from contextlib import contextmanager
-from copy import deepcopy
 import re
 
-from cylc.flow import LOG
-import jinja2.lexer
 from jinja2.nativetypes import NativeEnvironment  # type: ignore
 from jinja2.nodes import (  # type: ignore
     Literal,
@@ -33,157 +29,6 @@
 )
 
 
-def _strip_leading_zeros(string):
-    """Strip leading zeros from a string.
-
-    Examples:
-        >>> _strip_leading_zeros('1')
-        '1'
-        >>> _strip_leading_zeros('01')
-        '1'
-        >>> _strip_leading_zeros('001')
-        '1'
-        >>> _strip_leading_zeros('0001')
-        '1'
-        >>> _strip_leading_zeros('0')
-        '0'
-        >>> _strip_leading_zeros('000')
-        '0'
-
-    """
-    ret = string.lstrip('0')
-    return ret or '0'
-
-
-def _lexer_wrap(fcn):
-    """Helper for patch_jinja2_leading_zeros.
-
-    Patches the jinja2.lexer.Lexer.wrap method.
-    """
-    instances = set()  # record of uses of deprecated syntax
-
-    def _stream(stream):
-        """Patch the token stream to strip the leading zero where necessary."""
-        for lineno, token, value_str in stream:
-            if (
-                token == jinja2.lexer.TOKEN_INTEGER
-                and len(value_str) > 1
-                and value_str[0] == '0'
-            ):
-                instances.add(value_str)
-            yield (lineno, token, _strip_leading_zeros(value_str))
-
-    def _inner(
-        self,
-        stream,  # : t.Iterable[t.Tuple[int, str, str]],
-        name,  # : t.Optional[str] = None,
-        filename,  # : t.Optional[str] = None,
-    ):  # -> t.Iterator[Token]:
-        return fcn(self, _stream(stream), name, filename)
-
-    _inner.__wrapped__ = fcn  # save the un-patched function
-    _inner._instances = instances  # save the set of uses of deprecated syntax
-
-    return _inner
-
-
-@contextmanager
-def patch_jinja2_leading_zeros():
-    """Back support integers with leading zeros in Jinja2 v3.
-
-    Jinja2 v3 dropped support for integers with leading zeros, these are
-    heavily used throughout Rose configurations. Since there was no deprecation
-    warning in Jinja2 v2 we have implemented this patch to extend support for
-    a short period to help our users to transition.
-
-    This patch will issue a warning if usage of the deprecated syntax is
-    detected during the course of its usage.
-
-    Warning:
-        This is a *global* patch applied to the Jinja2 library whilst the
-        context manager is open. Do not use this with parallel/async code
-        as the patch could apply to code outside of the context manager.
-
-    Examples:
-        >>> env = NativeEnvironment()
-
-        The integer "1" is ok:
-        >>> env.parse('{{ 1 }}')
-        Template(body=[Output(nodes=[Const(value=1)])])
-
-        However "01" is no longer supported:
-        >>> env.parse('{{ 01 }}')
-        Traceback (most recent call last):
-        jinja2.exceptions.TemplateSyntaxError: expected token ...
-
-        The patch returns support (the leading-zero gets stripped):
-        >>> with patch_jinja2_leading_zeros():
-        ...     env.parse('{{ 01 }}')
-        Template(body=[Output(nodes=[Const(value=1)])])
-
-        The patch can handle any number of arbitrary leading zeros:
-        >>> with patch_jinja2_leading_zeros():
-        ...     env.parse('{{ 0000000001 }}')
-        Template(body=[Output(nodes=[Const(value=1)])])
-
-        Once the "with" closes we go back to vanilla Jinja2 behaviour:
-        >>> env.parse('{{ 01 }}')
-        Traceback (most recent call last):
-        jinja2.exceptions.TemplateSyntaxError: expected token ...
-
-    """
-    # clear any previously cashed lexer instances
-    jinja2.lexer._lexer_cache.clear()
-
-    # apply the code patch (new lexer instances will pick up these changes)
-    _integer_re = deepcopy(jinja2.lexer.integer_re)
-    jinja2.lexer.integer_re = re.compile(
-        rf'''
-            # Jinja2 no longer recognises zero-padded integers as integers
-            # so we must patch its regex to allow them to be detected.
-            (
-                [0-9](_?\d)* # decimal (which supports zero-padded integers)
-                |
-                {jinja2.lexer.integer_re.pattern}
-            )
-        ''',
-        re.IGNORECASE | re.VERBOSE,
-    )
-    jinja2.lexer.Lexer.wrap = _lexer_wrap(jinja2.lexer.Lexer.wrap)
-
-    # execute the body of the "with" statement
-    try:
-        yield
-    finally:
-        # report any usage of deprecated syntax
-        if jinja2.lexer.Lexer.wrap._instances:
-            num_examples = 5
-            LOG.warning(
-                'Support for integers with leading zeros (including'
-                ' lists of integers) was dropped in Jinja2 v3.'
-                ' Rose will extend support until a future version.'
-                '\nPlease amend your Rose configuration files,'
-                ' which currently contain:'
-                '\n * '
-                + (
-                    '\n * '.join(
-                        f'{before} => {_strip_leading_zeros(before)}'
-                        for before in list(
-                            jinja2.lexer.Lexer.wrap._instances
-                        )[:num_examples]
-                    )
-                )
-
-            )
-
-        # revert the code patch
-        jinja2.lexer.integer_re = _integer_re
-        jinja2.lexer.Lexer.wrap = jinja2.lexer.Lexer.wrap.__wrapped__
-
-        # clear any patched lexers to return Jinja2 to normal operation
-        jinja2.lexer._lexer_cache.clear()
-
-
 class Parser(NativeEnvironment):
 
     _LITERAL_NODES = (
@@ -248,11 +93,6 @@ def literal_eval(self, value):
             >>> parser.literal_eval('1,\n2,\n3')
             (1, 2, 3)
 
-            # back-supported jinja2 variants
-            >>> with patch_jinja2_leading_zeros():
-            ...     parser.literal_eval('042')
-            42
-
             # invalid examples
             >>> parser.literal_eval('1 + 1')
             Traceback (most recent call last):

cylc/rose/utilities.py

@@ -52,7 +52,7 @@
 from metomi.rose.config_tree import ConfigTree
 from metomi.rose.env import UnboundEnvironmentVariableError, env_var_process
 
-from cylc.rose.jinja2_parser import Parser, patch_jinja2_leading_zeros
+from cylc.rose.jinja2_parser import Parser
 
 if TYPE_CHECKING:
     from cylc.flow.option_parsers import Values
@@ -184,26 +184,22 @@ def process_config(
 
     # Add the entire plugin_result to ROSE_SUITE_VARIABLES to allow for
     # programatic access.
-    with patch_jinja2_leading_zeros():
-        # BACK COMPAT: patch_jinja2_leading_zeros
-        # back support zero-padded integers for a limited time to help
-        # users migrate before upgrading cylc-flow to Jinja2>=3.1
-        parser = Parser()
-        for key, value in plugin_result['template_variables'].items():
-            # The special variables are already Python variables.
-            if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
-                try:
-                    plugin_result['template_variables'][key] = (
-                        parser.literal_eval(value)
-                    )
-                except Exception:
-                    raise ConfigProcessError(
-                        [templating, key],
-                        value,
-                        f'Invalid template variable: {value}'
-                        '\nMust be a valid Python or Jinja2 literal'
-                        ' (note strings "must be quoted").',
-                    ) from None
+    parser = Parser()
+    for key, value in plugin_result['template_variables'].items():
+        # The special variables are already Python variables.
+        if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
+            try:
+                plugin_result['template_variables'][key] = (
+                    parser.literal_eval(value)
+                )
+            except Exception:
+                raise ConfigProcessError(
+                    [templating, key],
+                    value,
+                    f'Invalid template variable: {value}'
+                    '\nMust be a valid Python or Jinja2 literal'
+                    ' (note strings "must be quoted").',
+                ) from None
 
     # Add ROSE_SUITE_VARIABLES to plugin_result of templating engines in use.
     plugin_result['template_variables']['ROSE_SUITE_VARIABLES'] = (