SedRegex: Add support for Windows

The only multiprocessing method available on Windows is 'spawn',
which requires functions to be picklable and they can't access
Limnoria's global state.

This means that they can't access the message history, so we need
to pass messages as a list (though we prefilter it to avoid wasting
resources).

Additionally, this adds support for calling loadPluginModule in
the child process before unpickling so that we can unpickle functions
defined in plugins.

This also paves the way toward supporting the 'forkserver' method
on POSIX systems, which has the same restrictions as the 'spawn' method
but is safer than 'fork' as we use threads in the main process.

Author: progval

plugins/SedRegex/plugin.py

@@ -1,7 +1,7 @@
 ###
 # Copyright (c) 2015, Michael Daniel Telatynski <postmaster@webdevguru.co.uk>
-# Copyright (c) 2015-2020, James Lu <james@overdrivenetworks.com>
-# Copyright (c) 2020-2021, Valentin Lorentz
+# Copyright (c) 2015-2025, James Lu <james@overdrivenetworks.com>
+# Copyright (c) 2020-2026, Valentin Lorentz
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -30,6 +30,7 @@
 
 ###
 
+import multiprocessing
 from supybot.commands import *
 from supybot.commands import ProcessTimeoutError
 import supybot.plugins as plugins
@@ -38,6 +39,7 @@
 import supybot.ircutils as ircutils
 import supybot.ircdb as ircdb
 import supybot.utils as utils
+import supybot.world as world
 
 import re
 
@@ -112,6 +114,14 @@ def apply_substitution(pattern, replacement, m, count):
 
     return axe_spaces(subst)
 
+def apply_substitution_to_first_matching_message(pattern, replacement,
+                                                 messages, count):
+    m = get_first_matching_message(pattern, messages)
+    if m:
+        subst = apply_substitution(pattern, replacement, m, count)
+        return (m, subst)
+
+
 class SedRegex(callbacks.Plugin):
     """
     Enable SedRegex on the desired channels:
@@ -232,9 +242,21 @@ def doPrivmsg(self, irc, msg):
         if self.registryValue('boldReplacementText', msg.channel, irc.network):
             replacement = ircutils.bold(replacement)
         try:
-            message = process(self._replacer_process, irc, msg,
-                    target, pattern, replacement, count, iterable, sedRegex,
-                    timeout=regex_timeout, pn=self.name(), cn='replacer')
+            if isinstance(world.SUPYPROCESS_MULTIPROCESSING_CONTEXT,
+                          multiprocessing.context.ForkContext):
+                # global state is shared with child processes, so the child
+                # process has access to history and can lazily filter it
+                message = process(self._replacer_process, irc, msg,
+                        target, pattern, replacement, count, iterable, sedRegex,
+                        timeout=regex_timeout, pn=self.name(), cn='replacer')
+            else:
+                # SpawnContext (or possibly ForkServerContext in future
+                # versions of Limnoria): global state is not shared with child
+                # processes, so we have to filter the message list in the main
+                # process and pass it to the child.
+                message = self._replacer(irc, msg,
+                        target, pattern, replacement, count, iterable, sedRegex,
+                        timeout=regex_timeout, pn=self.name(), cn='replacer')
         except ProcessTimeoutError:
             irc.error(_("Search timed out."))
         except SearchNotFoundError:
@@ -277,6 +299,28 @@ def _replacer_process(self, irc, msg, target, pattern, replacement, count,
                          msg.args[1], len(irc.state.history), msg.args[0])
         raise SearchNotFoundError()
 
+    def _replacer(self, irc, msg, target, pattern, replacement, count,
+                          messages, sedRegex, **kwargs):
+        ignoreRegex = self.registryValue('ignoreRegex', msg.channel, irc.network)
+        messages = filter_messages(irc.network, msg, target, messages,
+                                   ignoreRegex, sedRegex)
+        messages = list(messages)  # materialize the iterator to pickle it
+
+        try:
+            result = process(apply_substitution_to_first_matching_message,
+                             pattern, replacement, messages, count,
+                             preload_plugins=["SedRegex"], **kwargs)
+            if result:
+                (m, subst) = result
+                return self._format_result(irc, msg, m, subst)
+        except Exception as e:
+            self.log.warning(_("SedRegex error: %s"), e, exc_info=True)
+            raise
+
+        self.log.debug(_("SedRegex: Search %r not found in the last %i messages of %s."),
+                         msg.args[1], len(irc.state.history), msg.args[0])
+        raise SearchNotFoundError()
+
     doNotice = doPrivmsg
 
 Class = SedRegex

plugins/SedRegex/test.py

@@ -1,6 +1,6 @@
 ###
-# Copyright (c) 2017-2020, James Lu <james@overdrivenetworks.com>
-# Copyright (c) 2020-2021, Valentin Lorentz
+# Copyright (c) 2017-2025, James Lu <james@overdrivenetworks.com>
+# Copyright (c) 2020-2026, Valentin Lorentz
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,11 @@
 
 from __future__ import print_function
 import unittest
+import contextlib
+import multiprocessing
 from supybot.test import *
+import supybot.world as world
+import supybot.callbacks as callbacks
 
 class SedRegexTestCase(ChannelPluginTestCase):
     other = "blah!blah@someone.else"
@@ -324,4 +328,31 @@ def testFmtStringOtherPerson(self):
 
     # TODO: test ignores
 
+
+    @unittest.skipIf(
+        world.disableMultiprocessing,
+        "Test requires multiprocessing to be enabled"
+    )
+    def testSpawnContext(self):
+        """Test that SedRegex works with 'spawn' multiprocessing context
+        (Windows compatibility)."""
+        with useSpawnContext():
+            self.feedMsg('hello world')
+            self.feedMsg('s/world/everyone/')
+            m = self.getMsg(' ')
+            self.assertIn('hello everyone', str(m))
+
+    @unittest.skipIf(
+        world.disableMultiprocessing,
+        "Test requires multiprocessing to be enabled"
+    )
+    def testSpawnReDoSTimeout(self):
+        """Test that ReDoS protection works with 'spawn' context."""
+        with useSpawnContext():
+            for idx in range(500):
+                self.feedMsg("ACCCCCCCCCCCCCCCCCCCCCCCCCCCCX")
+            self.feedMsg(r"s/A(B|C+)+D/this should abort/")
+            m = self.getMsg(' ', timeout=1)
+            self.assertIn('timed out', str(m))
+
 # vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:

src/commands.py

@@ -1,7 +1,7 @@
 ###
 # Copyright (c) 2002-2005, Jeremiah Fincher
 # Copyright (c) 2009-2010,2015, James McCoy
-# Copyright (c) 2010-2021, Valentin Lorentz
+# Copyright (c) 2010-2026, Valentin Lorentz
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -103,6 +103,17 @@ def _process_target(f, q, heap_size, *args, **kwargs):
     except Exception as e:
         q.put([True, e])
 
+def _process_queued_target(f_q, q, heap_size, *args, preload_plugins, **kwargs):
+    """Called by :func:`process` on non-``fork`` multiprocessing contexts"""
+    from .plugin import loadPluginModule
+
+    for plugin in preload_plugins:
+        loadPluginModule(plugin, ignoreDeprecation=True)
+
+    f = f_q.get()
+    _process_target(f, q, heap_size, *args, **kwargs)
+
+
 def process(f, *args, **kwargs):
     """Runs a function <f> in a subprocess.
     
@@ -121,13 +132,14 @@ def process(f, *args, **kwargs):
     if world.disableMultiprocessing:
         pn = kwargs.pop('pn', 'Unknown')
         cn = kwargs.pop('cn', 'unknown')
+        kwargs.pop('preload_plugins')
         try:
             return f(*args, **kwargs)
         except Exception as e:
             raise e
 
     try:
-        q = multiprocessing.Queue()
+        q = world.SUPYPROCESS_MULTIPROCESSING_CONTEXT.Queue()
     except OSError:
         log.error('Using multiprocessing.Queue raised an OSError.\n'
                 'This is probably caused by your system denying semaphore\n'
@@ -137,9 +149,21 @@ def process(f, *args, **kwargs):
                 '(See https://github.com/travis-ci/travis-core/issues/187\n'
                 'for more information about this bug.)\n')
         raise
-    targetArgs = (f, q, heap_size) + args
-    p = callbacks.CommandProcess(target=_process_target,
-                                 args=targetArgs, kwargs=kwargs)
+    if isinstance(world.SUPYPROCESS_MULTIPROCESSING_CONTEXT,
+                  multiprocessing.context.ForkContext):
+        # no need to pickle f
+        targetArgs = (f, q, heap_size) + args
+        p = callbacks.CommandProcess(target=_process_target,
+                                     args=targetArgs, kwargs=kwargs)
+    else:
+        # f must be picklable, but it probably comes from a plugin module,
+        # so we need to load the plugin module first, before unpickling it.
+        # so we put it on the queue instead of an argument.
+        f_q = world.SUPYPROCESS_MULTIPROCESSING_CONTEXT.Queue()
+        f_q.put(f)
+        targetArgs = (f_q, q, heap_size) + args
+        p = callbacks.CommandProcess(target=_process_queued_target,
+                                     args=targetArgs, kwargs=kwargs)
     try:
         p.start()
     except OSError as e:

src/test.py

@@ -1,7 +1,7 @@
 ###
 # Copyright (c) 2002-2005, Jeremiah Fincher
 # Copyright (c) 2011, James McCoy
-# Copyright (c) 2010-2021, Valentin Lorentz
+# Copyright (c) 2010-2026, Valentin Lorentz
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -37,8 +37,11 @@
 import shutil
 import urllib
 import unittest
+import unittest.mock
 import functools
 import threading
+import contextlib
+import multiprocessing
 
 from . import (callbacks, conf, drivers, httpserver, i18n, ircdb, irclib,
         ircmsgs, ircutils, log, plugin, registry, utils, world)
@@ -660,5 +663,53 @@ class ChannelHTTPPluginTestCase(ChannelPluginTestCase, HTTPPluginTestCase):
     def setUp(self):
         ChannelPluginTestCase.setUp(self, forceSetup=True)
 
-# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:
 
+# allows plugins to test the 'spawn' and 'forkserver' methods when
+# the global multiprocessing context is 'fork'.
+#
+# This has to be here because plugins can't define it in their test.py,
+# because Python unpickles the Process before we get a change to load the
+# plugin.
+try:
+    _SPAWN_MULTIPROCESSING_CONTEXT = multiprocessing.get_context('spawn')
+except ValueError:
+    @contextlib.contextmanager
+    def useSpawnContext():
+        raise unittest.SkipTest("'spawn' start method is not supported")
+else:
+    class _SpawnSupyProcess(_SPAWN_MULTIPROCESSING_CONTEXT.Process):
+        def __init__(self, *args, **kwargs):
+            world.processesSpawned += 1
+            super(world.SupyProcess, self).__init__(*args, **kwargs)
+            log.debug('Spawning process %q.', self.name)
+
+    class _SpawnCommandProcess(_SpawnSupyProcess):
+        """Just does some extra logging and error-recovery for commands that need
+        to run in processes.
+        """
+        def __init__(self, target=None, args=(), kwargs={}):
+            pn = kwargs.pop('pn', 'Unknown')
+            cn = kwargs.pop('cn', 'unknown')
+            procName = 'Process #%s (for %s.%s)' % (world.processesSpawned,
+                                                     pn,
+                                                     cn)
+            log.debug('Spawning process %s (args: %r)', procName, args)
+            super().__init__(target=target, name=procName,
+                             args=args, kwargs=kwargs)
+
+    @contextlib.contextmanager
+    def useSpawnContext():
+        """Temporarily redefine SupyProcess and CommandProcess to use the
+        'spawn' multiprocessing context, simulating Windows behaviour on POSIX
+        systems."""
+        with contextlib.ExitStack() as stack:
+            stack.enter_context(unittest.mock.patch(
+                "supybot.world.SUPYPROCESS_MULTIPROCESSING_CONTEXT",
+                _SPAWN_MULTIPROCESSING_CONTEXT))
+            stack.enter_context(unittest.mock.patch(
+                "supybot.world.SupyProcess", _SpawnSupyProcess))
+            stack.enter_context(unittest.mock.patch(
+                "supybot.callbacks.CommandProcess", _SpawnCommandProcess))
+            yield
+
+# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79: