Skip to content

Commit 22c70c6

Browse files
committed
SedRegex: Cache ircdb.checkIgnored during a request
Benchmarking shows a 2 to 4 seconds improvement for all multiprocessing methods: | method | cached | target check | far | middle | last | | --------- | ------ | ------------------- | ---- | ------ | ---- | | fork | no | before checkIgnored | 17.0 | 17.1 | 13.3 | | fork | yes | before checkIgnored | 12.6 | 13.5 | 11.0 | | fork | no | after checkIgnored | 16.7 | 17.1 | 13.4 | | fork | yes | after checkIgnored | 12.6 | 13.4 | 11.2 | | list fork | no | before checkIgnored | | list fork | yes | before checkIgnored | 10.2 | 10.8 | 10.7 | | list fork | no | after checkIgnored | 14.7 | 15.3 | 14.9 | | list fork | yes | after checkIgnored | 10.2 | 10.7 | 10.5 | | forkserver | no | before checkIgnored | 171.9 | 171.9 | 172.3 | | forkserver | yes | before checkIgnored | 168.5 | 167.8 | 169.6 | | forkserver | no | after checkIgnored | | forkserver | yes | after checkIgnored | ```py def testPerfLast(self): for i in range(10000): self.feedMsg(f'xxx {i} yyy') self.assertResponse('ping', 'pong') before = time.time() for _ in range(1000): self.feedMsg('s/xxx 9999 yyy/foo/') # no ^ or $, so it matches itself after = time.time() print(after - before) assert False def testPerfMiddle(self): for i in range(10000): self.feedMsg(f'xxx {i} yyy') self.assertResponse('ping', 'pong') before = time.time() for _ in range(1000): self.feedMsg('s/^xxx 1234 yyy$/foo/') # I meant to use 9876 here. so it's actually the same test as testPerfFar. too bad. after = time.time() print(after - before) assert False def testPerfFar(self): for i in range(10000): self.feedMsg(f'xxx {i} yyy') self.assertResponse('ping', 'pong') before = time.time() for _ in range(1000): self.feedMsg('s/^xxx 1234 yyy$/foo/') after = time.time() print(after - before) assert False ``` "pickle fork" means I used fork with this patch: ```diff diff --git a/plugins/SedRegex/plugin.py b/plugins/SedRegex/plugin.py @@ -242,8 +246,7 @@ def doPrivmsg(self, irc, msg): if self.registryValue('boldReplacementText', msg.channel, irc.network): replacement = ircutils.bold(replacement) try: - if isinstance(world.SUPYPROCESS_MULTIPROCESSING_CONTEXT, - multiprocessing.context.ForkContext): + if False: # global state is shared with child processes, so the child # process has access to history and can lazily filter it message = process(self._replacer_process, irc, msg, ```
1 parent 5a92109 commit 22c70c6

1 file changed

Lines changed: 4 additions & 0 deletions

File tree

plugins/SedRegex/plugin.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
###
3232

33+
import functools
3334
import multiprocessing
3435
from supybot.commands import *
3536
from supybot.commands import ProcessTimeoutError
@@ -64,6 +65,9 @@ class SearchNotFoundError(Exception):
6465
def filter_messages(network, msg, target, messages, ignoreRegex, sedRegex):
6566
"""Applies all filters but the user-provided regexp, so it is safe to
6667
run in the main process."""
68+
# short-lived cache for the duration of this request.
69+
checkIgnored = functools.cache(ircdb.checkIgnored)
70+
6771
for m in messages:
6872
if m.command in ('PRIVMSG', 'NOTICE') and \
6973
ircutils.strEqual(m.args[0], msg.args[0]) and \

0 commit comments

Comments
 (0)