Rope and Jedi race working with async functions

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
       - checkout
       - run: pip install -e .[all] .[test]
       - run: py.test test/
-      - run: pylint pyls test
+      - run: pylint pyls test --ignore-patterns="pyls_race_py3.py"
       - run: pycodestyle pyls test
       - run: pyflakes pyls test
 

pyls/_utils.py

@@ -4,8 +4,15 @@
 import logging
 import os
 import threading
+import sys
 
 log = logging.getLogger(__name__)
+PY3 = sys.version[0] == '3'
+if PY3:
+    # noqa pylint: disable=import-error
+    from .pyls_race_py3 import async_race
+else:
+    from .pyls_race_py2 import async_race
 
 
 def debounce(interval_s, keyed_by=None):
@@ -163,7 +170,7 @@ def is_process_alive(pid):
             return True
 
 
-def race_hooks(hook_caller, pool, **kwargs):
+def race_hooks(hook_caller, **kwargs):
     """Given a pluggy hook spec, execute impls in parallel returning the first non-None result.
 
     Note this does not support a lot of pluggy functionality, e.g. hook wrappers.
@@ -174,15 +181,7 @@ def race_hooks(hook_caller, pool, **kwargs):
     if not impls:
         return None
 
-    def _apply(impl):
-        return impl, impl.function(**kwargs)
-
-    # imap unordered gives us an iterator over the items in the order they finish.
-    # We have to be careful to set chunksize to 1 to ensure hooks each get their own thread.
-    # Unfortunately, there's no way to interrupt these threads, so we just have to leave them be.
-    result = None
-    while result is None:
-        first_impl, result = next(pool.imap_unordered(_apply, impls, chunksize=1))
+    first_impl, result = async_race(impls, **kwargs)
     log.debug("Hook from plugin %s returned: %s", first_impl.plugin_name,
               result)
     return result

pyls/pyls_race_py2.py

@@ -0,0 +1,4 @@
+# pylint: disable=unused-argument
+def async_race(impls, **kwargs):
+    """Dummy function when running the server on python 2."""
+    return None

pyls/pyls_race_py3.py

@@ -0,0 +1,28 @@
+import trio
+
+
+def async_race(impls, **kwargs):
+    """Create  the race between rope and jedi using async functions."""
+    # pylint: disable=syntax-error
+    async def race(impls):
+        send_channel, receive_channel = trio.open_memory_channel(0)
+
+        # pylint: disable=syntax-error
+        async def _apply(impl):
+            return impl, impl.function(**kwargs)
+
+        # pylint: disable=syntax-error
+        async def jockey(impl):
+            if impl.plugin_name == 'rope_completion':
+                await trio.sleep(0.1)
+            await send_channel.send(await _apply(impl))
+
+        async with trio.open_nursery() as nursery:
+            for impl in impls:
+                nursery.start_soon(jockey, impl)
+
+            winner = await receive_channel.receive()
+            if winner is not None or winner is []:
+                nursery.cancel_scope.cancel()
+                return winner
+    return trio.run(race, impls)

pyls/python_ls.py

@@ -2,7 +2,7 @@
 import logging
 import socketserver
 import threading
-from multiprocessing import dummy as multiprocessing
+import sys
 
 from pyls_jsonrpc.dispatchers import MethodDispatcher
 from pyls_jsonrpc.endpoint import Endpoint
@@ -17,9 +17,9 @@
 LINT_DEBOUNCE_S = 0.5  # 500 ms
 PARENT_PROCESS_WATCH_INTERVAL = 10  # 10 s
 MAX_WORKERS = 64
-PLUGGY_RACE_POOL_SIZE = 5
 PYTHON_FILE_EXTENSIONS = ('.py', '.pyi')
 CONFIG_FILEs = ('pycodestyle.cfg', 'setup.cfg', 'tox.ini', '.flake8')
+PY3 = sys.version[0] == '3'
 
 
 class _StreamHandlerWrapper(socketserver.StreamRequestHandler, object):
@@ -76,7 +76,6 @@ class PythonLanguageServer(MethodDispatcher):
     def __init__(self, rx, tx, check_parent_process=False):
         self.workspace = None
         self.config = None
-        self._pool = None
 
         self._jsonrpc_stream_reader = JsonRpcStreamReader(rx)
         self._jsonrpc_stream_writer = JsonRpcStreamWriter(tx)
@@ -148,7 +147,7 @@ def capabilities(self):
             'referencesProvider': True,
             'renameProvider': True,
             'signatureHelpProvider': {
-                'triggerCharacters': ['(', ',']
+                'triggerCharacters': ['(', ',', '=']
             },
             'textDocumentSync': lsp.TextDocumentSyncKind.INCREMENTAL,
             'experimental': merge(self._hook('pyls_experimental_capabilities'))
@@ -165,7 +164,6 @@ def m_initialize(self, processId=None, rootUri=None, rootPath=None, initializati
         self.config = config.Config(rootUri, initializationOptions or {},
                                     processId, _kwargs.get('capabilities', {}))
         self._dispatchers = self._hook('pyls_dispatchers')
-        self._pool = multiprocessing.Pool(PLUGGY_RACE_POOL_SIZE)
         self._hook('pyls_initialize')
 
         if self._check_parent_process and processId is not None:
@@ -195,10 +193,9 @@ def code_lens(self, doc_uri):
 
     def completions(self, doc_uri, position):
         rope_enabled = self.config.settings()['plugins']['rope_completion']['enabled']
-        if rope_enabled:
+        if rope_enabled and PY3:
             completions = _utils.race_hooks(
                 self._hook_caller('pyls_completions'),
-                self._pool,
                 document=self.workspace.get_document(doc_uri) if doc_uri else None,
                 position=position,
                 config=self.config,

setup.py

@@ -67,7 +67,8 @@ def get_readme():
         'pycodestyle',
         'pydocstyle>=2.0.0',
         'pyflakes>=1.6.0',
-        'rope>=0.10.5'
+        'rope>=0.10.5',
+        'trio; python_version>="3.5"'
     ],
 
     # List additional groups of dependencies here (e.g. development