Skip to content

Commit

Permalink
WIP: multi process
Browse files Browse the repository at this point in the history
  • Loading branch information
@Shougo
Shougo committed Jan 30, 2018
1 parent ce51f0c commit 38b9a65
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 137 deletions.
116 changes: 54 additions & 62 deletions rplugin/python3/deoplete/child.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,17 @@
# License: MIT license
# ============================================================================

import os.path
import re
import copy
import time

from collections import defaultdict
from threading import Thread
from queue import Queue
from neovim import attach

from deoplete import logger
from deoplete.exceptions import SourceInitError
from deoplete.process import Process
from deoplete.util import (bytepos2charpos, charpos2bytepos, error, error_tb,
import_plugin,
get_buffer_config, get_custom,
get_syn_names, convert2candidates)

Expand All @@ -32,68 +30,17 @@ def __init__(self, vim):
self._custom = []
self._profile_flag = None
self._profile_start_time = 0
self._loaded_sources = {}
self._loaded_filters = {}
self._source_errors = defaultdict(int)
self._filter_errors = defaultdict(int)
self._prev_results = {}

self._proc = None
self._queue_in = Queue()
self._queue_out = Queue()

def enable_logging(self):
self.is_debug_enabled = True

def add_source(self, s, context):
self._start_thread(context, context['serveraddr'])
self._queue_put('add_source', [s])

def add_filter(self, f):
self._queue_put('add_filter', [f])

def set_source_attributes(self, context):
self._queue_put('set_source_attributes', [context])

def set_custom(self, custom):
self._queue_put('set_custom', [custom])

def merge_results(self, context):
self._queue_put('merge_results', [context])
if self._queue_out.empty():
return (False, [])
return self._queue_out.get()

def on_event(self, context):
self._queue_put('on_event', [context])
if context['event'] == 'VimLeavePre':
self._stop_thread()

def _start_thread(self, context, serveraddr):
if not self._proc:
self._proc = Process(
[context['python3'], context['dp_main']],
context, context['cwd'])
time.sleep(0.1)
error(self._vim, self._proc.communicate(100))

def _stop_thread(self):
if self._proc:
self._proc.kill()
self._proc = None

def _queue_put(self, name, args):
self._queue_in.put([name, args])

def _attach_vim(self, serveraddr):
if len(serveraddr.split(':')) == 2:
serveraddr, port = serveraddr.split(':')
port = int(port)
self._vim = attach('tcp', address=serveraddr, port=port)
else:
self._vim = attach('socket', address=serveraddr)

def _main_loop(self, serveraddr):
def main_loop(self, serveraddr):
self._vim.vars['hoge'] = 1
self._attach_vim(serveraddr)

while 1:
self.debug('main_loop: begin')
Expand All @@ -112,11 +59,56 @@ def _main_loop(self, serveraddr):
elif message == 'merge_results':
self._merge_results(args[0])

def _add_source(self, s):
self._sources[s.name] = s
def _add_source(self, path):
source = None
try:
Source = import_plugin(path, 'source', 'Source')
if not Source:
return

source = Source(self._vim)
name = os.path.splitext(os.path.basename(path))[0]
source.name = getattr(source, 'name', name)
source.path = path
if source.name in self._loaded_sources:
# Duplicated name
error_tb(self._vim, 'duplicated source: %s' % source.name)
error_tb(self._vim, 'path: "%s" "%s"' %
(path, self._loaded_sources[source.name]))
source = None
except Exception:
error_tb(self._vim, 'Could not load source: %s' % name)
finally:
if source:
self._loaded_sources[source.name] = path
self._sources[source.name] = source
self.debug('Loaded Source: %s (%s)', source.name, path)

def _add_filter(self, path):
f = None
try:
Filter = import_plugin(path, 'filter', 'Filter')
if not Filter:
return

def _add_filter(self, f):
self._filters[f.name] = f
f = Filter(self._vim)
name = os.path.splitext(os.path.basename(path))[0]
f.name = getattr(f, 'name', name)
f.path = path
if f.name in self._loaded_filters:
# Duplicated name
error_tb(self._vim, 'duplicated filter: %s' % f.name)
error_tb(self._vim, 'path: "%s" "%s"' %
(path, self._loaded_filters[f.name]))
f = None
except Exception:
# Exception occurred when loading a filter. Log stack trace.
error_tb(self._vim, 'Could not load filter: %s' % name)
finally:
if f:
self._loaded_filters[f.name] = path
self._filters[f.name] = f
self.debug('Loaded Filter: %s (%s)', f.name, path)

def _merge_results(self, context):
results = self._gather_results(context)
Expand Down
95 changes: 23 additions & 72 deletions rplugin/python3/deoplete/deoplete.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@
# License: MIT license
# ============================================================================

import os.path

import deoplete.util # noqa
import deoplete.filter # noqa
import deoplete.source # noqa

from deoplete import logger
from deoplete.child import Child
from deoplete.util import (error, error_tb, find_rplugins, import_plugin)
from deoplete.parent import Parent
from deoplete.util import (error, error_tb, find_rplugins)


class Deoplete(logger.LoggingMixin):
Expand All @@ -27,11 +25,11 @@ def __init__(self, vim):
self._loaded_sources = {}
self._loaded_filters = {}

self._children = []
self._child_count = 0
self._max_children = 1
for n in range(0, self._max_children):
self._children.append(Child(vim))
self._parents = []
self._parent_count = 0
self._max_parents = 1
for n in range(0, self._max_parents):
self._parents.append(Parent(vim))

# Enable logging before "Init" for more information, and e.g.
# deoplete-jedi picks up the log filename from deoplete's handler in
Expand All @@ -52,8 +50,8 @@ def enable_logging(self):
logging = self._vim.vars['deoplete#_logging']
logger.setup(self._vim, logging['level'], logging['logfile'])
self.is_debug_enabled = True
for child in self._children:
child.enable_logging()
for parent in self._parents:
parent.enable_logging()

def completion_begin(self, context):
self.debug('completion_begin: %s', context['input'])
Expand Down Expand Up @@ -92,8 +90,8 @@ def completion_begin(self, context):
def merge_results(self, context):
is_async = False
merged_results = []
for child in self._children:
result = child.merge_results(context)
for parent in self._parents:
result = parent.merge_results(context)
is_async = is_async or result[0]
merged_results += result[1]

Expand Down Expand Up @@ -138,33 +136,10 @@ def load_sources(self, context):
continue
self._loaded_paths.add(path)

name = os.path.splitext(os.path.basename(path))[0]

source = None
try:
Source = import_plugin(path, 'source', 'Source')
if not Source:
continue

source = Source(self._vim)
source.name = getattr(source, 'name', name)
source.path = path
if source.name in self._loaded_sources:
# Duplicated name
error_tb(self._vim, 'duplicated source: %s' % source.name)
error_tb(self._vim, 'path: "%s" "%s"' %
(path, self._loaded_sources[source.name]))
source = None
except Exception:
error_tb(self._vim, 'Could not load source: %s' % name)
finally:
if source:
self._loaded_sources[source.name] = path
self._children[self._child_count].add_source(
source, context)
self._child_count += 1
self._child_count %= self._max_children
self.debug('Loaded Source: %s (%s)', source.name, path)
self._parents[self._parent_count].add_source(path, context)

self._parent_count += 1
self._parent_count %= self._max_parents

self.set_source_attributes(context)
self.set_custom(context)
Expand All @@ -176,41 +151,17 @@ def load_filters(self, context):
continue
self._loaded_paths.add(path)

name = os.path.splitext(os.path.basename(path))[0]

f = None
try:
Filter = import_plugin(path, 'filter', 'Filter')
if not Filter:
continue

f = Filter(self._vim)
f.name = getattr(f, 'name', name)
f.path = path
if f.name in self._loaded_filters:
# Duplicated name
error_tb(self._vim, 'duplicated filter: %s' % f.name)
error_tb(self._vim, 'path: "%s" "%s"' %
(path, self._loaded_filters[f.name]))
f = None
except Exception:
# Exception occurred when loading a filter. Log stack trace.
error_tb(self._vim, 'Could not load filter: %s' % name)
finally:
if f:
self._loaded_filters[f.name] = path
for child in self._children:
child.add_filter(f)
self.debug('Loaded Filter: %s (%s)', f.name, path)
for parent in self._parents:
parent.add_filter(path, context)

def set_source_attributes(self, context):
for child in self._children:
child.set_source_attributes(context)
for parent in self._parents:
parent.set_source_attributes(context)

def set_custom(self, context):
self._custom = context['custom']
for child in self._children:
child.set_custom(self._custom)
for parent in self._parents:
parent.set_custom(self._custom)

def check_recache(self, context):
if context['runtimepath'] != self._runtimepath:
Expand All @@ -228,5 +179,5 @@ def on_event(self, context):
self.debug('on_event: %s', context['event'])
self.check_recache(context)

for child in self._children:
child.on_event(context)
for parent in self._parents:
parent.on_event(context)
37 changes: 34 additions & 3 deletions rplugin/python3/deoplete/dp_main.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,38 @@
# ============================================================================
# FILE: dp_main.py
# AUTHOR: Shougo Matsushita <Shougo.Matsu at gmail.com>
# License: MIT license
# ============================================================================

def main():
print('foo')
import sys
from neovim import attach


def attach_vim(serveraddr):
if len(serveraddr.split(':')) == 2:
serveraddr, port = serveraddr.split(':')
port = int(port)
vim = attach('tcp', address=serveraddr, port=port)
else:
vim = attach('socket', path=serveraddr)

# sync path
for path in vim.call(
'globpath', vim.options['runtimepath'],
'rplugin/python3', 1).split('\n'):
sys.path.append(path)
# Remove current path
del sys.path[0]

return vim


def main(serveraddr):
vim = attach_vim(serveraddr)
from deoplete.util import error
error(vim, 'hoge')
return


if __name__ == '__main__':
main()
main(sys.argv[1])
Loading

0 comments on commit 38b9a65

Please sign in to comment.