From d67c82a6e09c9316c58e52e4a1ce4badfc3bc71e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 18 May 2017 17:47:56 +0100 Subject: [PATCH 01/89] Make a start on kernel discovery framework --- jupyter_client/discovery.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 jupyter_client/discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py new file mode 100644 index 000000000..7e640c32f --- /dev/null +++ b/jupyter_client/discovery.py @@ -0,0 +1,80 @@ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + + +class KernelSpecFinder(object): + """Find kernels from installed kernelspec directories. + """ + id = 'spec' + + def __init__(self): + self.ksm = KernelSpecManager() + + def find_kernels(self): + for name, resdir in self.ksm.find_kernel_specs().items(): + spec = self.ksm._get_kernel_spec_by_name(name, resdir) + yield name, { + # TODO: get full language info + 'language': {'name': spec.language}, + 'display_name': spec.display_name, + 'argv': spec.argv, + } + + def make_manager(self, name): + spec = self.ksm.get_kernel_spec(name) + return KernelManager(kernel_cmd=spec.argv) # TODO: env + + +class IPykernelFinder(object): + """Find ipykernel on this Python version by trying to import it. + """ + id = 'pyimport' + + def _check_for_kernel(self): + try: + from ipykernel.kernelspec import RESOURCES, get_kernel_dict + from ipykernel.ipkernel import IPythonKernel + except ImportError: + return None + else: + return { + 'spec': get_kernel_dict(), + 'language_info': IPythonKernel.language_info, + 'resources_dir': RESOURCES, + } + + def find_kernels(self): + info = self._check_for_kernel() + + if info: + yield 'kernel', { + 'language': info['language_info'], + 'display_name': info['spec']['display_name'], + 'argv': info['spec']['argv'], + } + + def make_manager(self): + info = self._check_for_kernel() + if info is None: + raise Exception("ipykernel is not importable") + return KernelManager(kernel_cmd=info['spec']['argv']) + + +class MetaKernelFinder(object): + def __init__(self): + self.finders = [ + KernelSpecFinder(), + IPykernelFinder(), + ] + + def find_kernels(self): + for finder in self.finders: + for kid, attributes in finder.find_kernels(): + id = finder.id + '/' + kid + yield id, attributes + + def make_manager(self, id): + finder_id, kernel_id = id.split('/', 1) + for finder in self.finders: + if finder_id == finder.id: + return finder.make_manager(kernel_id) From 6406393ff0fcb66fc547f77f81541af1e507978b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 19 May 2017 14:31:37 +0100 Subject: [PATCH 02/89] Undeprecate KernelManager.kernel_cmd, add extra_env --- jupyter_client/discovery.py | 4 ++-- jupyter_client/manager.py | 24 ++++++++---------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 7e640c32f..737ee38b1 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -22,7 +22,7 @@ def find_kernels(self): def make_manager(self, name): spec = self.ksm.get_kernel_spec(name) - return KernelManager(kernel_cmd=spec.argv) # TODO: env + return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) class IPykernelFinder(object): @@ -53,7 +53,7 @@ def find_kernels(self): 'argv': info['spec']['argv'], } - def make_manager(self): + def make_manager(self, name): info = self._check_for_kernel() if info is None: raise Exception("ipykernel is not importable") diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index d50a5fbb8..4e0387762 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -22,7 +22,7 @@ from ipython_genutils.importstring import import_item from .localinterfaces import is_local_ip, local_ips from traitlets import ( - Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName + Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName, Dict ) from jupyter_client import ( launch_kernel, @@ -87,23 +87,13 @@ def kernel_spec(self): self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec - kernel_cmd = List(Unicode(), config=True, - help="""DEPRECATED: Use kernel_name instead. - - The Popen Command to launch the kernel. - Override this if you have a custom kernel. - If kernel_cmd is specified in a configuration file, - Jupyter does not pass any arguments to the kernel, - because it cannot make any assumptions about the - arguments that the kernel understands. In particular, - this means that the kernel does not receive the - option --debug if it given on the Jupyter command line. - """ + kernel_cmd = List(Unicode(), + help="""The Popen Command to launch the kernel.""" ) - def _kernel_cmd_changed(self, name, old, new): - warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " - "start different kernels.") + extra_env = Dict( + help="""Extra environment variables to be set for the kernel.""" + ) @property def ipykernel(self): @@ -254,6 +244,8 @@ def start_kernel(self, **kw): # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) + elif self.extra_env: + env.update(self.extra_env) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) From 6ca3ec77ab675a0c1b31dbd3f67538589f1d63cc Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:35:52 +0100 Subject: [PATCH 03/89] Use entry points to find kernel finders --- jupyter_client/discovery.py | 24 +++++++++++++++++++----- setup.py | 4 ++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 737ee38b1..d6ff84b68 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,6 +1,10 @@ +import entrypoints +import logging + from .kernelspec import KernelSpecManager from .manager import KernelManager +log = logging.getLogger(__name__) class KernelSpecFinder(object): """Find kernels from installed kernelspec directories. @@ -61,11 +65,21 @@ def make_manager(self, name): class MetaKernelFinder(object): - def __init__(self): - self.finders = [ - KernelSpecFinder(), - IPykernelFinder(), - ] + def __init__(self, finders): + self.finders = finders + + @classmethod + def from_entrypoints(cls): + finders = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + try: + finder = ep.load()() # Load and instantiate + except Exception: + log.error('Error loading kernel finder', exc_info=True) + else: + finders.append(finder) + + return cls(finders) def find_kernels(self): for finder in self.finders: diff --git a/setup.py b/setup.py index 341af7fb2..a48d4c428 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,10 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + ], + 'jupyter_client.kernel_finders' : [ + 'spec = jupyter_client.discovery:KernelSpecFinder', + 'pyimport = jupyter_client.discovery:IPykernelFinder', ] }, ) From dddda322e93807b9b09af0d8bbf1967c4faa90b8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 16:56:34 +0100 Subject: [PATCH 04/89] Tests for kernel discovery machinery --- jupyter_client/discovery.py | 21 +++++++++++++++-- jupyter_client/tests/test_discovery.py | 32 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 jupyter_client/tests/test_discovery.py diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index d6ff84b68..f43cb48a4 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod import entrypoints import logging @@ -6,7 +7,23 @@ log = logging.getLogger(__name__) -class KernelSpecFinder(object): +class KernelFinderBase(ABC): + id = None # Should be a short string identifying the finder class. + + @abstractmethod + def find_kernels(self): + """Return an iterator of (kernel_name, kernel_info_dict) tuples.""" + pass + + @abstractmethod + def make_manager(self, name): + """Make and return a KernelManager instance to start a specified kernel + + name will be one of the kernel names produced by find_kernels() + """ + pass + +class KernelSpecFinder(KernelFinderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -29,7 +46,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(object): +class IPykernelFinder(KernelFinderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py new file mode 100644 index 000000000..f6a462327 --- /dev/null +++ b/jupyter_client/tests/test_discovery.py @@ -0,0 +1,32 @@ +import sys + +from jupyter_client import KernelManager +from jupyter_client import discovery + +def test_ipykernel_finder(): + import ipykernel # Fail clearly if ipykernel not installed + ikf = discovery.IPykernelFinder() + + res = list(ikf.find_kernels()) + assert len(res) == 1, res + id, info = res[0] + assert id == 'kernel' + assert info['argv'][0] == sys.executable + +class DummyKernelFinder(discovery.KernelFinderBase): + """A dummy kernel finder for testing MetaKernelFinder""" + id = 'dummy' + + def find_kernels(self): + yield 'sample', {'argv': ['dummy_kernel']} + + def make_manager(self, name): + return KernelManager(kernel_cmd=['dummy_kernel']) + +def test_meta_kernel_finder(): + mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) + assert list(mkf.find_kernels()) == \ + [('dummy/sample', {'argv': ['dummy_kernel']})] + + manager = mkf.make_manager('dummy/sample') + assert manager.kernel_cmd == ['dummy_kernel'] From 1509dacd4c526eb3976577ee14c557abc77c97ed Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 26 Jul 2017 17:47:10 +0100 Subject: [PATCH 05/89] Use older ABC definition style with metaclass --- jupyter_client/discovery.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index f43cb48a4..9fd4e6327 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -1,13 +1,14 @@ -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod import entrypoints import logging +import six from .kernelspec import KernelSpecManager from .manager import KernelManager log = logging.getLogger(__name__) -class KernelFinderBase(ABC): +class KernelFinderBase(six.with_metaclass(ABCMeta, object)): id = None # Should be a short string identifying the finder class. @abstractmethod From 38ccbdc0751a35d46abca80ae64a106d492841cd Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:21:28 +0100 Subject: [PATCH 06/89] Rename kernel finders -> kernel providers MetaKernelFinder -> KernelFinder --- jupyter_client/discovery.py | 55 ++++++++++++++++---------- jupyter_client/tests/test_discovery.py | 14 +++---- setup.py | 2 +- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 9fd4e6327..53ccab11b 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -8,8 +8,8 @@ log = logging.getLogger(__name__) -class KernelFinderBase(six.with_metaclass(ABCMeta, object)): - id = None # Should be a short string identifying the finder class. +class KernelProviderBase(six.with_metaclass(ABCMeta, object)): + id = None # Should be a short string identifying the provider class. @abstractmethod def find_kernels(self): @@ -24,7 +24,7 @@ def make_manager(self, name): """ pass -class KernelSpecFinder(KernelFinderBase): +class KernelSpecProvider(KernelProviderBase): """Find kernels from installed kernelspec directories. """ id = 'spec' @@ -47,7 +47,7 @@ def make_manager(self, name): return KernelManager(kernel_cmd=spec.argv, extra_env=spec.env) -class IPykernelFinder(KernelFinderBase): +class IPykernelProvider(KernelProviderBase): """Find ipykernel on this Python version by trying to import it. """ id = 'pyimport' @@ -82,31 +82,46 @@ def make_manager(self, name): return KernelManager(kernel_cmd=info['spec']['argv']) -class MetaKernelFinder(object): - def __init__(self, finders): - self.finders = finders +class KernelFinder(object): + """Manages a collection of kernel providers to find available kernels + """ + def __init__(self, providers): + self.providers = providers @classmethod def from_entrypoints(cls): - finders = [] - for ep in entrypoints.get_group_all('jupyter_client.kernel_finders'): + """Load all kernel providers advertised by entry points. + + Kernel providers should use the "jupyter_client.kernel_providers" + entry point group. + + Returns an instance of KernelFinder. + """ + providers = [] + for ep in entrypoints.get_group_all('jupyter_client.kernel_providers'): try: - finder = ep.load()() # Load and instantiate + provider = ep.load()() # Load and instantiate except Exception: - log.error('Error loading kernel finder', exc_info=True) + log.error('Error loading kernel provider', exc_info=True) else: - finders.append(finder) + providers.append(provider) - return cls(finders) + return cls(providers) def find_kernels(self): - for finder in self.finders: - for kid, attributes in finder.find_kernels(): - id = finder.id + '/' + kid + """Iterate over available kernels. + + Yields 2-tuples of (id_str, attributes) + """ + for provider in self.providers: + for kid, attributes in provider.find_kernels(): + id = provider.id + '/' + kid yield id, attributes def make_manager(self, id): - finder_id, kernel_id = id.split('/', 1) - for finder in self.finders: - if finder_id == finder.id: - return finder.make_manager(kernel_id) + """Make a KernelManager instance for a given kernel ID. + """ + provider_id, kernel_id = id.split('/', 1) + for provider in self.providers: + if provider_id == provider.id: + return provider.make_manager(kernel_id) diff --git a/jupyter_client/tests/test_discovery.py b/jupyter_client/tests/test_discovery.py index f6a462327..9d7833ba3 100644 --- a/jupyter_client/tests/test_discovery.py +++ b/jupyter_client/tests/test_discovery.py @@ -3,9 +3,9 @@ from jupyter_client import KernelManager from jupyter_client import discovery -def test_ipykernel_finder(): +def test_ipykernel_provider(): import ipykernel # Fail clearly if ipykernel not installed - ikf = discovery.IPykernelFinder() + ikf = discovery.IPykernelProvider() res = list(ikf.find_kernels()) assert len(res) == 1, res @@ -13,8 +13,8 @@ def test_ipykernel_finder(): assert id == 'kernel' assert info['argv'][0] == sys.executable -class DummyKernelFinder(discovery.KernelFinderBase): - """A dummy kernel finder for testing MetaKernelFinder""" +class DummyKernelProvider(discovery.KernelProviderBase): + """A dummy kernel provider for testing KernelFinder""" id = 'dummy' def find_kernels(self): @@ -24,9 +24,9 @@ def make_manager(self, name): return KernelManager(kernel_cmd=['dummy_kernel']) def test_meta_kernel_finder(): - mkf = discovery.MetaKernelFinder(finders=[DummyKernelFinder()]) - assert list(mkf.find_kernels()) == \ + kf = discovery.KernelFinder(providers=[DummyKernelProvider()]) + assert list(kf.find_kernels()) == \ [('dummy/sample', {'argv': ['dummy_kernel']})] - manager = mkf.make_manager('dummy/sample') + manager = kf.make_manager('dummy/sample') assert manager.kernel_cmd == ['dummy_kernel'] diff --git a/setup.py b/setup.py index a48d4c428..2399a8139 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def run(self): 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], - 'jupyter_client.kernel_finders' : [ + 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecFinder', 'pyimport = jupyter_client.discovery:IPykernelFinder', ] From 3c09a5732d569be86fd92f06b3deeab6413c4e65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:26:52 +0100 Subject: [PATCH 07/89] Missed a rename --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2399a8139..099ddc90a 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,8 @@ def run(self): 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', ], 'jupyter_client.kernel_providers' : [ - 'spec = jupyter_client.discovery:KernelSpecFinder', - 'pyimport = jupyter_client.discovery:IPykernelFinder', + 'spec = jupyter_client.discovery:KernelSpecProvider', + 'pyimport = jupyter_client.discovery:IPykernelProvider', ] }, ) From e92e5c194adde5188039269255a0f82c2c45627d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 9 Oct 2017 15:41:43 +0100 Subject: [PATCH 08/89] Add dependency on entrypoints --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 099ddc90a..f042f00b3 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ def run(self): 'jupyter_core', 'pyzmq>=13', 'python-dateutil>=2.1', + 'entrypoints', ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], From aad40cb6ebb785f8ce51ee70b9872d62aef09e32 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:17:58 +0100 Subject: [PATCH 09/89] Document new kernel providers system --- docs/index.rst | 1 + docs/kernel_providers.rst | 146 ++++++++++++++++++++++++++++++++++++ jupyter_client/discovery.py | 18 +++-- 3 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 docs/kernel_providers.rst diff --git a/docs/index.rst b/docs/index.rst index a0b8855cc..41e218ccc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ with Jupyter kernels. kernels wrapperkernels + kernel_providers .. toctree:: :maxdepth: 2 diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst new file mode 100644 index 000000000..8d51ec7f4 --- /dev/null +++ b/docs/kernel_providers.rst @@ -0,0 +1,146 @@ +================ +Kernel providers +================ + +.. note:: + This is a new interface under development. Not all Jupyter applications + use this yet. See :ref:`kernelspecs` for the established way of discovering + kernel types. + +By writing a kernel provider, you can extend how Jupyter applications discover +and start kernels. To do so, subclass +:class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID +and overriding two methods. + +.. class:: MyKernelProvider + + .. attribute:: id + + A short string identifying this provider. Cannot contain forward slash + (``/``). + + .. method:: find_kernels() + + Get the available kernel types this provider knows about. + Return an iterable of 2-tuples: (name, attributes). + *name* is a short string identifying the kernel type. + *attributes* is a dictionary with information to allow selecting a kernel. + + .. method:: make_manager(name) + + Prepare and return a :class:`~jupyter_client.KernelManager` instance + ready to start a new kernel instance of the type identified by *name*. + The input will be one of the names given by :meth:`find_kernels`. + +For example, imagine we want to tell Jupyter about kernels for a new language +called *oblong*:: + + # oblong_provider.py + from jupyter_client.discover import KernelProviderBase + from jupyter_client import KernelManager + from shutil import which + + class OblongKernelProvider(KernelProviderBase): + id = 'oblong' + + def find_kernels(self): + if not which('oblong-kernel'): + return # Check it's available + + # Two variants - for a real kernel, these could be different + # environments + yield 'standard', { + 'display_name': 'Oblong (standard)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + yield 'rounded', { + 'display_name': 'Oblong (rounded)', + 'language': {'name': 'oblong'}, + 'argv': ['oblong-kernel'], + } + + def make_manager(self, name): + if name == 'standard': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '0'}) + elif name == 'rounded': + return KernelManager(kernel_cmd=['oblong-kernel'], + extra_env={'ROUNDED': '1'}) + else: + raise ValueError("Unknown kernel %s" % name) + +You would then register this with an *entry point*. In your ``setup.py``, put +something like this:: + + setup(... + entry_points = { + 'jupyter_client.kernel_providers' : [ + # The name before the '=' should match the id attribute + 'oblong = oblong_provider:OblongKernelProvider', + ] + }) + +To find and start kernels in client code, use +:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel +providers, but it wraps a set of kernel providers. The kernel names it works +with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example +above). + +:: + + from jupyter_client.discovery import KernelFinder + kf = KernelFinder.from_entrypoints() + + ## Find available kernel types + for name, attributes in kf.find_kernels(): + print(name, ':', attributes['display_name']) + # oblong/standard : Oblong (standard) + # oblong/rounded : Oblong(rounded) + # ... + + ## Start a kernel by name + manager = kf.make_manager('oblong/standard') + manager.start_kernel() + +.. module:: jupyter_client.discovery + +.. autoclass:: KernelFinder + + .. automethod:: from_entrypoints + + .. automethod:: find_kernels + + .. automethod:: make_manager + +Included kernel providers +========================= + +``jupyter_client`` includes two kernel providers: + +.. autoclass:: KernelSpecProvider + + .. seealso:: :ref:`kernelspecs` + +.. autoclass:: IPykernelProvider + +Glossary +======== + +Kernel instance + A running kernel, a process which can accept ZMQ connections from frontends. + Its state includes a namespace and an execution counter. + +Kernel type + Allows starting multiple, initially similar kernel instances. The kernel type + entails the combination of software to run the kernel, and the context in + which it starts. For instance, one kernel type may be associated with one + conda environment containing ``ipykernel``. The same kernel software in + another environment would be a different kernel type. Another software package + for a kernel, such as ``IRkernel``, would also be a different kernel type. + +Kernel provider + A Python class to discover kernel types and allow a client to start instances + of those kernel types. For instance, one kernel provider might find conda + environments containing ``ipykernel`` and allow starting kernel instances in + these environments. diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 53ccab11b..6f6b52f0d 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -25,7 +25,7 @@ def make_manager(self, name): pass class KernelSpecProvider(KernelProviderBase): - """Find kernels from installed kernelspec directories. + """Offers kernel types from installed kernelspec directories. """ id = 'spec' @@ -48,7 +48,9 @@ def make_manager(self, name): class IPykernelProvider(KernelProviderBase): - """Find ipykernel on this Python version by trying to import it. + """Offers a kernel type using the Python interpreter it's running in. + + This checks if ipykernel is importable first. """ id = 'pyimport' @@ -83,7 +85,9 @@ def make_manager(self, name): class KernelFinder(object): - """Manages a collection of kernel providers to find available kernels + """Manages a collection of kernel providers to find available kernel types + + *providers* should be a list of kernel provider instances. """ def __init__(self, providers): self.providers = providers @@ -109,17 +113,17 @@ def from_entrypoints(cls): return cls(providers) def find_kernels(self): - """Iterate over available kernels. + """Iterate over available kernel types. - Yields 2-tuples of (id_str, attributes) + Yields 2-tuples of (prefixed_name, attributes) """ for provider in self.providers: for kid, attributes in provider.find_kernels(): id = provider.id + '/' + kid yield id, attributes - def make_manager(self, id): - """Make a KernelManager instance for a given kernel ID. + def make_manager(self, name): + """Make a KernelManager instance for a given kernel type. """ provider_id, kernel_id = id.split('/', 1) for provider in self.providers: From c09b8aced5912901b4e96e11d9d4eeb3346b4fd7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 15:25:41 +0100 Subject: [PATCH 10/89] Break it up a bit with a subheading --- docs/kernel_providers.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 8d51ec7f4..c5a62cc3d 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -81,6 +81,9 @@ something like this:: ] }) +Finding kernel types +==================== + To find and start kernels in client code, use :class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel providers, but it wraps a set of kernel providers. The kernel names it works From cc8176b088fdf7a5ea955fec980c6b90e8bd21f7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:06:43 +0100 Subject: [PATCH 11/89] Update doc with Carol's suggestions --- docs/kernel_providers.rst | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index c5a62cc3d..65cbd9c8f 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -3,12 +3,18 @@ Kernel providers ================ .. note:: - This is a new interface under development. Not all Jupyter applications - use this yet. See :ref:`kernelspecs` for the established way of discovering - kernel types. + This is a new interface under development, and may still change. + Not all Jupyter applications use this yet. + See :ref:`kernelspecs` for the established way of discovering kernel types. + +Creating a kernel provider +========================== By writing a kernel provider, you can extend how Jupyter applications discover -and start kernels. To do so, subclass +and start kernels. For example, you could find kernels in an environment system +like conda, or kernels on remote systems which you can access. + +To write a kernel provider, subclass :class:`jupyter_client.discovery.KernelProviderBase`, giving your provider an ID and overriding two methods. @@ -47,8 +53,8 @@ called *oblong*:: if not which('oblong-kernel'): return # Check it's available - # Two variants - for a real kernel, these could be different - # environments + # Two variants - for a real kernel, these could be something like + # different conda environments. yield 'standard', { 'display_name': 'Oblong (standard)', 'language': {'name': 'oblong'}, @@ -85,8 +91,9 @@ Finding kernel types ==================== To find and start kernels in client code, use -:class:`jupyter_client.discovery.KernelFinder`. This has a similar API to kernel -providers, but it wraps a set of kernel providers. The kernel names it works +:class:`jupyter_client.discovery.KernelFinder`. This uses multiple kernel +providers to find available kernels. Like a kernel provider, it has methods +``find_kernels`` and ``make_manager``. The kernel names it works with have the provider ID as a prefix, e.g. ``oblong/rounded`` (from the example above). @@ -116,8 +123,8 @@ above). .. automethod:: make_manager -Included kernel providers -========================= +Kernel providers included in ``jupyter_client`` +=============================================== ``jupyter_client`` includes two kernel providers: @@ -135,9 +142,9 @@ Kernel instance Its state includes a namespace and an execution counter. Kernel type - Allows starting multiple, initially similar kernel instances. The kernel type - entails the combination of software to run the kernel, and the context in - which it starts. For instance, one kernel type may be associated with one + The software to run a kernel instance, along with the context in which a + kernel starts. One kernel type allows starting multiple, initially similar + kernel instances. For instance, one kernel type may be associated with one conda environment containing ``ipykernel``. The same kernel software in another environment would be a different kernel type. Another software package for a kernel, such as ``IRkernel``, would also be a different kernel type. From 1f74c5f40f9948b1978451ceb473beb319cd7649 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Oct 2017 18:07:53 +0100 Subject: [PATCH 12/89] Fix variable name --- jupyter_client/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/discovery.py b/jupyter_client/discovery.py index 6f6b52f0d..2bfe92b2a 100644 --- a/jupyter_client/discovery.py +++ b/jupyter_client/discovery.py @@ -125,7 +125,7 @@ def find_kernels(self): def make_manager(self, name): """Make a KernelManager instance for a given kernel type. """ - provider_id, kernel_id = id.split('/', 1) + provider_id, kernel_id = name.split('/', 1) for provider in self.providers: if provider_id == provider.id: return provider.make_manager(kernel_id) From 16608fc7835cba0bbd9e62feac31481dcd517c64 Mon Sep 17 00:00:00 2001 From: didier amyot Date: Wed, 18 Oct 2017 20:29:25 -0400 Subject: [PATCH 13/89] Fix typo in documentation. --- docs/kernel_providers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernel_providers.rst b/docs/kernel_providers.rst index 65cbd9c8f..2e1b7e295 100644 --- a/docs/kernel_providers.rst +++ b/docs/kernel_providers.rst @@ -42,7 +42,7 @@ For example, imagine we want to tell Jupyter about kernels for a new language called *oblong*:: # oblong_provider.py - from jupyter_client.discover import KernelProviderBase + from jupyter_client.discovery import KernelProviderBase from jupyter_client import KernelManager from shutil import which From 936dfe0584441ababc8e6d86740f4791f7739a19 Mon Sep 17 00:00:00 2001 From: frelon Date: Wed, 1 Nov 2017 12:56:55 +0100 Subject: [PATCH 14/89] Updated URL for Jupyter Kernels The old URL points to a "This page has moved"-page --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 3319dda31..2fe1500aa 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -6,7 +6,7 @@ Making kernels for Jupyter A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for -`several other languages `_. +`several other languages `_. When Jupyter starts a kernel, it passes it a connection file. This specifies how to set up communications with the frontend. From aca5f7084014ec69d51f5141a4fd1bdfb1aa3a3b Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 10 Nov 2017 14:43:08 +0100 Subject: [PATCH 15/89] tornado 5 support - use IOLoop.current over IOLoop.instance - drop removed `loop` arg from PeriodicCallback - deprecate now-unused IOLoopKernelRestarter.loop --- jupyter_client/ioloop/manager.py | 20 ++++---------------- jupyter_client/ioloop/restarter.py | 27 +++++++++------------------ jupyter_client/session.py | 4 ++-- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/jupyter_client/ioloop/manager.py b/jupyter_client/ioloop/manager.py index 511a73f55..cc285291b 100644 --- a/jupyter_client/ioloop/manager.py +++ b/jupyter_client/ioloop/manager.py @@ -1,15 +1,7 @@ """A kernel manager with a tornado IOLoop""" -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import @@ -24,10 +16,6 @@ from jupyter_client.manager import KernelManager from .restarter import IOLoopKernelRestarter -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - def as_zmqstream(f): def wrapped(self, *args, **kwargs): @@ -37,9 +25,9 @@ def wrapped(self, *args, **kwargs): class IOLoopKernelManager(KernelManager): - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + return ioloop.IOLoop.current() restarter_class = Type( default_value=IOLoopKernelRestarter, diff --git a/jupyter_client/ioloop/restarter.py b/jupyter_client/ioloop/restarter.py index 6f531744c..69079eecf 100644 --- a/jupyter_client/ioloop/restarter.py +++ b/jupyter_client/ioloop/restarter.py @@ -4,37 +4,28 @@ restarts the kernel if it dies. """ -#----------------------------------------------------------------------------- -# Copyright (c) The Jupyter Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. from __future__ import absolute_import +import warnings from zmq.eventloop import ioloop - from jupyter_client.restarter import KernelRestarter from traitlets import ( Instance, ) -#----------------------------------------------------------------------------- -# Code -#----------------------------------------------------------------------------- - class IOLoopKernelRestarter(KernelRestarter): """Monitor and autorestart a kernel.""" - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return ioloop.IOLoop.instance() + warnings.warn("IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2", + DeprecationWarning, stacklevel=4, + ) + return ioloop.IOLoop.current() _pcallback = None @@ -42,7 +33,7 @@ def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( - self.poll, 1000*self.time_to_dead, self.loop + self.poll, 1000*self.time_to_dead, ) self._pcallback.start() diff --git a/jupyter_client/session.py b/jupyter_client/session.py index af60ac259..33b1c0b4a 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -191,9 +191,9 @@ def _context_default(self): session = Instance('jupyter_client.session.Session', allow_none=True) - loop = Instance('zmq.eventloop.ioloop.IOLoop') + loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): - return IOLoop.instance() + return IOLoop.current() def __init__(self, **kwargs): super(SessionFactory, self).__init__(**kwargs) From 172d6cdea80bf189a894171fdd39cc6031ae562d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 21 Sep 2017 15:04:12 +0200 Subject: [PATCH 16/89] Configure interrupt mode via spec. - interrupt_mode="signal" is the default and current behaviour - With interrupt_mode="message", instead of a signal, a `interrupt_request` message on the control port will be sent --- jupyter_client/kernelspec.py | 18 ++++++++++++------ jupyter_client/manager.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3465ac7a4..d2248cc58 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -13,7 +13,9 @@ pjoin = os.path.join from ipython_genutils.py3compat import PY3 -from traitlets import HasTraits, List, Unicode, Dict, Set, Bool, Type +from traitlets import ( + HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum +) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH @@ -28,6 +30,9 @@ class KernelSpec(HasTraits): language = Unicode() env = Dict() resource_dir = Unicode() + interrupt_mode = CaselessStrEnum( + ['message', 'signal'], default_value='signal' + ) metadata = Dict() @classmethod @@ -46,6 +51,7 @@ def to_dict(self): env=self.env, display_name=self.display_name, language=self.language, + interrupt_mode=self.interrupt_mode, metadata=self.metadata, ) @@ -227,7 +233,7 @@ def get_all_specs(self): def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. - + Returns the path that was deleted. """ save_native = self.ensure_native_kernel @@ -263,7 +269,7 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, If ``user`` is False, it will attempt to install into the systemwide kernel registry. If the process does not have appropriate permissions, an :exc:`OSError` will be raised. - + If ``prefix`` is given, the kernelspec will be installed to PREFIX/share/jupyter/kernels/KERNEL_NAME. This can be sys.prefix for installation inside virtual or conda envs. @@ -284,16 +290,16 @@ def install_kernel_spec(self, source_dir, kernel_name=None, user=False, DeprecationWarning, stacklevel=2, ) - + destination = self._get_destination_dir(kernel_name, user=user, prefix=prefix) self.log.debug('Installing kernelspec in %s', destination) - + kernel_dir = os.path.dirname(destination) if kernel_dir not in self.kernel_dirs: self.log.warning("Installing to %s, which is not in %s. The kernelspec may not be found.", kernel_dir, self.kernel_dirs, ) - + if os.path.isdir(destination): self.log.info('Removing existing kernelspec in %s', destination) shutil.rmtree(destination) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 4e0387762..2bcc1629a 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -246,7 +246,7 @@ def start_kernel(self, **kw): env.update(self.kernel_spec.env or {}) elif self.extra_env: env.update(self.extra_env) - + # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, @@ -403,11 +403,18 @@ def interrupt_kernel(self): platforms. """ if self.has_kernel: - if sys.platform == 'win32': - from .win_interrupt import send_interrupt - send_interrupt(self.kernel.win32_interrupt_event) - else: - self.signal_kernel(signal.SIGINT) + interrupt_mode = self.kernel_spec.interrupt_mode + if interrupt_mode == 'signal': + if sys.platform == 'win32': + from .win_interrupt import send_interrupt + send_interrupt(self.kernel.win32_interrupt_event) + else: + self.signal_kernel(signal.SIGINT) + + elif interrupt_mode == 'message': + msg = self.session.msg("interrupt_request", content={}) + self._connect_control_socket() + self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") From f0e33ba7532ab50ffd54fbc0912edc61815eba03 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 30 Oct 2017 16:30:12 +0100 Subject: [PATCH 17/89] Update docs. --- docs/kernels.rst | 7 +++++++ docs/messaging.rst | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/kernels.rst b/docs/kernels.rst index 2fe1500aa..76fa67699 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -132,6 +132,13 @@ JSON serialised dictionary containing the following keys and values: is found, a kernel with a matching `language` will be used. This allows a notebook written on any Python or Julia kernel to be properly associated with the user's Python or Julia kernel, even if they aren't listed under the same name as the author's. +- **interrupt_mode** (optional): May be either ``signal`` or ``message`` and + specifies how a client is supposed to interrupt cell execution on this kernel, + either by sending an interrupt ``signal`` via the operating system's + signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + ``interrupt_request`` message on the control channel (see + :ref:`msging_interrupt`). If this is not specified + the client will default to ``signal`` mode. - **env** (optional): A dictionary of environment variables to set for the kernel. These will be added to the current environment variables before the kernel is started. diff --git a/docs/messaging.rst b/docs/messaging.rst index 776dda681..ec8efd99f 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -959,6 +959,27 @@ Message type: ``shutdown_reply``:: socket, they simply send a forceful process termination signal, since a dead process is unlikely to respond in any useful way to messages. +.. _msging_interrupt: + +Kernel interrupt +---------------- + +In case a kernel can not catch operating system interrupt signals (e.g. the used +runtime handles signals and does not allow a user program to define a callback), +a kernel can choose to be notified using a message instead. For this to work, +the kernels kernelspec must set `interrupt_mode` to ``message``. An interruption +will then result in the following message on the `control` channel: + +Message type: ``interrupt_request``:: + + content = {} + +Message type: ``interrupt_reply``:: + + content = {} + +.. versionadded:: 5.3 + Messages on the IOPub (PUB/SUB) channel ======================================= From 21b95699dcb5917a8cf87c8aae3bd67b9e281f3c Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 6 Nov 2017 11:59:10 +0100 Subject: [PATCH 18/89] Bump protocol version. --- docs/messaging.rst | 2 +- jupyter_client/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/messaging.rst b/docs/messaging.rst index ec8efd99f..7c533a7de 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -21,7 +21,7 @@ Versioning The Jupyter message specification is versioned independently of the packages that use it. -The current version of the specification is 5.2. +The current version of the specification is 5.3. .. note:: *New in* and *Changed in* messages in this document refer to versions of the diff --git a/jupyter_client/_version.py b/jupyter_client/_version.py index 90dd2e93e..7f96345ae 100644 --- a/jupyter_client/_version.py +++ b/jupyter_client/_version.py @@ -1,5 +1,5 @@ version_info = (5, 1, 0) __version__ = '.'.join(map(str, version_info)) -protocol_version_info = (5, 2) +protocol_version_info = (5, 3) protocol_version = "%i.%i" % protocol_version_info From 6674afae21cce681c1fae6a37879d40c181cc91c Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 13 Nov 2017 14:31:20 +0100 Subject: [PATCH 19/89] disable pyzmq zero-copy optimizations during session tests --- jupyter_client/tests/test_session.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 43819a898..e80274367 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -8,6 +8,10 @@ import sys import uuid from datetime import datetime +try: + from unittest import mock +except ImportError: + import mock import pytest @@ -34,6 +38,14 @@ def setUp(self): self.session = ss.Session() +@pytest.fixture +def no_copy_threshold(): + """Disable zero-copy optimizations in pyzmq >= 17""" + with mock.patch.object(zmq, 'COPY_THRESHOLD', 1): + yield + + +@pytest.mark.usefixtures('no_copy_threshold') class TestSession(SessionTestCase): def test_msg(self): From e2772bd54c864b805b1cac36b3141fe27b1ba726 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 13 Nov 2017 15:11:37 +0100 Subject: [PATCH 20/89] Fix signal name. --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 76fa67699..5308c603f 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -135,7 +135,7 @@ JSON serialised dictionary containing the following keys and values: - **interrupt_mode** (optional): May be either ``signal`` or ``message`` and specifies how a client is supposed to interrupt cell execution on this kernel, either by sending an interrupt ``signal`` via the operating system's - signalling facilities (e.g. `SIGTERM` on POSIX systems), or by sending an + signalling facilities (e.g. `SIGINT` on POSIX systems), or by sending an ``interrupt_request`` message on the control channel (see :ref:`msging_interrupt`). If this is not specified the client will default to ``signal`` mode. From 948d653e86a3923f23ffd92117f104c8bbc234c3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 22 Nov 2017 13:01:11 +0100 Subject: [PATCH 21/89] extend special handling of sys.executable to pythonX[.Y] this should allow ipykernel's wheel-installed specs to specify `python3` or `python2` and prevent python2 kernels from launching with sys.executable if the Python version is 3. --- jupyter_client/manager.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 2bcc1629a..52bf8b781 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -11,11 +11,6 @@ import signal import sys import time -import warnings -try: - from queue import Empty # Py 3 -except ImportError: - from Queue import Empty # Py 2 import zmq @@ -29,7 +24,6 @@ kernelspec, ) from .connect import ConnectionFileMixin -from .session import Session from .managerabc import ( KernelManagerABC ) @@ -164,8 +158,10 @@ def format_kernel_cmd(self, extra_arguments=None): else: cmd = self.kernel_spec.argv + extra_arguments - if cmd and cmd[0] == 'python': - # executable is 'python', use sys.executable. + if cmd and cmd[0] in {'python', + 'python%i' % sys.version_info[0], + 'python%i.%i' % sys.version_info[:2]}: + # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without From 250178fe53dcf5c20098e29ea94951caa3aa371e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 10 Feb 2017 16:57:22 +0000 Subject: [PATCH 22/89] Add 'jupyter kernel' command A simple lead in to the 'kernel nanny' work, this adds a command so you can do: jupyter kernel --kernel python --- jupyter_client/kernelapp.py | 66 +++++++++++++++++++++++++++++++++++++ scripts/jupyter-kernel | 5 +++ setup.py | 1 + 3 files changed, 72 insertions(+) create mode 100644 jupyter_client/kernelapp.py create mode 100755 scripts/jupyter-kernel diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py new file mode 100644 index 000000000..4c1c99e3c --- /dev/null +++ b/jupyter_client/kernelapp.py @@ -0,0 +1,66 @@ +import os +import signal +import uuid + +from jupyter_core.application import JupyterApp +from tornado.ioloop import IOLoop +from traitlets import Unicode + +from . import __version__ +from .kernelspec import KernelSpecManager +from .manager import KernelManager + +class KernelApp(JupyterApp): + version = __version__ + description = "Run a kernel locally" + + classes = [KernelManager, KernelSpecManager] + + aliases = { + 'kernel': 'KernelApp.kernel_name', + 'ip': 'KernelManager.ip', + } + + kernel_name = Unicode( + help = 'The name of a kernel to start' + ).tag(config=True) + + def initialize(self, argv=None): + super(KernelApp, self).initialize(argv) + self.km = KernelManager(kernel_name=self.kernel_name, + config=self.config) + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() + + def setup_signals(self): + if os.name == 'nt': + return + + def shutdown_handler(signo, frame): + self.loop.add_callback_from_signal(self.shutdown, signo) + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, shutdown_handler) + + def shutdown(self, signo): + self.log.info('Shutting down on signal %d' % signo) + self.km.shutdown_kernel() + self.loop.stop() + + def log_connection_info(self): + cf = self.km.connection_file + self.log.info('Connection file: %s', cf) + self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + + def start(self): + self.log.info('Starting kernel %r', self.kernel_name) + try: + self.km.start_kernel() + self.log_connection_info() + self.setup_signals() + self.loop.start() + finally: + self.km.cleanup() + + +main = KernelApp.launch_instance diff --git a/scripts/jupyter-kernel b/scripts/jupyter-kernel new file mode 100755 index 000000000..31144d405 --- /dev/null +++ b/scripts/jupyter-kernel @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from jupyter_client.kernelapp import main + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index f042f00b3..022cbc56e 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def run(self): 'console_scripts': [ 'jupyter-kernelspec = jupyter_client.kernelspecapp:KernelSpecApp.launch_instance', 'jupyter-run = jupyter_client.runapp:RunApp.launch_instance', + 'jupyter-kernel = jupyter_client.kernelapp:main', ], 'jupyter_client.kernel_providers' : [ 'spec = jupyter_client.discovery:KernelSpecProvider', From 9359b338c90f8e259ac4b307d99ce20ca3b2cbf7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:23:18 +0000 Subject: [PATCH 23/89] Use native kernel by default --- jupyter_client/kernelapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 4c1c99e3c..071a0f3ed 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -7,7 +7,7 @@ from traitlets import Unicode from . import __version__ -from .kernelspec import KernelSpecManager +from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME from .manager import KernelManager class KernelApp(JupyterApp): @@ -21,7 +21,7 @@ class KernelApp(JupyterApp): 'ip': 'KernelManager.ip', } - kernel_name = Unicode( + kernel_name = Unicode(NATIVE_KERNEL_NAME, help = 'The name of a kernel to start' ).tag(config=True) From ae03ddde10c215a8df1efe4a29d5bfa91b1efdfa Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 10:32:40 +0000 Subject: [PATCH 24/89] More description --- jupyter_client/kernelapp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 071a0f3ed..799d85ee4 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -2,7 +2,7 @@ import signal import uuid -from jupyter_core.application import JupyterApp +from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode @@ -11,8 +11,10 @@ from .manager import KernelManager class KernelApp(JupyterApp): + """Launch a kernel by name in a local subprocess. + """ version = __version__ - description = "Run a kernel locally" + description = "Run a kernel locally in a subprocess" classes = [KernelManager, KernelSpecManager] @@ -20,9 +22,10 @@ class KernelApp(JupyterApp): 'kernel': 'KernelApp.kernel_name', 'ip': 'KernelManager.ip', } + flags = {'debug': base_flags['debug']} kernel_name = Unicode(NATIVE_KERNEL_NAME, - help = 'The name of a kernel to start' + help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): @@ -34,6 +37,7 @@ def initialize(self, argv=None): self.loop = IOLoop.current() def setup_signals(self): + """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" if os.name == 'nt': return From 7e6d16711c6f16782a497dc6dbf76911c334f46e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:22:38 +0000 Subject: [PATCH 25/89] Add test of 'jupyter kernel' --- jupyter_client/kernelapp.py | 11 +++++ jupyter_client/tests/test_kernelapp.py | 57 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 jupyter_client/tests/test_kernelapp.py diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index 799d85ee4..a2ab17812 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -35,6 +35,7 @@ def initialize(self, argv=None): cf_basename = 'kernel-%s.json' % uuid.uuid4() self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) self.loop = IOLoop.current() + self.loop.add_callback(self._record_started) def setup_signals(self): """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" @@ -56,6 +57,16 @@ def log_connection_info(self): self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) + def _record_started(self): + """For tests, create a file to indicate that we've started + + Do not rely on this except in our own tests! + """ + fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') + if fn is not None: + with open(fn, 'wb'): + pass + def start(self): self.log.info('Starting kernel %r', self.kernel_name) try: diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py new file mode 100644 index 000000000..b41a02bc6 --- /dev/null +++ b/jupyter_client/tests/test_kernelapp.py @@ -0,0 +1,57 @@ +from __future__ import division + +import os +import shutil +from subprocess import Popen, PIPE +import sys +from tempfile import mkdtemp +import time + +def _launch(extra_env): + env = os.environ.copy() + env.update(extra_env) + return Popen([sys.executable, '-c', + 'from jupyter_client.kernelapp import main; main()'], + env=env, stderr=PIPE) + +WAIT_TIME = 10 +POLL_FREQ = 10 + +def test_kernelapp_lifecycle(): + # Check that 'jupyter kernel' starts and terminates OK. + runtime_dir = mkdtemp() + startup_dir = mkdtemp() + started = os.path.join(startup_dir, 'started') + try: + p = _launch({'JUPYTER_RUNTIME_DIR': runtime_dir, + 'JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE': started, + }) + # Wait for start + for _ in range(WAIT_TIME * POLL_FREQ): + if os.path.isfile(started): + break + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("No started file created in {} seconds" + .format(WAIT_TIME)) + + # Connection file should be there by now + files = os.listdir(runtime_dir) + assert len(files) == 1 + cf = files[0] + assert cf.startswith('kernel') + assert cf.endswith('.json') + + # Read the first three lines from stderr. This will hang if there are + # fewer lines to read; I don't see any way to avoid that without lots + # of extra complexity. + b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') + assert cf in b + + # Send SIGTERM to shut down + p.terminate() + p.wait(timeout=10) + finally: + shutil.rmtree(runtime_dir) + shutil.rmtree(startup_dir) + From 28f908f0da34ed0e0c85f58016334c434c18bb5f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:32:35 +0000 Subject: [PATCH 26/89] Workaround lack of timeout on Py2 --- jupyter_client/tests/test_kernelapp.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/jupyter_client/tests/test_kernelapp.py b/jupyter_client/tests/test_kernelapp.py index b41a02bc6..2533472d4 100644 --- a/jupyter_client/tests/test_kernelapp.py +++ b/jupyter_client/tests/test_kernelapp.py @@ -7,16 +7,28 @@ from tempfile import mkdtemp import time +PY3 = sys.version_info[0] >= 3 + def _launch(extra_env): env = os.environ.copy() env.update(extra_env) return Popen([sys.executable, '-c', 'from jupyter_client.kernelapp import main; main()'], - env=env, stderr=PIPE) + env=env, stderr=(PIPE if PY3 else None)) WAIT_TIME = 10 POLL_FREQ = 10 +def hacky_wait(p): + """Python 2 subprocess doesn't have timeouts :-(""" + for _ in range(WAIT_TIME * POLL_FREQ): + if p.poll() is not None: + return p.returncode + time.sleep(1 / POLL_FREQ) + else: + raise AssertionError("Process didn't exit in {} seconds" + .format(WAIT_TIME)) + def test_kernelapp_lifecycle(): # Check that 'jupyter kernel' starts and terminates OK. runtime_dir = mkdtemp() @@ -42,15 +54,13 @@ def test_kernelapp_lifecycle(): assert cf.startswith('kernel') assert cf.endswith('.json') - # Read the first three lines from stderr. This will hang if there are - # fewer lines to read; I don't see any way to avoid that without lots - # of extra complexity. - b = b''.join(p.stderr.readline() for _ in range(2)).decode('utf-8', 'replace') - assert cf in b - # Send SIGTERM to shut down p.terminate() - p.wait(timeout=10) + if PY3: + _, stderr = p.communicate(timeout=WAIT_TIME) + assert cf in stderr.decode('utf-8', 'replace') + else: + hacky_wait(p) finally: shutil.rmtree(runtime_dir) shutil.rmtree(startup_dir) From aa8b184c9c8134cae731c4652a87a704b6ee9f65 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:34:19 +0000 Subject: [PATCH 27/89] Restrict to older pytest on Python 3.3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 022cbc56e..1230f2142 100644 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def run(self): ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test:python_version == "3.3"': ['pytest<3.3.0'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From 5291f940c8cac341ed96c6b2dd73bbdd11db1df5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 11 Dec 2017 11:36:43 +0000 Subject: [PATCH 28/89] Another go at fixing pytest dependency on Python 3.3 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1230f2142..233f83a0e 100644 --- a/setup.py +++ b/setup.py @@ -85,8 +85,9 @@ def run(self): 'entrypoints', ], extras_require = { - 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], + 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], + 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From ed051077267f02bf630c32cce20406a52240474c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 11:57:57 +0000 Subject: [PATCH 29/89] Tolerate invalid kernel specs in get_all_specs() --- jupyter_client/kernelspec.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index d2248cc58..132d8f9c2 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -226,10 +226,17 @@ def get_all_specs(self): } """ d = self.find_kernel_specs() - return {kname: { - "resource_dir": d[kname], - "spec": self._get_kernel_spec_by_name(kname, d[kname]).to_dict() - } for kname in d} + res = {} + for kname, resource_dir in d.items(): + try: + spec = self._get_kernel_spec_by_name(kname, resource_dir) + res[kname] = { + "resource_dir": resource_dir, + "spec": spec.to_dict() + } + except Exception: + self.log.warning("Error loading kernelspec %r", kname, exc_info=True) + return res def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. From dd4a2d652de2ea898ae6213c148b59c60579afba Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 12:23:26 +0000 Subject: [PATCH 30/89] Improve performance of get_kernel_spec --- jupyter_client/kernelspec.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index d2248cc58..1a815970b 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -3,6 +3,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import errno import io import json import os @@ -199,15 +200,39 @@ def _get_kernel_spec_by_name(self, kernel_name, resource_dir): return self.kernel_spec_class.from_resource_dir(resource_dir) + def _find_spec_directory(self, kernel_name): + """Find the resource directory of a named kernel spec""" + for kernel_dir in self.kernel_dirs: + try: + files = os.listdir(kernel_dir) + except OSError as e: + if e.errno in (errno.ENOTDIR, errno.ENOENT): + continue + raise + for f in files: + path = pjoin(kernel_dir, f) + if f.lower() == kernel_name and _is_kernel_dir(path): + return path + + if kernel_name == NATIVE_KERNEL_NAME: + try: + from ipykernel.kernelspec import RESOURCES + except ImportError: + pass + else: + return RESOURCES + def get_kernel_spec(self, kernel_name): """Returns a :class:`KernelSpec` instance for the given kernel_name. Raises :exc:`NoSuchKernel` if the given kernel name is not found. """ - d = self.find_kernel_specs() - try: - resource_dir = d[kernel_name.lower()] - except KeyError: + if not _is_valid_kernel_name(kernel_name): + self.log.warning("Kernelspec name %r is invalid: %s", kernel_name, + _kernel_name_description) + + resource_dir = self._find_spec_directory(kernel_name.lower()) + if resource_dir is None: raise NoSuchKernel(kernel_name) return self._get_kernel_spec_by_name(kernel_name, resource_dir) From 22092fa65ee0fd007bf59bfc7998b7d57743f0d2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 15 Dec 2017 14:21:41 +0100 Subject: [PATCH 31/89] kill process group when killing kernel if killpg is available this should cleanup process trees (e.g. multiprocessing subprocesses) and make EADDRINUSE less likely during restart. --- jupyter_client/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 52bf8b781..f488d5c42 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -372,7 +372,10 @@ def _kill_kernel(self): # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: - self.kernel.kill() + if hasattr(signal, 'SIGKILL'): + self.signal_kernel(signal.SIGKILL) + else: + self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. From 5f076b7321abd41f0ecb20080bba02c3e63f2b66 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 14:09:39 +0000 Subject: [PATCH 32/89] Start writing release notes for 5.2 --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 +++ docs/environment.yml | 2 ++ 3 files changed, 41 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 35e21b5c6..1f44e346f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,42 @@ Changes in Jupyter Client ========================= +5.2 +=== + +`5.2 on GitHub `__ + +- Define Jupyter protocol version 5.3: + + - Kernels can now opt to be interrupted by a message sent on the control channel + instead of a system signal. See :ref:`kernelspecs` and :ref:`msging_interrupt` + (:ghpull:`294`). + +- New ``jupyter kernel`` command to launch an installed kernel by name + (:ghpull:`240`). +- Kernelspecs where the command starts with e.g. ``python3`` or + ``python3.6``—matching the version ``jupyter_client`` is running on—are now + launched with the same Python executable as the launching process (:ghpull:`306`). + This extends the special handling of ``python`` added in 5.0. +- Command line arguments specified by a kernelspec can now include + ``{resource_dir}``, which will be substituted with the kernelspec resource + directory path when the kernel is launched (:ghpull:`289`). +- Kernelspecs now have an optional ``metadata`` field to hold arbitrary metadata + about kernels—see :ref:`kernelspecs` (:ghpull:`274`). +- Make the ``KernelRestarter`` class used by a ``KernelManager`` configurable + (:ghpull:`290`). +- If a kernel dies soon after starting, reassign random ports before restarting + it, in case one of the previously chosen ports has been bound by another + process (:ghpull:`279`). +- Check for non-contiguous buffers before trying to send them through ZMQ + (:ghpull:`258`). +- Compatibility with upcoming Tornado version 5.0 (:ghpull:`304`). +- Simplify setup code by always using setuptools (:ghpull:`284`). +- Soften warnings when setting the sticky bit on runtime files fails + (:ghpull:`286`). +- Various corrections and improvements to documentation. + + 5.1 === diff --git a/docs/conf.py b/docs/conf.py index 849d7a56e..c3de08efd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', + 'sphinxcontrib_github_alt', ] # Add any paths that contain templates here, relative to this directory. @@ -55,6 +56,8 @@ copyright = '2015, Jupyter Development Team' author = 'Jupyter Development Team' +github_project_url = "https://github.com/jupyter/jupyter_client" + # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. diff --git a/docs/environment.yml b/docs/environment.yml index 3690c73b7..459e7ab3b 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -8,3 +8,5 @@ dependencies: - jupyter_core - sphinx>=1.3.6 - sphinx_rtd_theme +- pip: + - sphinxcontrib_github_alt From 6689764905a8916e1c553e943e82c9d6870f2ae6 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 15 Dec 2017 14:25:36 +0000 Subject: [PATCH 33/89] Add PR #314 to changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f44e346f..a78c79986 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ Changes in Jupyter Client about kernels—see :ref:`kernelspecs` (:ghpull:`274`). - Make the ``KernelRestarter`` class used by a ``KernelManager`` configurable (:ghpull:`290`). +- When killing a kernel on Unix, kill its process group (:ghpull:`314`). - If a kernel dies soon after starting, reassign random ports before restarting it, in case one of the previously chosen ports has been bound by another process (:ghpull:`279`). From 6251cf68e58ce89c3751ec7e356e86ee350f3652 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 16 Dec 2017 21:14:52 +0000 Subject: [PATCH 34/89] Add PRs #310 and #311 to changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a78c79986..23cc0c953 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,6 +32,11 @@ Changes in Jupyter Client - If a kernel dies soon after starting, reassign random ports before restarting it, in case one of the previously chosen ports has been bound by another process (:ghpull:`279`). +- Avoid unnecessary filesystem operations when finding a kernelspec with + :meth:`.KernelSpecManager.get_kernel_spec` (:ghpull:`311`). +- :meth:`.KernelSpecManager.get_all_specs` will no longer raise an exception on + encountering an invalid ``kernel.json`` file. It will raise a warning and + continue (:ghpull:`310`). - Check for non-contiguous buffers before trying to send them through ZMQ (:ghpull:`258`). - Compatibility with upcoming Tornado version 5.0 (:ghpull:`304`). From a8b474512ff10120f46bea370c09bd6526bb14fb Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 19 Dec 2017 15:52:50 +0100 Subject: [PATCH 35/89] require tornado --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 233f83a0e..5de9cfd18 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ def run(self): 'pyzmq>=13', 'python-dateutil>=2.1', 'entrypoints', + 'tornado>=4.1', ], extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], From 072a08727927ed0ac5c0b2cca0ccaa364109c060 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 30 Dec 2017 16:54:12 +0100 Subject: [PATCH 36/89] Parenthesize conditional requirement in setup.py Du to a likely bug in wheel, the conditional dependency on pytest ends up being unconditional. Seem like adding parenthesis fix that (as a work around). See https://github.com/pypa/setuptools/issues/1242 Closes #324 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 23cc0c953..d560cbfda 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changes in Jupyter Client ========================= +5.2.1 +===== + +- Add parenthesis to conditional pytest requirement to work around a bug in the + ``wheel`` package, that generate a ``.whl`` which otherwise always depends on + ``pytest`` see :ghissue:`324` and :ghpull:`325` + 5.2 === diff --git a/setup.py b/setup.py index 5de9cfd18..c184b40fa 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ def run(self): extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], - 'test:python_version >= "3.4" or python_version == "2.7"': ['pytest'], + 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From e426a64ee0e0409201fc6817b0c01f3f5b2d3602 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 3 Jan 2018 14:25:48 +0100 Subject: [PATCH 37/89] Exclude build docs from sdist. This shrinks the sdist from 2MB to ~250KB... just realized that after uploading 5.2.1 took way too long. Apparently 5.2.0 alsho shipped built docs. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 42edd273d..994648d70 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include README.md # Documentation graft docs exclude docs/\#* +exclude docs/_* # Examples graft examples From 41c5954a8257634b2d7f8905ed8c874946de606c Mon Sep 17 00:00:00 2001 From: stonebig Date: Sat, 6 Jan 2018 16:29:04 +0100 Subject: [PATCH 38/89] more complete error message to help inqiure on this https://github.com/jupyter/jupyter_client/issues/329 --- jupyter_client/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index f488d5c42..21f6ca925 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -219,9 +219,10 @@ def start_kernel(self, **kw): """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " + "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " - "Currently valid addresses are: %s" % local_ips() + "Currently valid addresses are: %s" % (self.ip, local_ips()) ) # write connection file / get default ports From c658076da5c76455429e58b7d0a7510d3fa8d5be Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Sat, 6 Jan 2018 19:03:11 +0100 Subject: [PATCH 39/89] Tell Travis not to test the push from MrMeeseeks Use the ability to exclude branches as describe there: - https://docs.travis-ci.com/user/customizing-the-build/#Safelisting-or-blocklisting-branches Relatively easy as MrMeeseeks push a known branch format. This of course cannot be tested until merged and backported, and another backport triggered. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0a3a96915..faec1b44c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,6 @@ after_success: matrix: allow_failures: - python: nightly +branches: + except: + - /^auto-backport-of-pr-[0-9]+$/ From 6b03b81ede614d5b99c1132f1cc7bafae69b8562 Mon Sep 17 00:00:00 2001 From: Dawid Manikowski Date: Tue, 9 Jan 2018 19:45:29 +0100 Subject: [PATCH 40/89] Migrate SSH tunneling from pyzmq As this part of code is mainly used in IPython we agreed to move it from pyzmq. It will be easier to maintain here and some planned changes in this code will be easier to apply and release. --- jupyter_client/connect.py | 24 +- jupyter_client/ssh/__init__.py | 1 + jupyter_client/ssh/forward.py | 92 ++++++++ jupyter_client/ssh/tunnel.py | 375 +++++++++++++++++++++++++++++++ jupyter_client/tests/test_ssh.py | 8 + 5 files changed, 488 insertions(+), 12 deletions(-) create mode 100644 jupyter_client/ssh/__init__.py create mode 100644 jupyter_client/ssh/forward.py create mode 100644 jupyter_client/ssh/tunnel.py create mode 100644 jupyter_client/tests/test_ssh.py diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 91efbc461..8e0dcd267 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -35,9 +35,9 @@ def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, - control_port=0, ip='', key=b'', transport='tcp', - signature_scheme='hmac-sha256', kernel_name='' - ): + control_port=0, ip='', key=b'', transport='tcp', + signature_scheme='hmac-sha256', kernel_name='' + ): """Generates a JSON config file, including the selection of random ports. Parameters @@ -193,7 +193,7 @@ def find_connection_file(filename='kernel-*.json', path=None, profile=None): path = ['.', jupyter_runtime_dir()] if isinstance(path, string_types): path = [path] - + try: # first, try explicit name return filefind(filename, path) @@ -208,11 +208,11 @@ def find_connection_file(filename='kernel-*.json', path=None, profile=None): else: # accept any substring match pat = '*%s*' % filename - + matches = [] for p in path: matches.extend(glob.glob(os.path.join(p, pat))) - + matches = [ os.path.abspath(m) for m in matches ] if not matches: raise IOError("Could not find %r in %r" % (filename, path)) @@ -249,7 +249,7 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ - from zmq.ssh import tunnel + from jupyter_core.ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: @@ -289,11 +289,11 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): class ConnectionFileMixin(LoggingConfigurable): """Mixin for configurable classes that work with connection files""" - + data_dir = Unicode() def _data_dir_default(self): return jupyter_data_dir() - + # The addresses for the communication channels connection_file = Unicode('', config=True, help="""JSON file in which to store connection info [default: kernel-.json] @@ -480,7 +480,7 @@ def write_connection_file(self): def load_connection_file(self, connection_file=None): """Load connection info from JSON dict in self.connection_file. - + Parameters ---------- connection_file: unicode, optional @@ -496,10 +496,10 @@ def load_connection_file(self, connection_file=None): def load_connection_info(self, info): """Load connection info from a dict containing connection info. - + Typically this data comes from a connection file and is called by load_connection_file. - + Parameters ---------- info: dict diff --git a/jupyter_client/ssh/__init__.py b/jupyter_client/ssh/__init__.py new file mode 100644 index 000000000..d7bc9d566 --- /dev/null +++ b/jupyter_client/ssh/__init__.py @@ -0,0 +1 @@ +from jupyter_client.ssh.tunnel import * diff --git a/jupyter_client/ssh/forward.py b/jupyter_client/ssh/forward.py new file mode 100644 index 000000000..a44c11769 --- /dev/null +++ b/jupyter_client/ssh/forward.py @@ -0,0 +1,92 @@ +# +# This file is adapted from a paramiko demo, and thus licensed under LGPL 2.1. +# Original Copyright (C) 2003-2007 Robey Pointer +# Edits Copyright (C) 2010 The IPython Team +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA. + +""" +Sample script showing how to do local port forwarding over paramiko. + +This script connects to the requested SSH server and sets up local port +forwarding (the openssh -L option) from a local port through a tunneled +connection to a destination reachable from the SSH server machine. +""" + +from __future__ import print_function + +import logging +import select +try: # Python 3 + import socketserver +except ImportError: # Python 2 + import SocketServer as socketserver + +logger = logging.getLogger('ssh') + + +class ForwardServer (socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +class Handler (socketserver.BaseRequestHandler): + + def handle(self): + try: + chan = self.ssh_transport.open_channel('direct-tcpip', + (self.chain_host, self.chain_port), + self.request.getpeername()) + except Exception as e: + logger.debug('Incoming request to %s:%d failed: %s' % (self.chain_host, + self.chain_port, + repr(e))) + return + if chan is None: + logger.debug('Incoming request to %s:%d was rejected by the SSH server.' % + (self.chain_host, self.chain_port)) + return + + logger.debug('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), + chan.getpeername(), (self.chain_host, self.chain_port))) + while True: + r, w, x = select.select([self.request, chan], [], []) + if self.request in r: + data = self.request.recv(1024) + if len(data) == 0: + break + chan.send(data) + if chan in r: + data = chan.recv(1024) + if len(data) == 0: + break + self.request.send(data) + chan.close() + self.request.close() + logger.debug('Tunnel closed ') + + +def forward_tunnel(local_port, remote_host, remote_port, transport): + # this is a little convoluted, but lets me configure things for the Handler + # object. (SocketServer doesn't give Handlers any way to access the outer + # server normally.) + class SubHander (Handler): + chain_host = remote_host + chain_port = remote_port + ssh_transport = transport + ForwardServer(('127.0.0.1', local_port), SubHander).serve_forever() + + +__all__ = ['forward_tunnel'] diff --git a/jupyter_client/ssh/tunnel.py b/jupyter_client/ssh/tunnel.py new file mode 100644 index 000000000..e1cd08027 --- /dev/null +++ b/jupyter_client/ssh/tunnel.py @@ -0,0 +1,375 @@ +"""Basic ssh tunnel utilities, and convenience functions for tunneling +zeromq connections. +""" + +# Copyright (C) 2010-2011 IPython Development Team +# Copyright (C) 2011- PyZMQ Developers +# +# Redistributed from IPython under the terms of the BSD License. + + +from __future__ import print_function + +import atexit +import os +import re +import signal +import socket +import sys +import warnings +from getpass import getpass, getuser +from multiprocessing import Process + +try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import paramiko + SSHException = paramiko.ssh_exception.SSHException +except ImportError: + paramiko = None + class SSHException(Exception): + pass +else: + from .forward import forward_tunnel + +try: + import pexpect +except ImportError: + pexpect = None + +from zmq.utils.strtypes import b + + +def select_random_ports(n): + """Select and return n random ports that are available.""" + ports = [] + sockets = [] + for i in range(n): + sock = socket.socket() + sock.bind(('', 0)) + ports.append(sock.getsockname()[1]) + sockets.append(sock) + for sock in sockets: + sock.close() + return ports + + +#----------------------------------------------------------------------------- +# Check for passwordless login +#----------------------------------------------------------------------------- +_password_pat = re.compile(b(r'pass(word|phrase):'), re.IGNORECASE) + + +def try_passwordless_ssh(server, keyfile, paramiko=None): + """Attempt to make an ssh connection without a password. + This is mainly used for requiring password input only once + when many tunnels may be connected to the same server. + + If paramiko is None, the default for the platform is chosen. + """ + if paramiko is None: + paramiko = sys.platform == 'win32' + if not paramiko: + f = _try_passwordless_openssh + else: + f = _try_passwordless_paramiko + return f(server, keyfile) + + +def _try_passwordless_openssh(server, keyfile): + """Try passwordless login with shell ssh command.""" + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko") + cmd = 'ssh -f ' + server + if keyfile: + cmd += ' -i ' + keyfile + cmd += ' exit' + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + p = pexpect.spawn(cmd, env=env) + while True: + try: + i = p.expect([ssh_newkey, _password_pat], timeout=.1) + if i == 0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + return True + else: + return False + + +def _try_passwordless_paramiko(server, keyfile): + """Try passwordless login with paramiko.""" + if paramiko is None: + msg = "Paramiko unavailable, " + if sys.platform == 'win32': + msg += "Paramiko is required for ssh tunneled connections on Windows." + else: + msg += "use OpenSSH." + raise ImportError(msg) + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True) + except paramiko.AuthenticationException: + return False + else: + client.close() + return True + + +def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Connect a socket to an address via an ssh tunnel. + + This is a wrapper for socket.connect(addr), when addr is not accessible + from the local machine. It simply creates an ssh tunnel using the remaining args, + and calls socket.connect('tcp://localhost:lport') where lport is the randomly + selected local port of the tunnel. + + """ + new_url, tunnel = open_tunnel(addr, server, keyfile=keyfile, password=password, paramiko=paramiko, timeout=timeout) + socket.connect(new_url) + return tunnel + + +def open_tunnel(addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Open a tunneled connection from a 0MQ url. + + For use inside tunnel_connection. + + Returns + ------- + + (url, tunnel) : (str, object) + The 0MQ url that has been forwarded, and the tunnel object + """ + + lport = select_random_ports(1)[0] + transport, addr = addr.split('://') + ip, rport = addr.split(':') + rport = int(rport) + if paramiko is None: + paramiko = sys.platform == 'win32' + if paramiko: + tunnelf = paramiko_tunnel + else: + tunnelf = openssh_tunnel + + tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password, timeout=timeout) + return 'tcp://127.0.0.1:%i' % lport, tunnel + + +def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """Create an ssh tunnel using command-line ssh that connects port lport + on this machine to localhost:rport on server. The tunnel + will automatically close when not in use, remaining open + for a minimum of timeout seconds for an initial connection. + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + keyfile and password may be specified, but ssh config is checked for defaults. + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + """ + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko_tunnel") + ssh = "ssh " + if keyfile: + ssh += "-i " + keyfile + + if ':' in server: + server, port = server.split(':') + ssh += " -p %s" % port + + cmd = "%s -O check %s" % (ssh, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + pid = int(output[output.find(b"(pid=")+5:output.find(b")")]) + cmd = "%s -O forward -L 127.0.0.1:%i:%s:%i %s" % ( + ssh, lport, remoteip, rport, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + atexit.register(_stop_tunnel, cmd.replace("-O forward", "-O cancel", 1)) + return pid + cmd = "%s -f -S none -L 127.0.0.1:%i:%s:%i %s sleep %i" % ( + ssh, lport, remoteip, rport, server, timeout) + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + tunnel = pexpect.spawn(cmd, env=env) + failed = False + while True: + try: + i = tunnel.expect([ssh_newkey, _password_pat], timeout=.1) + if i == 0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + if tunnel.exitstatus: + print(tunnel.exitstatus) + print(tunnel.before) + print(tunnel.after) + raise RuntimeError("tunnel '%s' failed to start" % (cmd)) + else: + return tunnel.pid + else: + if failed: + print("Password rejected, try again") + password = None + if password is None: + password = getpass("%s's password: " % (server)) + tunnel.sendline(password) + failed = True + + +def _stop_tunnel(cmd): + pexpect.run(cmd) + + +def _split_server(server): + if '@' in server: + username, server = server.split('@', 1) + else: + username = getuser() + if ':' in server: + server, port = server.split(':') + port = int(port) + else: + port = 22 + return username, server, port + + +def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """launch a tunner with paramiko in a subprocess. This should only be used + when shell ssh is unavailable (e.g. Windows). + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + If you are familiar with ssh tunnels, this creates the tunnel: + + ssh server -L localhost:lport:remoteip:rport + + keyfile and password may be specified, but ssh config is checked for defaults. + + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + + """ + if paramiko is None: + raise ImportError("Paramiko not available") + + if password is None: + if not _try_passwordless_paramiko(server, keyfile): + password = getpass("%s's password: " % (server)) + + p = Process(target=_paramiko_tunnel, + args=(lport, rport, server, remoteip), + kwargs=dict(keyfile=keyfile, password=password)) + p.daemon = True + p.start() + return p + + +def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None): + """Function for actually starting a paramiko tunnel, to be passed + to multiprocessing.Process(target=this), and not called directly. + """ + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True, password=password) +# except paramiko.AuthenticationException: +# if password is None: +# password = getpass("%s@%s's password: "%(username, server)) +# client.connect(server, port, username=username, password=password) +# else: +# raise + except Exception as e: + print('*** Failed to connect to %s:%d: %r' % (server, port, e)) + sys.exit(1) + + # Don't let SIGINT kill the tunnel subprocess + signal.signal(signal.SIGINT, signal.SIG_IGN) + + try: + forward_tunnel(lport, remoteip, rport, client.get_transport()) + except KeyboardInterrupt: + print('SIGINT: Port forwarding stopped cleanly') + sys.exit(0) + except Exception as e: + print("Port forwarding stopped uncleanly: %s" % e) + sys.exit(255) + + +if sys.platform == 'win32': + ssh_tunnel = paramiko_tunnel +else: + ssh_tunnel = openssh_tunnel + + +__all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh'] diff --git a/jupyter_client/tests/test_ssh.py b/jupyter_client/tests/test_ssh.py new file mode 100644 index 000000000..e1673f9f4 --- /dev/null +++ b/jupyter_client/tests/test_ssh.py @@ -0,0 +1,8 @@ +from jupyter_client.ssh.tunnel import select_random_ports + +def test_random_ports(): + for i in range(4096): + ports = select_random_ports(10) + assert len(ports) == 10 + for p in ports: + assert ports.count(p) == 1 From cd735300df4df8d82a1abd0a27e8e56beaeb03af Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 Jan 2018 09:51:20 -0800 Subject: [PATCH 41/89] handle classes having been torn down in atexit we could probably avoid this if we registered/unregistered atexit callbacks for instances instead of registering it once for classes at import time --- jupyter_client/channels.py | 5 ++++- jupyter_client/threaded.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index dd9906723..64e565189 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -80,7 +80,10 @@ def __init__(self, context=None, session=None, address=None): @staticmethod @atexit.register def _notice_exit(): - HBChannel._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if HBChannel is not None: + HBChannel._exiting = True def _create_socket(self): if self.socket is not None: diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index f437aa58b..fda3a084c 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -151,7 +151,10 @@ def __init__(self, loop): @staticmethod @atexit.register def _notice_exit(): - IOLoopThread._exiting = True + # Class definitions can be torn down during interpreter shutdown. + # We only need to set _exiting flag if this hasn't happened. + if IOLoopThread is not None: + IOLoopThread._exiting = True def run(self): """Run my loop, ignoring EINTR events in the poller""" From ca73a8741fafc46612bc60d9d3937e721986d982 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jan 2018 19:20:09 +0100 Subject: [PATCH 42/89] avoid calling private method in subclasses of KernelSpecManager on the result of a public overrideable method, which breaks subclasses that don't override get_all_specs --- jupyter_client/kernelspec.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 3e05d292d..78a5b564c 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -254,7 +254,14 @@ def get_all_specs(self): res = {} for kname, resource_dir in d.items(): try: - spec = self._get_kernel_spec_by_name(kname, resource_dir) + if self.__class__ is KernelSpecManager: + spec = self._get_kernel_spec_by_name(kname, resource_dir) + else: + # avoid calling private methods in subclasses, + # which may have overridden find_kernel_specs + # and get_kernel_spec, but not the newer get_all_specs + spec = self.get_kernel_spec(kname) + res[kname] = { "resource_dir": resource_dir, "spec": spec.to_dict() From 7dfa6c45031d7f6944f3cba76fefb3b8e5beec02 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 23 Jan 2018 19:36:44 +0100 Subject: [PATCH 43/89] test that KernelSpecManager subclasses work if they don't implement get_all_specs --- jupyter_client/tests/test_kernelspec.py | 35 +++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/jupyter_client/tests/test_kernelspec.py b/jupyter_client/tests/test_kernelspec.py index b2ec4195c..2919923ef 100644 --- a/jupyter_client/tests/test_kernelspec.py +++ b/jupyter_client/tests/test_kernelspec.py @@ -4,6 +4,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import copy import io import json from logging import StreamHandler @@ -11,6 +12,7 @@ from os.path import join as pjoin from subprocess import Popen, PIPE, STDOUT import sys +import tempfile import unittest import pytest @@ -156,7 +158,7 @@ def test_validate_kernel_name(self): 'Haskell-1-2-3', ]: assert kernelspec._is_valid_kernel_name(good) - + for bad in [ 'has space', u'ünicode', @@ -165,4 +167,33 @@ def test_validate_kernel_name(self): ]: assert not kernelspec._is_valid_kernel_name(bad) - + def test_subclass(self): + """Test get_all_specs in subclasses that override find_kernel_specs""" + ksm = self.ksm + resource_dir = tempfile.gettempdir() + native_name = kernelspec.NATIVE_KERNEL_NAME + native_kernel = ksm.get_kernel_spec(native_name) + + class MyKSM(kernelspec.KernelSpecManager): + def get_kernel_spec(self, name): + spec = copy.copy(native_kernel) + if name == 'fake': + spec.name = name + spec.resource_dir = resource_dir + elif name == native_name: + pass + else: + raise KeyError(name) + return spec + + def find_kernel_specs(self): + return { + 'fake': resource_dir, + native_name: native_kernel.resource_dir, + } + + # ensure that get_all_specs doesn't raise if only + # find_kernel_specs and get_kernel_spec are defined + myksm = MyKSM() + specs = myksm.get_all_specs() + assert sorted(specs) == ['fake', native_name] From 43d329bbe8ffb474c87d7d9e9c4c2973e8fe5025 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 24 Jan 2018 10:29:41 +0100 Subject: [PATCH 44/89] changelog for 5.2.2 --- docs/changelog.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d560cbfda..101d3f7f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,12 +4,28 @@ Changes in Jupyter Client ========================= +5.2.2 +===== + +`5.2.2 on GitHub `__ + +- Fix :meth:`.KernelSpecManager.get_all_specs` method in subclasses + that only override :meth:`.KernelSpecManager.find_kernel_specs` + and :meth:`.KernelSpecManager.get_kernel_spec`. + See :ghissue:`338` and :ghpull:`339`. +- Eliminate occasional error messages during process exit (:ghpull:`336`). +- Improve error message when attempting to bind on invalid address (:ghpull:`330`). +- Add missing direct dependency on tornado (:ghpull:`323`). + + 5.2.1 ===== +`5.2.1 on GitHub `__ + - Add parenthesis to conditional pytest requirement to work around a bug in the ``wheel`` package, that generate a ``.whl`` which otherwise always depends on - ``pytest`` see :ghissue:`324` and :ghpull:`325` + ``pytest`` see :ghissue:`324` and :ghpull:`325`. 5.2 === From f05abf7666c4ec5d34b385bf483fc0b20d67f22c Mon Sep 17 00:00:00 2001 From: Dawid Manikowski Date: Thu, 25 Jan 2018 10:34:46 +0100 Subject: [PATCH 45/89] Convert to relative import --- jupyter_client/connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/connect.py b/jupyter_client/connect.py index 8e0dcd267..2cc89de35 100644 --- a/jupyter_client/connect.py +++ b/jupyter_client/connect.py @@ -249,7 +249,7 @@ def tunnel_to_kernel(connection_info, sshserver, sshkey=None): (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ - from jupyter_core.ssh import tunnel + from .ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: From 2840d26e329b1ce6b752943cd2128469a8a481c6 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 31 Jan 2018 16:51:41 +0100 Subject: [PATCH 46/89] fix testing patch for pyzmq < 17 zmq.COPY_THRESHOLD is undefined prior to pyzmq 17, so we need `create=True` to define it in that case --- jupyter_client/tests/test_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index e80274367..55181e91b 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -41,7 +41,7 @@ def setUp(self): @pytest.fixture def no_copy_threshold(): """Disable zero-copy optimizations in pyzmq >= 17""" - with mock.patch.object(zmq, 'COPY_THRESHOLD', 1): + with mock.patch.object(zmq, 'COPY_THRESHOLD', 1, create=True): yield From 7b45731a43abcbce30b6e36f33624b62f191573a Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 8 Mar 2018 17:51:09 +0100 Subject: [PATCH 47/89] ThreadedClient: schedule IOLoop.stop in IOLoop thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calling stop doesn’t wake the IOLoop with asyncio (tornado 5) --- jupyter_client/threaded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index fda3a084c..a49f4d0b1 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -182,7 +182,7 @@ def stop(self): :meth:`~threading.Thread.start` is called again. """ if self.ioloop is not None: - self.ioloop.stop() + self.ioloop.add_callback(self.ioloop.stop) self.join() self.close() From 64aca4d4bcc769dd3af8cf5c59bcf6edbecd56b8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 8 Mar 2018 17:52:42 +0100 Subject: [PATCH 48/89] threadsafety in IOLoopThread - avoid instantiating an IOLoop outside the thread in which it will be used, which sometimes causes problems. - ensure asyncio eventloop is defined in the thread, if asyncio might be in use --- jupyter_client/threaded.py | 47 ++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/jupyter_client/threaded.py b/jupyter_client/threaded.py index a49f4d0b1..83a6ad0eb 100644 --- a/jupyter_client/threaded.py +++ b/jupyter_client/threaded.py @@ -3,7 +3,8 @@ from __future__ import absolute_import import atexit import errno -from threading import Thread +import sys +from threading import Thread, Event import time # import ZMQError in top-level namespace, to avoid ugly attribute-error messages @@ -41,9 +42,15 @@ def __init__(self, socket, session, loop): self.socket = socket self.session = session self.ioloop = loop + evt = Event() - self.stream = zmqstream.ZMQStream(self.socket, self.ioloop) - self.stream.on_recv(self._handle_recv) + def setup_stream(): + self.stream = zmqstream.ZMQStream(self.socket, self.ioloop) + self.stream.on_recv(self._handle_recv) + evt.set() + + self.ioloop.add_callback(setup_stream) + evt.wait() _is_alive = False def is_alive(self): @@ -142,11 +149,11 @@ class IOLoopThread(Thread): """Run a pyzmq ioloop in a thread to send and receive messages """ _exiting = False + ioloop = None - def __init__(self, loop): + def __init__(self): super(IOLoopThread, self).__init__() self.daemon = True - self.ioloop = loop or ioloop.IOLoop() @staticmethod @atexit.register @@ -156,8 +163,26 @@ def _notice_exit(): if IOLoopThread is not None: IOLoopThread._exiting = True + def start(self): + """Start the IOLoop thread + + Don't return until self.ioloop is defined, + which is created in the thread + """ + self._start_event = Event() + Thread.start(self) + self._start_event.wait() + def run(self): """Run my loop, ignoring EINTR events in the poller""" + if 'asyncio' in sys.modules: + # tornado may be using asyncio, + # ensure an eventloop exists for this thread + import asyncio + asyncio.set_event_loop(asyncio.new_event_loop()) + self.ioloop = ioloop.IOLoop() + # signal that self.ioloop is defined + self._start_event.set() while True: try: self.ioloop.start() @@ -185,6 +210,7 @@ def stop(self): self.ioloop.add_callback(self.ioloop.stop) self.join() self.close() + self.ioloop = None def close(self): if self.ioloop is not None: @@ -198,22 +224,19 @@ class ThreadedKernelClient(KernelClient): """ A KernelClient that provides thread-safe sockets with async callbacks on message replies. """ - _ioloop = None @property def ioloop(self): - if self._ioloop is None: - self._ioloop = ioloop.IOLoop() - return self._ioloop + return self.ioloop_thread.ioloop ioloop_thread = Instance(IOLoopThread, allow_none=True) def start_channels(self, shell=True, iopub=True, stdin=True, hb=True): + self.ioloop_thread = IOLoopThread() + self.ioloop_thread.start() + if shell: self.shell_channel._inspect = self._check_kernel_info_reply - self.ioloop_thread = IOLoopThread(self.ioloop) - self.ioloop_thread.start() - super(ThreadedKernelClient, self).start_channels(shell, iopub, stdin, hb) def _check_kernel_info_reply(self, msg): From 238314d4e6ee8ff08318185e9a6750530bc5e418 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 9 Mar 2018 15:01:06 -0800 Subject: [PATCH 49/89] Wrap setting of kernel_id with method that can then be overridden in subclasses. A recent requirement for Jupyter Enterprise Gateway is for clients to be able to specify the kernel_id for new kernels. Although `jupyter_client.start_kernel()` will honor client-provided kernel_ids, Notebook's override of `start_kernel()` changes the semantics of a non-null kernel_id in the argument list to mean an existing (persisted) kernel should be _started_. As a result, applications that derive from the kernel management infrastructure beyond Notebook cannot influence the derivation of the kernel's id via the existing argument list behavior. By introducing the `determine_kernel_id()` method, subclasses are able to derive the kernel's id however they wish. With the ability to know the kernel's id prior to its invocation, a number of things can be done that wouldn't be possible otherwise. For example, this provides the ability to setup a shared filesystem location possibly pre-populated with data relative to what the request (i.e., kernel) is going to need. --- jupyter_client/multikernelmanager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index a83be953c..ca92b1dec 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -90,7 +90,7 @@ def start_kernel(self, kernel_name=None, **kwargs): The kernel ID for the newly started kernel is returned. """ - kernel_id = kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) + kernel_id = self.determine_kernel_id(**kwargs) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) @@ -315,3 +315,13 @@ def connect_hb(self, kernel_id, identity=None): ======= stream : zmq Socket or ZMQStream """ + + def determine_kernel_id(self, **kwargs): + """ + Returns the kernel_id to use for this request. If kernel_id is already in the arguments list, + that value will be used. Otherwise, a newly generated uuid is used. Subclasses may override + this method to substitute other sources of kernel ids. + :param kwargs: + :return: string-ized version 4 uuid + """ + return kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) From 5f167deba0645bee5d1792b31bd955065040cd3a Mon Sep 17 00:00:00 2001 From: Min RK Date: Sun, 11 Mar 2018 15:24:48 +0100 Subject: [PATCH 50/89] Changelog for 5.2.3 --- docs/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 101d3f7f7..11cbd9f1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changes in Jupyter Client ========================= +5.2.3 +===== + +`5.2.3 on GitHub `__ + +- Fix hang on close in :class:`.ThreadedKernelClient` (used in QtConsole) + when using tornado with asyncio + (default behavior of tornado 5, see :ghpull:`352`). +- Fix errors when using deprecated :attr:`.KernelManager.kernel_cmd` + (:ghpull:`343`, :ghpull:`344`). + 5.2.2 ===== From ac9c2e9f526e606bb0f8fd0bb4aa29b19ae3dff0 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Sun, 11 Mar 2018 09:15:24 -0700 Subject: [PATCH 51/89] Add metadata for new pypi.org --- setup.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c184b40fa..8d8213fc1 100644 --- a/setup.py +++ b/setup.py @@ -60,22 +60,35 @@ def run(self): name = name, version = version_ns['__version__'], packages = packages, - description = "Jupyter protocol implementation and client libraries", + description = 'Jupyter protocol implementation and client libraries', author = 'Jupyter Development Team', author_email = 'jupyter@googlegroups.com', url = 'https://jupyter.org', license = 'BSD', platforms = "Linux, Mac OS X, Windows", keywords = ['Interactive', 'Interpreter', 'Shell', 'Web'], + project_urls = { + 'Documentation': 'https://jupyter-client.readthedocs.io', + 'Source': 'https://github.com/jupyter/jupyter_client/', + 'Tracker': 'https://github.com/jupyter/jupyter_client/issues', + }, classifiers = [ + 'Framework :: Jupyter', 'Intended Audience :: Developers', + 'Intended Audience :: Education', 'Intended Audience :: System Administrators', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', + 'Operating System :: MacOS', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', ], install_requires = [ 'traitlets', @@ -85,6 +98,7 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], From bcede5760d16041e5a8f4af34b2e603e7067eb4a Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 12 Mar 2018 07:43:53 -0700 Subject: [PATCH 52/89] Changes from @takluyver @minrk review --- setup.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 8d8213fc1..cc7dd18e1 100644 --- a/setup.py +++ b/setup.py @@ -79,16 +79,10 @@ def run(self): 'Intended Audience :: System Administrators', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', - 'Operating System :: MacOS', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX :: Linux', + 'Operating System :: Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', ], install_requires = [ 'traitlets', @@ -98,7 +92,7 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, >=3.3', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], From f4981732d90c26697a357c2d9b2f80297c54547c Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 12 Mar 2018 07:49:47 -0700 Subject: [PATCH 53/89] Fix OS classifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc7dd18e1..951c4c5aa 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def run(self): 'Intended Audience :: System Administrators', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', - 'Operating System :: Independent', + 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', From bad02e9e8ad95a95edd81f9b7d71b348e9b19dcb Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 12 Mar 2018 07:53:16 -0700 Subject: [PATCH 54/89] Fix Python version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 951c4c5aa..f57008515 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, >=3.3', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <=4', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], From 08a9f6564ef16325b710690c43c6359062e5febb Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 12 Mar 2018 08:02:52 -0700 Subject: [PATCH 55/89] one more time --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f57008515..b6058f856 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <=4', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], 'test:python_version == "3.3"': ['pytest<3.3.0'], From f23b71a8b4995fa1611c9eb681014afb3d544a5d Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Tue, 13 Mar 2018 16:09:46 -0700 Subject: [PATCH 56/89] Fix memory leak of kernel Popen object After analyzing various leaked items when running either Notebook or Jupyter Kernel Gateway, one item that recurred across each kernel startup and shutdown sequence was the Popen object stored in the kernel manager in `self.kernel`. The issue is that in normal circumstances, when a kernel's termination is successful via the ZMQ messaging, the process is never waited for (which, in this case, is probably unnecessary but advised) nor is the member variable set to None. In the failing case, where the message-based shutdown does not terminate the kernel process, the `_kill_kernel()` method is used, which performs the `wait()` and _nullifies_ the kernel member. This change ensures that sequence occurs in normal situations as well. --- jupyter_client/manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 21f6ca925..f0b1c65c2 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -272,6 +272,10 @@ def finish_shutdown(self, waittime=None, pollinterval=0.1): if self.is_alive(): time.sleep(pollinterval) else: + # If there's still a proc, wait and clear + if self.has_kernel: + self.kernel.wait() + self.kernel = None break else: # OK, we've waited long enough. From 06e8167b838bfcd006b74e4265f1b446f168f630 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Tue, 13 Mar 2018 16:19:34 -0700 Subject: [PATCH 57/89] Fix leak of IOLoopKernelManager object After analyzing various leaked items when running either Notebook or Jupyter Kernel Gateway, one item that recurred across each kernel startup and shutdown sequence was an instance of IOLoopKernelManager. (Of course, when using JKG, this instance was KernelGatewayIOLoopKernelManager since it derives from the former.) The leak is caused by the circular references established in the `self._restarter` and `self.session.parent` members. This change breaks the circular reference when the restarter is stopped and during `cleanup()` of the kernel manager. Once the references are broken, the kernel manager instance can be garbage collected. --- jupyter_client/ioloop/manager.py | 1 + jupyter_client/manager.py | 1 + 2 files changed, 2 insertions(+) diff --git a/jupyter_client/ioloop/manager.py b/jupyter_client/ioloop/manager.py index cc285291b..f6dee3641 100644 --- a/jupyter_client/ioloop/manager.py +++ b/jupyter_client/ioloop/manager.py @@ -54,6 +54,7 @@ def stop_restarter(self): if self.autorestart: if self._restarter is not None: self._restarter.stop() + self._restarter = None connect_shell = as_zmqstream(KernelManager.connect_shell) connect_iopub = as_zmqstream(KernelManager.connect_iopub) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 21f6ca925..e39bcee1d 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -286,6 +286,7 @@ def cleanup(self, connection_file=True): self.cleanup_ipc_files() self._close_control_socket() + self.session.parent = None def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. From 12da5fb80943835fb9fe7720c5dd75cf28a71718 Mon Sep 17 00:00:00 2001 From: Kevin Bates Date: Fri, 23 Mar 2018 08:49:22 -0700 Subject: [PATCH 58/89] Changed kernel_id generation method name and calling scenario Per review comments, the name of the method to generate a kernel_id was changed to `new_kernel_id()`. In addition, the method is now only called if `kernel_id` is not represented in the keyword arguments (`**kwargs`). --- jupyter_client/multikernelmanager.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jupyter_client/multikernelmanager.py b/jupyter_client/multikernelmanager.py index ca92b1dec..c2b61bde0 100644 --- a/jupyter_client/multikernelmanager.py +++ b/jupyter_client/multikernelmanager.py @@ -86,11 +86,11 @@ def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, - otherwise one will be picked using a uuid. + otherwise one will be generated using new_kernel_id(). The kernel ID for the newly started kernel is returned. """ - kernel_id = self.determine_kernel_id(**kwargs) + kernel_id = kwargs.pop('kernel_id', self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) @@ -316,12 +316,11 @@ def connect_hb(self, kernel_id, identity=None): stream : zmq Socket or ZMQStream """ - def determine_kernel_id(self, **kwargs): + def new_kernel_id(self, **kwargs): """ - Returns the kernel_id to use for this request. If kernel_id is already in the arguments list, - that value will be used. Otherwise, a newly generated uuid is used. Subclasses may override + Returns the id to associate with the kernel for this request. Subclasses may override this method to substitute other sources of kernel ids. :param kwargs: :return: string-ized version 4 uuid """ - return kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) + return unicode_type(uuid.uuid4()) From dd50d58ebee059f6a5e40c3f6df07901c3f4435b Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 26 Mar 2018 12:41:50 +0200 Subject: [PATCH 59/89] master is 6.0-dev --- jupyter_client/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_client/_version.py b/jupyter_client/_version.py index 7f96345ae..53bf21fbd 100644 --- a/jupyter_client/_version.py +++ b/jupyter_client/_version.py @@ -1,4 +1,4 @@ -version_info = (5, 1, 0) +version_info = (6, 0, 0, 'dev') __version__ = '.'.join(map(str, version_info)) protocol_version_info = (5, 3) From bf57d23cd757fd2a258df94c4a9548ed26716ac6 Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Tue, 30 Jan 2018 17:46:20 +0100 Subject: [PATCH 60/89] Prevent creating new console on Windows When running Jupyter via pythonw e.g. pythonw -m qtconsole, jupyter_client launches new kernel via python.exe which is a console application on Windows - a side-effect of that is a new empty console window created and shown as long as kernel is running. This patch adds CREATE_NO_WINDOW 0x08000000 to Windows specific creationflags. This flag is not exported by subprocess module therefore has to be provides numerically. --- jupyter_client/launcher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 285778a68..33f35a7c7 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -111,6 +111,10 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, DUPLICATE_SAME_ACCESS) env['JPY_PARENT_PID'] = str(int(handle)) + # Prevent creating new console window on pythonw + if redirect_out: + kwargs['creationflags'] = kwargs.setdefault('creationflags', 0) | 0x08000000 # CREATE_NO_WINDOW + else: # Create a new session. # This makes it easier to interrupt the kernel, From 0e1a2185ddce3580528f2b021b554878bfc11209 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Fri, 4 May 2018 12:56:29 -0700 Subject: [PATCH 61/89] Update gitignore configuration --- .gitignore | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8e3f39a69..e8630aa02 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ MANIFEST build dist -_build -docs/gh-pages *.py[co] __pycache__ *.egg-info @@ -16,3 +14,15 @@ __pycache__ .coverage .cache absolute.json + +# Sphinx documentation +_build +docs/_build/ +docs/gh-pages + +# PyBuilder +target/ + +# PyCharm +.idea/ +*.iml From 375ea01b4db7cde5f14e55cc4e7b6ac5bb005998 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Tue, 15 May 2018 12:56:40 -0700 Subject: [PATCH 62/89] make KernelManager configurable for all who inherit JupyterConsoleApp --- jupyter_client/consoleapp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jupyter_client/consoleapp.py b/jupyter_client/consoleapp.py index ce2ead429..2e27a2918 100644 --- a/jupyter_client/consoleapp.py +++ b/jupyter_client/consoleapp.py @@ -18,7 +18,7 @@ from traitlets.config.application import boolean_flag from ipython_genutils.path import filefind from traitlets import ( - Dict, List, Unicode, CUnicode, CBool, Any + Dict, List, Unicode, CUnicode, CBool, Any, Type ) from jupyter_core.application import base_flags, base_aliases @@ -110,7 +110,11 @@ class JupyterConsoleApp(ConnectionFileMixin): classes = classes flags = Dict(flags) aliases = Dict(aliases) - kernel_manager_class = KernelManager + kernel_manager_class = Type( + default_value=KernelManager, + config=True, + help='The kernel manager class to use.' + ) kernel_client_class = BlockingKernelClient kernel_argv = List(Unicode()) From f1d8a95bede1d9d5b2ac7af6be702913b51a53fd Mon Sep 17 00:00:00 2001 From: M Pacer Date: Tue, 15 May 2018 13:50:50 -0700 Subject: [PATCH 63/89] Explicitly require a pytest that knows how to yield --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b6058f856..81b4dd0e9 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ def run(self): python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], - 'test:python_version == "3.3"': ['pytest<3.3.0'], + 'test:python_version == "3.3"': ['pytest>=3,<3.3.0'], 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], }, cmdclass = { From 889f822e4fed0be4ff2288c1548b609ca1733d35 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 16 May 2018 09:38:22 -0700 Subject: [PATCH 64/89] update python version testing --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index faec1b44c..8d7820ba2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ language: python python: - "nightly" - - '3.6-dev' + - "3-7-dev" + - 3.6 - 3.5 - 3.4 - - 3.3 - 2.7 sudo: false install: From 6c7a8abf835938c6542a148f7eeaf2f10a993811 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Wed, 16 May 2018 09:48:04 -0700 Subject: [PATCH 65/89] dash-dot --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d7820ba2..a50ca9719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "nightly" - - "3-7-dev" + - "3.7-dev" - 3.6 - 3.5 - 3.4 From d62626ebbec9eaef57f573f2cdcba9ecb15869a6 Mon Sep 17 00:00:00 2001 From: M Pacer Date: Wed, 16 May 2018 15:30:25 -0700 Subject: [PATCH 66/89] remove other places that are py3.3 specific, min py version is now py3.4 --- jupyter_client/tests/test_session.py | 4 ++-- setup.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 55181e91b..2eec57076 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -141,7 +141,7 @@ def test_send(self): # buffers must be contiguous buf = memoryview(os.urandom(16)) - if sys.version_info >= (3,3): + if sys.version_info >= (3,4): with self.assertRaises(ValueError): self.session.send(A, msg, ident=b'foo', buffers=[buf[::2]]) @@ -339,7 +339,7 @@ def test_send_raw(self): A.close() B.close() ctx.term() - + def test_clone(self): s = self.session s._add_digest('initial') diff --git a/setup.py b/setup.py index 81b4dd0e9..c641e7469 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ import sys v = sys.version_info -if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,3)): - error = "ERROR: %s requires Python version 2.7 or 3.3 or above." % name +if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,4)): + error = "ERROR: %s requires Python version 2.7 or 3.4 or above." % name print(error, file=sys.stderr) sys.exit(1) @@ -92,10 +92,9 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', extras_require = { 'test': ['ipykernel', 'ipython', 'mock'], - 'test:python_version == "3.3"': ['pytest>=3,<3.3.0'], 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], }, cmdclass = { From da6d97d4bdf42764d096a153a52e6051a40356b1 Mon Sep 17 00:00:00 2001 From: Travis DePrato Date: Tue, 5 Jun 2018 20:33:28 -0400 Subject: [PATCH 67/89] don't include extra buffers in message signature --- jupyter_client/session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyter_client/session.py b/jupyter_client/session.py index 33b1c0b4a..38166bcd5 100644 --- a/jupyter_client/session.py +++ b/jupyter_client/session.py @@ -779,7 +779,8 @@ def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): to_send.extend(ident) to_send.append(DELIM) - to_send.append(self.sign(msg_list)) + # Don't include buffers in signature (per spec). + to_send.append(self.sign(msg_list[0:4])) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) From 3c5a8be93c300548259ae9c87498d2640e68d291 Mon Sep 17 00:00:00 2001 From: Todd Date: Fri, 8 Jun 2018 16:57:47 -0400 Subject: [PATCH 68/89] Include LICENSE file in wheels The license requires that all copies of the software include the license. This makes sure the license is included in the wheels. See the wheel documentation [here](https://wheel.readthedocs.io/en/stable/#including-the-license-in-the-generated-wheel-file) for more information. --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index e0ca7a784..a2327e90f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,8 @@ [bdist_wheel] universal=1 +[metadata] +license_file = COPYING.md + [nosetests] warningfilters=default From 00812c3ab66c7bea7ab6c340f979b57334b6240b Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Aug 2018 11:21:59 -0700 Subject: [PATCH 69/89] Remove commented debug statement that used old API. --- jupyter_client/channels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index 64e565189..8c4ebf3c4 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -142,7 +142,6 @@ def run(self): continue since_last_heartbeat = 0.0 - # io.rprint('Ping from HB channel') # dbg # no need to catch EFSM here, because the previous event was # either a recv or connect, which cannot be followed by EFSM self.socket.send(b'ping') From cfe57c9287352e7717caacc28463d4f46c9fb1bd Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 28 Aug 2018 08:12:28 -0400 Subject: [PATCH 70/89] Allow third-party kernels to get additional args This removes special treatment of IPython console so that other kernels can get command-line args. This doesn't allow the passing of flags, but does allow filenames, etc. Once this fix is in place, kernels can get these args via self.parent.extra_args --- jupyter_client/consoleapp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jupyter_client/consoleapp.py b/jupyter_client/consoleapp.py index 2e27a2918..8d5d329fa 100644 --- a/jupyter_client/consoleapp.py +++ b/jupyter_client/consoleapp.py @@ -285,10 +285,8 @@ def init_kernel_manager(self): self.exit(1) self.kernel_manager.client_factory = self.kernel_client_class - # FIXME: remove special treatment of IPython kernels kwargs = {} - if self.kernel_manager.ipykernel: - kwargs['extra_arguments'] = self.kernel_argv + kwargs['extra_arguments'] = self.kernel_argv self.kernel_manager.start_kernel(**kwargs) atexit.register(self.kernel_manager.cleanup_ipc_files) From 72e9fc4b6d3a2a0df35b0bfd5cdbd6bfd49222e7 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Fri, 19 Oct 2018 20:03:55 -0700 Subject: [PATCH 71/89] try to fix coverage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a50ca9719..88fd7e722 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ python: sudo: false install: - pip install --upgrade setuptools pip - - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov + - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov 'coverage<5' script: - py.test --cov jupyter_client jupyter_client after_success: From 7a46b6bc6b3400ef70cd911898764c9899edb142 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 30 Nov 2018 13:21:58 +0100 Subject: [PATCH 72/89] set close_fds=False when starting kernels on Windows Python 3.7 sets close_fds=True by default, closing the interrupt/parent handles we are trying to pass to the kernel. --- jupyter_client/launcher.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jupyter_client/launcher.py b/jupyter_client/launcher.py index 33f35a7c7..1ba206269 100644 --- a/jupyter_client/launcher.py +++ b/jupyter_client/launcher.py @@ -101,7 +101,8 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, except: from _subprocess import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP - # Launch the kernel process + + # create a handle on the parent to be inherited if independent: kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP else: @@ -115,6 +116,11 @@ def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, if redirect_out: kwargs['creationflags'] = kwargs.setdefault('creationflags', 0) | 0x08000000 # CREATE_NO_WINDOW + # Avoid closing the above parent and interrupt handles. + # close_fds is True by default on Python >=3.7 + # or when no stream is captured on Python <3.7 + # (we always capture stdin, so this is already False by default on <3.7) + kwargs['close_fds'] = False else: # Create a new session. # This makes it easier to interrupt the kernel, From ddcfb5af3bc907ae483c8a6a82030f040515db83 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 10 Dec 2018 15:28:52 +0100 Subject: [PATCH 73/89] add long_description to setup.py now that the new PyPI looks bad without long descriptions --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c641e7469..065991a09 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,8 @@ def run(self): version = version_ns['__version__'], packages = packages, description = 'Jupyter protocol implementation and client libraries', + long_description=open('README.md').read(), + long_description_content_type='text/markdown', author = 'Jupyter Development Team', author_email = 'jupyter@googlegroups.com', url = 'https://jupyter.org', From 7a1793b98d6eb83aa471f47214af55af507c0ebc Mon Sep 17 00:00:00 2001 From: Travis DePrato <773453+travigd@users.noreply.github.com> Date: Mon, 17 Dec 2018 22:23:51 -0500 Subject: [PATCH 74/89] Fix documentation error. --- docs/messaging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/messaging.rst b/docs/messaging.rst index 7c533a7de..6e5a7b220 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -415,7 +415,7 @@ Execution results Message type: ``execute_reply``:: content = { - # One of: 'ok' OR 'error' OR 'abort' + # One of: 'ok' OR 'error' OR 'aborted' 'status' : str, # The global kernel counter that increases by one with each request that From 1e9194a85c59f0f96779810c74e4e19e4dd08580 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 11 Feb 2019 15:33:07 +0100 Subject: [PATCH 75/89] Replace sleep by wait --- jupyter_client/channels.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jupyter_client/channels.py b/jupyter_client/channels.py index 8c4ebf3c4..51d4d11a8 100644 --- a/jupyter_client/channels.py +++ b/jupyter_client/channels.py @@ -7,7 +7,7 @@ import atexit import errno -from threading import Thread +from threading import Thread, Event import time import zmq @@ -73,6 +73,7 @@ def __init__(self, context=None, session=None, address=None): # running is False until `.start()` is called self._running = False + self._exit = Event() # don't start paused self._pause = False self.poller = zmq.Poller() @@ -138,7 +139,7 @@ def run(self): while self._running: if self._pause: # just sleep, and skip the rest of the loop - time.sleep(self.time_to_dead) + self._exit.wait(self.time_to_dead) continue since_last_heartbeat = 0.0 @@ -154,7 +155,7 @@ def run(self): # sleep the remainder of the cycle remainder = self.time_to_dead - (time.time() - request_time) if remainder > 0: - time.sleep(remainder) + self._exit.wait(remainder) continue else: # nothing was received within the time limit, signal heart failure @@ -183,6 +184,7 @@ def is_beating(self): def stop(self): """Stop the channel's event loop and join its thread.""" self._running = False + self._exit.set() self.join() self.close() From 58282614d3c55f18264eecd4c8bdbcbb71b0d317 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 14 Feb 2019 16:29:01 +0100 Subject: [PATCH 76/89] remove pytest-warnings maybe this is what's pinning pytest to 3.3 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 88fd7e722..2390b8546 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,8 @@ python: sudo: false install: - pip install --upgrade setuptools pip - - pip install --upgrade --pre -e .[test] pytest-cov pytest-warnings codecov 'coverage<5' + - pip install --upgrade --pre -e .[test] pytest-cov codecov 'coverage<5' + - pip freeze script: - py.test --cov jupyter_client jupyter_client after_success: From cd9532cf52c8204213cc3202226ccfa3bc109eaf Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 14 Feb 2019 16:32:25 +0100 Subject: [PATCH 77/89] eager upgrade strategy otherwise, pip incorrectly determines that pytest has been satisfied even though it hasn't --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2390b8546..86fbd0064 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ python: sudo: false install: - pip install --upgrade setuptools pip - - pip install --upgrade --pre -e .[test] pytest-cov codecov 'coverage<5' + - pip install --upgrade --upgrade-strategy eager --pre -e .[test] pytest-cov codecov 'coverage<5' - pip freeze script: - py.test --cov jupyter_client jupyter_client From d9ff831eda9642881b77e8791e27f5f378ee66b5 Mon Sep 17 00:00:00 2001 From: Travis DePrato Date: Thu, 14 Feb 2019 15:20:10 -0500 Subject: [PATCH 78/89] Clarify stop_on_error documentation. --- docs/messaging.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/messaging.rst b/docs/messaging.rst index 6e5a7b220..31bda2f4d 100644 --- a/docs/messaging.rst +++ b/docs/messaging.rst @@ -351,9 +351,9 @@ Message type: ``execute_request``:: # should not send these messages. 'allow_stdin' : True, - # A boolean flag, which, if True, does not abort the execution queue, if an exception is encountered. - # This allows the queued execution of multiple execute_requests, even if they generate exceptions. - 'stop_on_error' : False, + # A boolean flag, which, if True, aborts the execution queue if an exception is encountered. + # If False, queued execute_requests will execute even if this request generates an exception. + 'stop_on_error' : True, } .. versionchanged:: 5.0 From 4638b7446fe1bf9f142c3fb67031364edd9ebf0e Mon Sep 17 00:00:00 2001 From: Eric Dill Date: Wed, 6 Mar 2019 15:04:33 -0500 Subject: [PATCH 79/89] Remove ambiguity in the startup description --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 5308c603f..145dbba41 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -8,7 +8,7 @@ A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for `several other languages `_. -When Jupyter starts a kernel, it passes it a connection file. This specifies +At startup, Jupyter passes the kernel a connection file. This specifies how to set up communications with the frontend. There are two options for writing a kernel: From f2f0e63561f6f12897b9a5c9b3032347ea5a80b8 Mon Sep 17 00:00:00 2001 From: Eric Dill Date: Wed, 6 Mar 2019 15:14:09 -0500 Subject: [PATCH 80/89] Consider removing word for clarity --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 145dbba41..75c86ad65 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -143,7 +143,7 @@ JSON serialised dictionary containing the following keys and values: These will be added to the current environment variables before the kernel is started. - **metadata** (optional): A dictionary of additional attributes about this - kernel; used by clients to aid clients in kernel selection. Metadata added + kernel; used by clients to aid in kernel selection. Metadata added here should be namespaced for the tool reading and writing that metadata. For example, the kernel.json file for IPython looks like this:: From b7bc293bc625b63a7d673e16e10effd41822f98d Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 25 Mar 2019 18:53:17 -0700 Subject: [PATCH 81/89] Remove some warning in test, create all dates as UTC. Dateutils complains otherwise. --- jupyter_client/tests/test_jsonutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_client/tests/test_jsonutil.py b/jupyter_client/tests/test_jsonutil.py index 9583a22c8..13ade51ec 100644 --- a/jupyter_client/tests/test_jsonutil.py +++ b/jupyter_client/tests/test_jsonutil.py @@ -45,11 +45,11 @@ def test_parse_ms_precision(): base = '2013-07-03T16:34:52' digits = '1234567890' - parsed = jsonutil.parse_date(base) + parsed = jsonutil.parse_date(base+'Z') assert isinstance(parsed, datetime.datetime) for i in range(len(digits)): ts = base + '.' + digits[:i] - parsed = jsonutil.parse_date(ts) + parsed = jsonutil.parse_date(ts+'Z') if i >= 1 and i <= 6: assert isinstance(parsed, datetime.datetime) else: From 1375c949aa3674e4be7f28721348f6eeae7436d8 Mon Sep 17 00:00:00 2001 From: Eric Dill Date: Sat, 30 Mar 2019 06:54:39 -0400 Subject: [PATCH 82/89] Disambiguate client startup and kernel startup --- docs/kernels.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 75c86ad65..568517b4d 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -8,7 +8,7 @@ A 'kernel' is a program that runs and introspects the user's code. IPython includes a kernel for Python code, and people have written kernels for `several other languages `_. -At startup, Jupyter passes the kernel a connection file. This specifies +At kernel startup, Jupyter passes the kernel a connection file. This specifies how to set up communications with the frontend. There are two options for writing a kernel: From 7b9c834e5912a77d9217f2de9229b570aefed877 Mon Sep 17 00:00:00 2001 From: SpencerPark Date: Mon, 1 Apr 2019 12:06:07 -0400 Subject: [PATCH 83/89] Set the default connection_file such that it preserves an existing configuration. --- jupyter_client/kernelapp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jupyter_client/kernelapp.py b/jupyter_client/kernelapp.py index a2ab17812..2d50a95d3 100644 --- a/jupyter_client/kernelapp.py +++ b/jupyter_client/kernelapp.py @@ -30,10 +30,12 @@ class KernelApp(JupyterApp): def initialize(self, argv=None): super(KernelApp, self).initialize(argv) + + cf_basename = 'kernel-%s.json' % uuid.uuid4() + self.config.setdefault('KernelManager', {}).setdefault('connection_file', os.path.join(self.runtime_dir, cf_basename)) self.km = KernelManager(kernel_name=self.kernel_name, config=self.config) - cf_basename = 'kernel-%s.json' % uuid.uuid4() - self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) + self.loop = IOLoop.current() self.loop.add_callback(self._record_started) From 56533ff2f3a9370e85659adf3a5ed8e1abf4cbbd Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 25 Apr 2019 11:04:19 +0100 Subject: [PATCH 84/89] Build docs with Python 3.7 on conda This might fix some failures to build on RTD, where Sphinx is failing to import something from the typing module. --- docs/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/environment.yml b/docs/environment.yml index 459e7ab3b..b7ed943c1 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - pyzmq -- python==3.5 +- python==3.7 - traitlets>=4.1 - jupyter_core - sphinx>=1.3.6 From 0658a7bd208830187c441b61ce9a71e9a0ceed13 Mon Sep 17 00:00:00 2001 From: Alex Rudy Date: Wed, 8 May 2019 12:10:56 -0400 Subject: [PATCH 85/89] Demonstrate kernel failures with multiple processes --- jupyter_client/client.py | 2 +- jupyter_client/tests/test_kernelmanager.py | 120 ++++++++++++++++++++- 2 files changed, 120 insertions(+), 2 deletions(-) diff --git a/jupyter_client/client.py b/jupyter_client/client.py index 763af85a7..8bdd75893 100644 --- a/jupyter_client/client.py +++ b/jupyter_client/client.py @@ -51,7 +51,7 @@ class KernelClient(ConnectionFileMixin): # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): - return zmq.Context.instance() + return zmq.Context() # The classes to use for the various channels shell_channel_class = Type(ChannelABC) diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index a23b33fa6..63ef56394 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -11,6 +11,9 @@ from subprocess import PIPE import sys import time +import threading +import multiprocessing as mp +import pytest from unittest import TestCase from traitlets.config.loader import Config @@ -28,7 +31,7 @@ def setUp(self): def tearDown(self): self.env_patch.stop() - + def _install_test_kernel(self): kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'signaltest') os.makedirs(kernel_dir) @@ -127,3 +130,118 @@ def test_start_new_kernel(self): self.assertTrue(km.is_alive()) self.assertTrue(kc.is_alive()) + +@pytest.mark.parallel +class TestParallel: + + @pytest.fixture(autouse=True) + def env(self): + env_patch = test_env() + env_patch.start() + yield + env_patch.stop() + + @pytest.fixture(params=['tcp', 'ipc']) + def transport(self, request): + return request.param + + @pytest.fixture + def config(self, transport): + c = Config() + c.transport = transport + if transport == 'ipc': + c.ip = 'test' + return c + + def _install_test_kernel(self): + kernel_dir = pjoin(paths.jupyter_data_dir(), 'kernels', 'signaltest') + os.makedirs(kernel_dir) + with open(pjoin(kernel_dir, 'kernel.json'), 'w') as f: + f.write(json.dumps({ + 'argv': [sys.executable, + '-m', 'jupyter_client.tests.signalkernel', + '-f', '{connection_file}'], + 'display_name': "Signal Test Kernel", + })) + + def test_start_sequence_kernels(self, config): + """Ensure that a sequence of kernel startups doesn't break anything.""" + + self._install_test_kernel() + self._run_signaltest_lifecycle(config) + self._run_signaltest_lifecycle(config) + self._run_signaltest_lifecycle(config) + + def test_start_parallel_thread_kernels(self, config): + self._install_test_kernel() + self._run_signaltest_lifecycle(config) + + thread = threading.Thread(target=self._run_signaltest_lifecycle, args=(config,)) + thread2 = threading.Thread(target=self._run_signaltest_lifecycle, args=(config,)) + try: + thread.start() + thread2.start() + finally: + thread.join() + thread2.join() + + def test_start_parallel_process_kernels(self, config): + self._install_test_kernel() + + self._run_signaltest_lifecycle(config) + thread = threading.Thread(target=self._run_signaltest_lifecycle, args=(config,)) + proc = mp.Process(target=self._run_signaltest_lifecycle, args=(config,)) + try: + thread.start() + proc.start() + finally: + thread.join() + proc.join() + + assert proc.exitcode == 0 + + def test_start_sequence_process_kernels(self, config): + self._install_test_kernel() + self._run_signaltest_lifecycle(config) + proc = mp.Process(target=self._run_signaltest_lifecycle, args=(config,)) + try: + proc.start() + finally: + proc.join() + + assert proc.exitcode == 0 + + def _prepare_kernel(self, km, startup_timeout=TIMEOUT, **kwargs): + km.start_kernel(**kwargs) + kc = km.client() + kc.start_channels() + try: + kc.wait_for_ready(timeout=startup_timeout) + except RuntimeError: + kc.stop_channels() + km.shutdown_kernel() + raise + + return kc + + def _run_signaltest_lifecycle(self, config=None): + km = KernelManager(config=config, kernel_name='signaltest') + kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE) + + def execute(cmd): + kc.execute(cmd) + reply = kc.get_shell_msg(TIMEOUT) + content = reply['content'] + assert content['status'] == 'ok' + return content + + execute("start") + assert km.is_alive() + execute('check') + assert km.is_alive() + + km.restart_kernel(now=True) + assert km.is_alive() + execute('check') + + km.shutdown_kernel() \ No newline at end of file From d3ddee0cd264a552b05b51240d13fb88b9a4b094 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 13 May 2019 12:05:29 +0200 Subject: [PATCH 86/89] drop Python 3.4 support --- .travis.yml | 1 - setup.py | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 86fbd0064..e53df98d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - "3.7-dev" - 3.6 - 3.5 - - 3.4 - 2.7 sudo: false install: diff --git a/setup.py b/setup.py index 065991a09..4616d5d1e 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ import sys v = sys.version_info -if v[:2] < (2,7) or (v[0] >= 3 and v[:2] < (3,4)): - error = "ERROR: %s requires Python version 2.7 or 3.4 or above." % name +if v[:2] < (2, 7) or (v[0] >= 3 and v[:2] < (3, 5)): + error = "ERROR: %s requires Python version 2.7 or 3.5 or above." % name print(error, file=sys.stderr) sys.exit(1) @@ -94,10 +94,9 @@ def run(self): 'entrypoints', 'tornado>=4.1', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', extras_require = { - 'test': ['ipykernel', 'ipython', 'mock'], - 'test:(python_version >= "3.4" or python_version == "2.7")': ['pytest'], + 'test': ['ipykernel', 'ipython', 'mock', 'pytest'], }, cmdclass = { 'bdist_egg': bdist_egg if 'bdist_egg' in sys.argv else bdist_egg_disabled, From ddd945d983b99be4a448804329918a623c6079c2 Mon Sep 17 00:00:00 2001 From: martinRenou Date: Tue, 21 May 2019 15:35:33 +0200 Subject: [PATCH 87/89] Add xeus to the documentation on how to write a kernel for Jupyter --- docs/kernels.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/kernels.rst b/docs/kernels.rst index 568517b4d..41e2eb812 100644 --- a/docs/kernels.rst +++ b/docs/kernels.rst @@ -11,7 +11,7 @@ includes a kernel for Python code, and people have written kernels for At kernel startup, Jupyter passes the kernel a connection file. This specifies how to set up communications with the frontend. -There are two options for writing a kernel: +There are three options for writing a kernel: 1. You can reuse the IPython kernel machinery to handle the communications, and just describe how to execute your code. This is much simpler if the target @@ -19,6 +19,17 @@ There are two options for writing a kernel: 2. You can implement the kernel machinery in your target language. This is more work initially, but the people using your kernel might be more likely to contribute to it if it's in the language they know. +3. You can use the `xeus `_ library that is + a C++ implementation of the Jupyter kernel protocol. Kernel authors only need to + implement the language-specific logic in their implementation + (execute code, auto-completion...). This is the simplest + solution if your target language can be driven from C or C++: e.g. if it has + a C-API like most scripting languages. Check out the + `xeus documentation `_ for more details. + Examples of kernels based on xeus include: + - `xeus-cling `_ + - `xeus-python `_ + - `JuniperKernel `_ Connection files ================ From 59a74df2b59d5f575f513b070f8c5f765c392b40 Mon Sep 17 00:00:00 2001 From: Maarten Breddels Date: Tue, 28 May 2019 23:46:37 +0200 Subject: [PATCH 88/89] feat(fork): send fork message to kernel --- jupyter_client/blocking/client.py | 1 + jupyter_client/client.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/jupyter_client/blocking/client.py b/jupyter_client/blocking/client.py index c0196ba36..609ccd55c 100644 --- a/jupyter_client/blocking/client.py +++ b/jupyter_client/blocking/client.py @@ -154,6 +154,7 @@ def _recv_reply(self, msg_id, timeout=None): return reply + fork = reqrep(KernelClient.fork) execute = reqrep(KernelClient.execute) history = reqrep(KernelClient.history) complete = reqrep(KernelClient.complete) diff --git a/jupyter_client/client.py b/jupyter_client/client.py index 8bdd75893..bf2aa0cd3 100644 --- a/jupyter_client/client.py +++ b/jupyter_client/client.py @@ -254,6 +254,12 @@ def execute(self, code, silent=False, store_history=True, self.shell_channel.send(msg) return msg['header']['msg_id'] + def fork(self): + content = {} + msg = self.session.msg('fork', content) + self.shell_channel.send(msg) + return msg['header']['msg_id'] + def complete(self, code, cursor_pos=None): """Tab complete text in the kernel's namespace. From c4b83484cb8e1612a9a4abf5c40101fb4b3bf334 Mon Sep 17 00:00:00 2001 From: Maarten Breddels Date: Tue, 28 May 2019 23:34:38 +0200 Subject: [PATCH 89/89] TEMP: add help script to test and connection files --- conn.json | 12 ++++++++++++ conn_fork.json | 12 ++++++++++++ fork_kernel.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 conn.json create mode 100644 conn_fork.json create mode 100644 fork_kernel.py diff --git a/conn.json b/conn.json new file mode 100644 index 000000000..21835e03d --- /dev/null +++ b/conn.json @@ -0,0 +1,12 @@ +{ + "shell_port": 56573, + "iopub_port": 56574, + "stdin_port": 56575, + "control_port": 56576, + "hb_port": 56577, + "ip": "127.0.0.1", + "key": "1371111a-7b97c86926519da79e2acbf2", + "transport": "tcp", + "signature_scheme": "hmac-sha256", + "kernel_name": "" +} \ No newline at end of file diff --git a/conn_fork.json b/conn_fork.json new file mode 100644 index 000000000..9fbd52ad1 --- /dev/null +++ b/conn_fork.json @@ -0,0 +1,12 @@ +{ + "shell_port": 66573, + "iopub_port": 66574, + "stdin_port": 66575, + "control_port": 66576, + "hb_port": 66577, + "ip": "127.0.0.1", + "key": "1371111a-7b97c86926519da79e2acbf3", + "transport": "tcp", + "signature_scheme": "hmac-sha256", + "kernel_name": "" +} \ No newline at end of file diff --git a/fork_kernel.py b/fork_kernel.py new file mode 100644 index 000000000..0e5eb505c --- /dev/null +++ b/fork_kernel.py @@ -0,0 +1,16 @@ +import sys +from jupyter_client import BlockingKernelClient +import logging +logging.basicConfig() + +connection_file = sys.argv[1] + +client = BlockingKernelClient() +client.log.setLevel('DEBUG') +client.load_connection_file(sys.argv[1]) +client.start_channels() +client.wait_for_ready(100) + +client.fork() +msg = client.shell_channel.get_msg(timeout=100) +print(msg) \ No newline at end of file