ProcessPoolExecutor run in jupyter will hang up if exception raised, with ipython 8.0.1 on debian

Notebook
1

run below code in jupyter notebook with ipthon 8.0.1 on debian/ubuntu, the cell will never return.

this issue disappear on windows

from concurrent.futures import ProcessPoolExecutor
def test():
    raise
executor = ProcessPoolExecutor(max_workers=1)
executor.submit(test).result()

ipython                           8.0.1
jupyter                           1.0.0
jupyter-client                    7.1.2
jupyter-console                   6.4.0
jupyter-contrib-core              0.3.3
jupyter-contrib-nbextensions      0.5.1
jupyter-core                      4.9.1
jupyter-highlight-selected-word   0.2.0
jupyter-latex-envs                1.4.6
jupyter-nbextensions-configurator 0.4.1
jupyter-server                    1.13.4
jupyterlab                        3.2.8
jupyterlab-pygments               0.1.2
jupyterlab-server                 2.10.3
jupyterlab-widgets                1.0.2

[IPKernelApp] ERROR | Exception in message handler:
concurrent.futures.process._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/concurrent/futures/process.py", line 243, in _process_worker
    r = call_item.fn(*call_item.args, **call_item.kwargs)
  File "/tmp/ipykernel_53/1368191033.py", line 4, in test
    raise
RuntimeError: No active exception to reraise
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3251, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "/tmp/ipykernel_53/1368191033.py", line 9, in <module>
    executor.submit(test).result()
  File "/usr/local/lib/python3.10/concurrent/futures/_base.py", line 446, in result
    return self.__get_result()
  File "/usr/local/lib/python3.10/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
RuntimeError: No active exception to reraise

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'RuntimeError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3191, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3268, in run_code
    self.showtraceback(running_compiled_code=True)
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'AssertionError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2814, in _run_cell
    return runner(coro)
  File "/usr/local/lib/python3.10/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3012, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 3210, in run_ast_nodes
    self.showtraceback()
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'AssertionError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 357, in dispatch_shell
    await result
  File "/usr/local/lib/python3.10/site-packages/ipykernel/kernelbase.py", line 652, in execute_request
    reply_content = await reply_content
  File "/usr/local/lib/python3.10/site-packages/ipykernel/ipkernel.py", line 353, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/usr/local/lib/python3.10/site-packages/ipykernel/zmqshell.py", line 532, in run_cell
    return super().run_cell(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2768, in run_cell
    result = self._run_cell(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 2819, in _run_cell
    self.showtraceback(running_compiled_code=True)
  File "/usr/local/lib/python3.10/site-packages/IPython/core/interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "/usr/local/lib/python3.10/site-packages/IPython/core/ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError
2

+1 on this… experienced on windows + macos on python 3.9, and consistently with ipython 8. Slightly different reproducer but I think is similar:

import joblib
import time

def errorfunc(): 
    time.sleep(1)
    raise ValueError("I failed")
    
jobs = [joblib.delayed(errorfunc)() for _ in range(10)]
joblib.Parallel(5)(jobs)

[IPKernelApp] ERROR | Exception in message handler:
joblib.externals.loky.process_executor._RemoteTraceback:
"""
Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\joblib\externals\loky\process_executor.py", line 436, in _process_worker
    r = call_item()
  File "C:\Users\me\mypython\lib\site-packages\joblib\externals\loky\process_executor.py", line 288, in __call__
    return self.fn(*self.args, **self.kwargs)
  File "C:\Users\me\mypython\lib\site-packages\joblib\_parallel_backends.py", line 595, in __call__
    return self.func(*args, **kwargs)
  File "C:\Users\me\mypython\lib\site-packages\joblib\parallel.py", line 262, in __call__
    return [func(*args, **kwargs)
  File "C:\Users\me\mypython\lib\site-packages\joblib\parallel.py", line 262, in <listcomp>
    return [func(*args, **kwargs)
  File "C:\Users\nayrm\AppData\Local\Temp\ipykernel_10556\111525020.py", line 6, in errorfunc
ValueError: I failed
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 3251, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "C:\Users\nayrm\AppData\Local\Temp\ipykernel_10556\111525020.py", line 9, in <module>
    joblib.Parallel(5)(jobs)
  File "C:\Users\me\mypython\lib\site-packages\joblib\parallel.py", line 1056, in __call__    self.retrieve()
  File "C:\Users\me\mypython\lib\site-packages\joblib\parallel.py", line 935, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "C:\Users\me\mypython\lib\site-packages\joblib\_parallel_backends.py", line 542, in wrap_future_result
    return future.result(timeout=timeout)
  File "C:\Users\me\mypython\lib\concurrent\futures\_base.py", line 445, in result
    return self.__get_result()
  File "C:\Users\me\mypython\lib\concurrent\futures\_base.py", line 390, in __get_result
    raise self._exception
ValueError: I failed

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'ValueError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 3191, in run_ast_nodes
    if await self.run_code(code, result, async_=asy):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 3268, in run_code
    self.showtraceback(running_compiled_code=True)
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'AssertionError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 2814, in _run_cell
    return runner(coro)
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\async_helpers.py", line 129, in _pseudo_sync_runner
    coro.send(None)
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 3012, in run_cell_async
    has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 3210, in run_ast_nodes
    self.showtraceback()
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1934, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'AssertionError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\me\mypython\lib\site-packages\ipykernel\kernelbase.py", line 357, in dispatch_shell
    await result
  File "C:\Users\me\mypython\lib\site-packages\ipykernel\kernelbase.py", line 652, in execute_request
    reply_content = await reply_content
  File "C:\Users\me\mypython\lib\site-packages\ipykernel\ipkernel.py", line 353, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "C:\Users\me\mypython\lib\site-packages\ipykernel\zmqshell.py", line 532, in run_cell
    return super().run_cell(*args, **kwargs)
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 2768, in run_cell
    result = self._run_cell(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 2819, in _run_cell
    self.showtraceback(running_compiled_code=True)
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\interactiveshell.py", line 1936, in showtraceback
    stb = self.InteractiveTB.structured_traceback(etype,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 1105, in structured_traceback
    return FormattedTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 999, in structured_traceback
    return VerboseTB.structured_traceback(
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 871, in structured_traceback
    formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
  File "C:\Users\me\mypython\lib\site-packages\IPython\core\ultratb.py", line 775, in format_exception_as_a_whole
    assert etb is not None
AssertionError

conda install ipython=7.31.1 resolved the issue in both cases