diff --git a/changelogs/fragments/python38-macos.yaml b/changelogs/fragments/python38-macos.yaml new file mode 100644 index 0000000000..e418dbedbd --- /dev/null +++ b/changelogs/fragments/python38-macos.yaml @@ -0,0 +1,2 @@ +bugfixes: +- TaskQueueManager - Explicitly set the mutliprocessing start method to ``fork`` to avoid issues with the default on macOS now being ``spawn``. diff --git a/lib/ansible/executor/process/worker.py b/lib/ansible/executor/process/worker.py index 5d3e1ed499..d01ff7f5cc 100644 --- a/lib/ansible/executor/process/worker.py +++ b/lib/ansible/executor/process/worker.py @@ -19,7 +19,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import multiprocessing import os import sys import traceback @@ -41,13 +40,14 @@ from ansible.executor.task_executor import TaskExecutor from ansible.executor.task_result import TaskResult from ansible.module_utils._text import to_text from ansible.utils.display import Display +from ansible.utils.multiprocessing import context as multiprocessing_context __all__ = ['WorkerProcess'] display = Display() -class WorkerProcess(multiprocessing.Process): +class WorkerProcess(multiprocessing_context.Process): ''' The worker thread class, which uses TaskExecutor to run tasks read from a job queue and pushes results into a results queue diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index cee7571008..475e71d192 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -19,7 +19,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import multiprocessing import os import tempfile @@ -41,6 +40,7 @@ from ansible.utils.helpers import pct_to_int from ansible.vars.hostvars import HostVars from ansible.vars.reserved import warn_if_reserved from ansible.utils.display import Display +from ansible.utils.multiprocessing import context as multiprocessing_context __all__ = ['TaskQueueManager'] @@ -97,7 +97,7 @@ class TaskQueueManager: self._unreachable_hosts = dict() try: - self._final_q = multiprocessing.Queue() + self._final_q = multiprocessing_context.Queue() except OSError as e: raise AnsibleError("Unable to use multiprocessing, this is normally caused by lack of access to /dev/shm: %s" % to_native(e)) diff --git a/lib/ansible/utils/multiprocessing.py b/lib/ansible/utils/multiprocessing.py new file mode 100644 index 0000000000..dcc186597a --- /dev/null +++ b/lib/ansible/utils/multiprocessing.py @@ -0,0 +1,21 @@ +# Copyright (c) 2019 Matt Martz +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import multiprocessing + +# Explicit multiprocessing context using the fork start method +# This exists as a compat layer now that Python3.8 has changed the default +# start method for macOS to ``spawn`` which is incompatible with our +# code base currently +# +# This exists in utils to allow it to be easily imported into various places +# without causing circular import or dependency problems +try: + context = multiprocessing.get_context('fork') +except AttributeError: + # Py2 has no context functionality, and only supports fork + context = multiprocessing