From 6ce9cf7741679449fc3ac6347bd7209ae697cc5b Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Wed, 3 Apr 2019 22:35:59 -0400 Subject: [PATCH] Change default smart connection to ssh on macOS and remove paramiko from requirements.txt (#54738) * Remove default use of paramiko connection plugin on macOS This fix was originally to work around a bug that caused a kernel panic on macOS that has since been fixed. * Remove paramiko from requirements.txt * Move paramiko checking to common place * Drop the warnings obfiscation code * Update pip installation instructions to reflect upstream instructions * Fix tests on CentOS 6 (Python 2.6) that now show Python deprecation warnings * Add changelog fragment --- .../macos-paramiko-default-to-ssh.yaml | 2 + .../installation_guide/intro_installation.rst | 77 +++++++++++-------- .../rst/porting_guides/porting_guide_2.8.rst | 4 +- lib/ansible/module_utils/compat/paramiko.py | 19 +++++ .../modules/network/iosxr/iosxr_user.py | 12 +-- .../modules/network/nxos/nxos_file_copy.py | 12 +-- .../modules/network/panos/_panos_admpwd.py | 13 +--- .../network/panos/_panos_cert_gen_ssh.py | 15 +--- lib/ansible/playbook/play_context.py | 13 +--- .../plugins/connection/paramiko_ssh.py | 22 +----- requirements.txt | 1 - test/integration/targets/assert/runme.sh | 3 + .../targets/callback_default/runme.sh | 3 + 13 files changed, 92 insertions(+), 104 deletions(-) create mode 100644 changelogs/fragments/macos-paramiko-default-to-ssh.yaml create mode 100644 lib/ansible/module_utils/compat/paramiko.py diff --git a/changelogs/fragments/macos-paramiko-default-to-ssh.yaml b/changelogs/fragments/macos-paramiko-default-to-ssh.yaml new file mode 100644 index 0000000000..4b164c67ff --- /dev/null +++ b/changelogs/fragments/macos-paramiko-default-to-ssh.yaml @@ -0,0 +1,2 @@ +minor_changes: + - change default connection plugin on macOS when using smart mode to ssh instead of paramiko (https://github.com/ansible/ansible/pull/54738) diff --git a/docs/docsite/rst/installation_guide/intro_installation.rst b/docs/docsite/rst/installation_guide/intro_installation.rst index 350f59d38d..0beba6b773 100644 --- a/docs/docsite/rst/installation_guide/intro_installation.rst +++ b/docs/docsite/rst/installation_guide/intro_installation.rst @@ -31,10 +31,10 @@ Major bugs will still have maintenance releases when needed, though these are in If you are wishing to run the latest released version of Ansible and you are running Red Hat Enterprise Linux (TM), CentOS, Fedora, Debian, or Ubuntu, we recommend using the OS package manager. -For other installation options, we recommend installing via "pip", which is the Python package manager, though other options are also available. +For other installation options, we recommend installing via ``pip``, which is the Python package manager. If you wish to track the development release to use and test the latest features, we will share -information about running from source. It's not necessary to install the program to run from source. +information about running from source. It's not necessary to install the program to run from source. .. _control_node_requirements: @@ -230,9 +230,9 @@ Older versions of FreeBSD worked with something like this (substitute for your c Latest Releases on macOS ++++++++++++++++++++++++++ -The preferred way to install Ansible on a Mac is via pip. +The preferred way to install Ansible on a Mac is via ``pip``. -The instructions can be found in `Latest Releases via Pip`_ section. If you are running macOS version 10.12 or older, then you ought to upgrade to the latest pip (9.0.3 or newer) to connect to the Python Package Index securely. +The instructions can be found in `Latest Releases via Pip`_ section. If you are running macOS version 10.12 or older, then you should upgrade to the latest ``pip`` to connect to the Python Package Index securely. .. _from_pkgutil: @@ -293,30 +293,47 @@ Update of the software will be managed by the swupd tool:: Latest Releases via Pip +++++++++++++++++++++++ -Ansible can be installed via "pip", the Python package manager. If 'pip' isn't already available in -your version of Python, you can get pip by:: +Ansible can be installed via ``pip``, the Python package manager. If ``pip`` isn't already available on your system of Python, run the following commands to install it:: - $ sudo easy_install pip + $ curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + $ python get-pip.py --user -Then install Ansible with [1]_:: +Then install Ansible [1]_:: - $ sudo pip install ansible + $ pip install --user ansible Or if you are looking for the latest development version:: - $ pip install git+https://github.com/ansible/ansible.git@devel + $ pip install --user git+https://github.com/ansible/ansible.git@devel -If you are installing on macOS Mavericks, you may encounter some noise from your compiler. A workaround is to do the following:: +If you are installing on macOS Mavericks (10.9), you may encounter some noise from your compiler. A workaround is to do the following:: - $ sudo CFLAGS=-Qunused-arguments CPPFLAGS=-Qunused-arguments pip install ansible + $ CFLAGS=-Qunused-arguments CPPFLAGS=-Qunused-arguments pip install --user ansible -Readers that use virtualenv can also install Ansible under virtualenv, though we'd recommend to not worry about it and just install Ansible globally. Do not use easy_install to install Ansible directly. +In order to use the ``paramiko`` connection plugin or modules that require ``paramiko``, install the required module [2]_:: + + $ pip install --user paramiko + +Ansble can also be installed inside a new or existing ``virtualenv``:: + + $ python -m virtualenv ansible # Create a virtualenv if one does not already exist + $ source ansible/bin/activate # Activate the virtual environment + $ pip install ansible + +If you wish to install Ansible globally, run the following commands:: + + $ sudo python get-pip.py + $ sudo pip install ansible .. note:: - Older versions of pip defaults to http://pypi.python.org/simple, which no longer works. - Please make sure you have an updated pip (version 10 or greater) installed before installing Ansible. - Refer `here `_ about installing latest pip. + Running ``pip`` with ``sudo`` will make global changes to the system. Since ``pip`` does not coordinate with system package managers, it could make changes to you system that leave it in an inconsistent on non-functioning state. This is particularly true for macOS. Installing with ``--user`` is recommended unless you understand fully the implications of modifying global files on the system. + +.. note:: + + Older versions of ``pip`` default to http://pypi.python.org/simple, which no longer works. + Please make sure you have the latest version of ``pip`` before installing Ansible. + If you have an older version of ``pip`` installed, you can upgrade by following `pip's upgrade instructions `_ . .. _tagged_releases: @@ -335,9 +352,9 @@ These releases are also tagged in the `git repository `_ if -you have a GitHub account. This is also where we keep the issue tracker for sharing +you have a GitHub account. This is also where we keep the issue tracker for sharing bugs and feature ideas. @@ -439,3 +453,4 @@ bugs and feature ideas. #ansible IRC chat channel .. [1] If you have issues with the "pycrypto" package install on macOS, then you may need to try ``CC=clang sudo -E pip install pycrypto``. +.. [2] ``paramiko`` was included in Ansible's ``requirements.txt`` prior to 2.8. diff --git a/docs/docsite/rst/porting_guides/porting_guide_2.8.rst b/docs/docsite/rst/porting_guides/porting_guide_2.8.rst index ccda1c48c2..b47b88be44 100644 --- a/docs/docsite/rst/porting_guides/porting_guide_2.8.rst +++ b/docs/docsite/rst/porting_guides/porting_guide_2.8.rst @@ -323,7 +323,7 @@ Noteworthy module changes You should use the ``win_service`` module to control the running state of the service. This will be removed in Ansible 2.12. * The ``status`` module option for ``win_nssm`` has changed its default value to ``present``. Before, the default was ``start``. - Consequently, the service is no longer started by default after creation with ``win_nssm``, and you should use + Consequently, the service is no longer started by default after creation with ``win_nssm``, and you should use the ``win_service`` module to start it if needed. * The ``app_parameters`` module option for ``win_nssm`` has been deprecated; use ``argument`` instead. This will be removed in Ansible 2.12. @@ -340,6 +340,8 @@ Noteworthy module changes Plugins ======= +* Ansible no longer defaults to the ``paramiko`` connection plugin when using macOS as the control node. Ansible will now use the ``ssh`` connection plugin by default on a macOS control node. Since ``ssh`` supports connection persistence between tasks and playbook runs, it performs better than ``paramiko``. If you are using password authentication, you will need to install ``sshpass`` when using the ``ssh`` connection plugin. Or you can explicitly set the connection type to ``paramiko`` to maintain the pre-2.8 behavior on macOS. + * Connection plugins have been standardized to allow use of ``ansible__user`` and ``ansible__password`` variables. Variables such as ``ansible__pass`` and ``ansible__username`` are treated diff --git a/lib/ansible/module_utils/compat/paramiko.py b/lib/ansible/module_utils/compat/paramiko.py new file mode 100644 index 0000000000..12d75dd5cc --- /dev/null +++ b/lib/ansible/module_utils/compat/paramiko.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Ansible Project +# Simplified BSD License (see licenses/simplified_bsd.txt or https://opensource.org/licenses/BSD-2-Clause) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +PARAMIKO_IMPORT_ERR = None + +paramiko = None +try: + import paramiko +except ImportError: + try: + import ansible_paramiko as paramiko + except (ImportError, AttributeError) as err: # paramiko and gssapi are incompatible and raise AttributeError not ImportError + PARAMIKO_IMPORT_ERR = err +except AttributeError as err: # paramiko and gssapi are incompatible and raise AttributeError not ImportError + PARAMIKO_IMPORT_ERR = err diff --git a/lib/ansible/modules/network/iosxr/iosxr_user.py b/lib/ansible/modules/network/iosxr/iosxr_user.py index 95f978a57d..4015c7eb55 100644 --- a/lib/ansible/modules/network/iosxr/iosxr_user.py +++ b/lib/ansible/modules/network/iosxr/iosxr_user.py @@ -206,6 +206,7 @@ from copy import deepcopy import collections from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.compat.paramiko import paramiko from ansible.module_utils.network.common.utils import remove_default_spec from ansible.module_utils.network.iosxr.iosxr import get_config, load_config, is_netconf, is_cliconf from ansible.module_utils.network.iosxr.iosxr import iosxr_argument_spec, build_xml, etree_findall @@ -216,15 +217,6 @@ try: except ImportError: HAS_B64 = False -HAS_PARAMIKO = True -try: - import paramiko -except ImportError: - try: - import ansible_paramiko as paramiko - except ImportError: - HAS_PARAMIKO = False - class PublicKeyManager(object): def __init__(self, module, result): @@ -693,7 +685,7 @@ def main(): msg='library base64 is required but does not appear to be ' 'installed. It can be installed using `pip install base64`' ) - if not HAS_PARAMIKO: + if paramiko is None: module.fail_json( msg='library paramiko is required but does not appear to be ' 'installed. It can be installed using `pip install paramiko`' diff --git a/lib/ansible/modules/network/nxos/nxos_file_copy.py b/lib/ansible/modules/network/nxos/nxos_file_copy.py index 8cd2fd2e37..c1f6bbaca9 100644 --- a/lib/ansible/modules/network/nxos/nxos_file_copy.py +++ b/lib/ansible/modules/network/nxos/nxos_file_copy.py @@ -160,20 +160,12 @@ import re import time import traceback +from ansible.module_utils.compat.paramiko import paramiko from ansible.module_utils.network.nxos.nxos import run_commands from ansible.module_utils.network.nxos.nxos import nxos_argument_spec, check_args from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_native, to_text, to_bytes -HAS_PARAMIKO = True -try: - import paramiko -except ImportError: - try: - import ansible_paramiko as paramiko - except ImportError: - HAS_PARAMIKO = False - try: from scp import SCPClient HAS_SCP = True @@ -394,7 +386,7 @@ def main(): 'installed. It can be installed using `pip install pexpect`' ) else: - if not HAS_PARAMIKO: + if paramiko is None: module.fail_json( msg='library paramiko is required when file_pull is False but does not appear to be ' 'installed. It can be installed using `pip install paramiko`' diff --git a/lib/ansible/modules/network/panos/_panos_admpwd.py b/lib/ansible/modules/network/panos/_panos_admpwd.py index 8bf85abd63..1f716c7349 100644 --- a/lib/ansible/modules/network/panos/_panos_admpwd.py +++ b/lib/ansible/modules/network/panos/_panos_admpwd.py @@ -81,20 +81,11 @@ ANSIBLE_METADATA = {'metadata_version': '1.1', 'status': ['deprecated'], 'supported_by': 'community'} - from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.compat.paramiko import paramiko import time import sys -HAS_LIB = True -try: - import paramiko -except ImportError: - try: - import ansible_paramiko as paramiko - except ImportError: - HAS_LIB = False - _PROMPTBUFF = 4096 @@ -189,7 +180,7 @@ def main(): newpassword=dict(no_log=True, required=True) ) module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=True) - if not HAS_LIB: + if paramiko is None: module.fail_json(msg='paramiko is required for this module') ip_address = module.params["ip_address"] diff --git a/lib/ansible/modules/network/panos/_panos_cert_gen_ssh.py b/lib/ansible/modules/network/panos/_panos_cert_gen_ssh.py index 990e861c5c..16e9533327 100644 --- a/lib/ansible/modules/network/panos/_panos_cert_gen_ssh.py +++ b/lib/ansible/modules/network/panos/_panos_cert_gen_ssh.py @@ -87,20 +87,11 @@ ANSIBLE_METADATA = {'metadata_version': '1.1', 'supported_by': 'community'} -from ansible.module_utils.basic import AnsibleModule from ansible.module_utils._text import to_native +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.compat.paramiko import paramiko import time -HAS_LIB = True -try: - import paramiko -except ImportError: - try: - import ansible_paramiko as paramiko - except ImportError: - HAS_LIB = False - - _PROMPTBUFF = 4096 @@ -174,7 +165,7 @@ def main(): ) module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False, required_one_of=[['key_filename', 'password']]) - if not HAS_LIB: + if paramiko is None: module.fail_json(msg='paramiko is required for this module') ip_address = module.params["ip_address"] diff --git a/lib/ansible/playbook/play_context.py b/lib/ansible/playbook/play_context.py index 4159cb96f5..2907e32c15 100644 --- a/lib/ansible/playbook/play_context.py +++ b/lib/ansible/playbook/play_context.py @@ -28,6 +28,7 @@ import sys from ansible import constants as C from ansible import context from ansible.errors import AnsibleError +from ansible.module_utils.compat.paramiko import paramiko from ansible.module_utils.six import iteritems from ansible.playbook.attribute import FieldAttribute from ansible.playbook.base import Base @@ -408,18 +409,12 @@ class PlayContext(Base): conn_type = None if self._attributes['connection'] == 'smart': conn_type = 'ssh' - if sys.platform.startswith('darwin') and self.password: - # due to a current bug in sshpass on OSX, which can trigger - # a kernel panic even for non-privileged users, we revert to - # paramiko on that OS when a SSH password is specified + # see if SSH can support ControlPersist if not use paramiko + if not check_for_controlpersist(self.ssh_executable) and paramiko is not None: conn_type = "paramiko" - else: - # see if SSH can support ControlPersist if not use paramiko - if not check_for_controlpersist(self.ssh_executable): - conn_type = "paramiko" # if someone did `connection: persistent`, default it to using a persistent paramiko connection to avoid problems - elif self._attributes['connection'] == 'persistent': + elif self._attributes['connection'] == 'persistent' and paramiko is not None: conn_type = 'paramiko' if conn_type: diff --git a/lib/ansible/plugins/connection/paramiko_ssh.py b/lib/ansible/plugins/connection/paramiko_ssh.py index e590775ea9..4f8b903cb3 100644 --- a/lib/ansible/plugins/connection/paramiko_ssh.py +++ b/lib/ansible/plugins/connection/paramiko_ssh.py @@ -149,6 +149,7 @@ from ansible.errors import ( AnsibleError, AnsibleFileNotFound, ) +from ansible.module_utils.compat.paramiko import PARAMIKO_IMPORT_ERR, paramiko from ansible.module_utils.six import iteritems from ansible.module_utils.six.moves import input from ansible.plugins.connection import ConnectionBase @@ -168,23 +169,6 @@ Are you sure you want to continue connecting (yes/no)? # SSH Options Regex SETTINGS_REGEX = re.compile(r'(\w+)(?:\s*=\s*|\s+)(.+)') -# prevent paramiko warning noise -- see http://stackoverflow.com/questions/3920502/ -HAVE_PARAMIKO = False -PARAMIKO_IMP_ERR = None -with warnings.catch_warnings(): - warnings.simplefilter("ignore") - try: - import paramiko - HAVE_PARAMIKO = True - except ImportError: - try: - import ansible_paramiko as paramiko - HAVE_PARAMIKO = True - except (ImportError, AttributeError) as err: # paramiko and gssapi are incompatible and raise AttributeError not ImportError - PARAMIKO_IMP_ERR = err - except AttributeError as err: # paramiko and gssapi are incompatible and raise AttributeError not ImportError - PARAMIKO_IMP_ERR = err - class MyAddPolicy(object): """ @@ -313,8 +297,8 @@ class Connection(ConnectionBase): def _connect_uncached(self): ''' activates the connection object ''' - if not HAVE_PARAMIKO: - raise AnsibleError("paramiko is not installed: %s" % to_native(PARAMIKO_IMP_ERR)) + if paramiko is None: + raise AnsibleError("paramiko is not installed: %s" % to_native(PARAMIKO_IMPORT_ERR)) port = self._play_context.port or 22 display.vvv("ESTABLISH PARAMIKO SSH CONNECTION FOR USER: %s on PORT %s TO %s" % (self._play_context.remote_user, port, self._play_context.remote_addr), diff --git a/requirements.txt b/requirements.txt index ac904c49b8..2b77e2f880 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,4 @@ # be suitable) jinja2 PyYAML -paramiko cryptography diff --git a/test/integration/targets/assert/runme.sh b/test/integration/targets/assert/runme.sh index 45ef3aafb5..ca0a858726 100755 --- a/test/integration/targets/assert/runme.sh +++ b/test/integration/targets/assert/runme.sh @@ -25,6 +25,9 @@ run_test() { sed -i -e 's/ *$//' "${OUTFILE}.${testname}.stdout" sed -i -e 's/ *$//' "${OUTFILE}.${testname}.stderr" + # Scrub deprication warning that shows up in Python 2.6 on CentOS 6 + sed -i -e '/RandomPool_DeprecationWarning/d' "${OUTFILE}.${testname}.stderr" + diff -u "${ORIGFILE}.${testname}.stdout" "${OUTFILE}.${testname}.stdout" || diff_failure diff -u "${ORIGFILE}.${testname}.stderr" "${OUTFILE}.${testname}.stderr" || diff_failure } diff --git a/test/integration/targets/callback_default/runme.sh b/test/integration/targets/callback_default/runme.sh index c956ab8903..f52b4367af 100755 --- a/test/integration/targets/callback_default/runme.sh +++ b/test/integration/targets/callback_default/runme.sh @@ -21,6 +21,9 @@ run_test() { { ansible-playbook -i inventory test.yml \ > >(set +x; tee "${OUTFILE}.${testname}.stdout"); } \ 2> >(set +x; tee "${OUTFILE}.${testname}.stderr" >&2) + # Scrub deprication warning that shows up in Python 2.6 on CentOS 6 + sed -i -e '/RandomPool_DeprecationWarning/d' "${OUTFILE}.${testname}.stderr" + diff -u "${ORIGFILE}.${testname}.stdout" "${OUTFILE}.${testname}.stdout" || diff_failure diff -u "${ORIGFILE}.${testname}.stderr" "${OUTFILE}.${testname}.stderr" || diff_failure }