Remove use of simplejson throughout code base (#43548)

* Remove use of simplejson throughout code base. Fixes #42761

* Address failing tests

* Remove simplejson from contrib and other outlying files

* Add changelog fragment for simplejson removal

changelogs/fragments/remove-simplejson.yaml

@@ -0,0 +1,2 @@
+major_changes:
+- Remove support for simplejson (https://github.com/ansible/ansible/issues/42761)

contrib/inventory/abiquo.py

@@ -46,10 +46,7 @@ import sys
 import time
 import ConfigParser
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 from ansible.module_utils.urls import open_url
 

contrib/inventory/apache-libcloud.py

@@ -42,10 +42,7 @@ from libcloud.compute.types import Provider
 from libcloud.compute.providers import get_driver
 import libcloud.security as sec
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class LibcloudInventory(object):

contrib/inventory/brook.py

@@ -84,10 +84,7 @@ try:
 except ImportError:
     from configparser import ConfigParser
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     import libbrook

contrib/inventory/cloudforms.py

@@ -31,10 +31,7 @@ from requests.auth import HTTPBasicAuth
 import warnings
 from ansible.errors import AnsibleError
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class CloudFormsInventory(object):

contrib/inventory/cloudstack.py

@@ -74,12 +74,7 @@ from __future__ import print_function
 
 import sys
 import argparse
-
-try:
 import json
-except:
-    import simplejson as json
-
 
 try:
     from cs import CloudStack, CloudStackException, read_config

contrib/inventory/cobbler.py

@@ -65,10 +65,7 @@ import re
 from time import time
 import xmlrpclib
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 from six import iteritems
 

contrib/inventory/collins.py

@@ -75,10 +75,7 @@ import sys
 from time import time
 import traceback
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 from six import iteritems
 from six.moves.urllib.parse import urlencode

contrib/inventory/consul_io.py

@@ -191,10 +191,7 @@ if os.getenv('ANSIBLE_INVENTORY_CONSUL_IO_LOG_ENABLED'):
     setup_logging()
 
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     import consul

contrib/inventory/digital_ocean.py

@@ -149,10 +149,7 @@ try:
 except ImportError:
     import configparser as ConfigParser
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class DoManager:

contrib/inventory/ec2.py

@@ -179,10 +179,7 @@ except ImportError:
 from six.moves import configparser
 from collections import defaultdict
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 DEFAULTS = {
     'all_elasticache_clusters': 'False',

contrib/inventory/fleet.py

@@ -29,10 +29,7 @@ import subprocess
 import re
 import string
 from optparse import OptionParser
-try:
 import json
-except:
-    import simplejson as json
 
 # Options
 # ------------------------------

contrib/inventory/gce.py

@@ -101,10 +101,7 @@ else:
 import logging
 logging.getLogger('libcloud.common.google').addHandler(logging.NullHandler())
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     from libcloud.compute.types import Provider

contrib/inventory/landscape.py

@@ -40,10 +40,7 @@ import sys
 
 from landscape_api.base import API, HTTPError
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 _key = 'landscape'
 

contrib/inventory/linode.py

@@ -81,10 +81,7 @@ import sys
 import argparse
 from time import time
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     from chube import load_chube_config

contrib/inventory/openshift.py

@@ -28,10 +28,7 @@ version_added: None
 author: Michael Scherer
 '''
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 import os
 import os.path
 import sys

contrib/inventory/openstack_inventory.py

@@ -59,10 +59,7 @@ import time
 from distutils.version import StrictVersion
 from io import StringIO
 
-try:
 import json
-except:
-    import simplejson as json
 
 import openstack as sdk
 from openstack.cloud import inventory as sdk_inventory

contrib/inventory/ovirt.py

@@ -71,11 +71,7 @@ import argparse
 import ConfigParser
 from collections import defaultdict
 
-try:
 import json
-except ImportError:
-    # noinspection PyUnresolvedReferences,PyPackageRequirements
-    import simplejson as json
 
 try:
     # noinspection PyUnresolvedReferences

contrib/inventory/ovirt4.py

@@ -70,10 +70,7 @@ try:
 except ImportError:
     import configparser
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     import ovirtsdk4 as sdk

contrib/inventory/packet_net.py

@@ -55,10 +55,7 @@ except ImportError as e:
 import traceback
 
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 ini_section = 'packet'

contrib/inventory/proxmox.py

@@ -25,10 +25,7 @@
 #
 # { "groups": ["utility", "databases"], "a": false, "b": true }
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 import os
 import sys
 from optparse import OptionParser

contrib/inventory/rax.py

@@ -155,10 +155,7 @@ import ConfigParser
 
 from six import iteritems
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     import pyrax

contrib/inventory/rudder.py

@@ -59,10 +59,7 @@ from time import time
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils.six.moves.urllib.parse import urlparse
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class RudderInventory(object):

contrib/inventory/scaleway.py

@@ -47,10 +47,7 @@ import sys
 import time
 import traceback
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 EMPTY_GROUP = {
     'children': [],

contrib/inventory/serf.py

@@ -38,10 +38,7 @@ import sys
 # https://pypi.org/project/serfclient/
 from serfclient import SerfClient, EnvironmentConfig
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 _key = 'serf'
 

contrib/inventory/softlayer.py

@@ -36,10 +36,7 @@ import SoftLayer
 import re
 import argparse
 import itertools
-try:
 import json
-except:
-    import simplejson as json
 
 
 class SoftLayerInventory(object):

contrib/inventory/spacewalk.py

@@ -48,13 +48,10 @@ import time
 from optparse import OptionParser
 import subprocess
 import ConfigParser
+import json
 
 from six import iteritems
 
-try:
-    import json
-except:
-    import simplejson as json
 
 base_dir = os.path.dirname(os.path.realpath(__file__))
 default_ini_file = os.path.join(base_dir, "spacewalk.ini")

contrib/inventory/ssh_config.py

@@ -45,10 +45,7 @@ import os.path
 import sys
 from collections import MutableSequence
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 import paramiko
 

contrib/inventory/stacki.py

@@ -49,10 +49,7 @@ import sys
 import yaml
 from distutils.version import StrictVersion
 
-try:
 import json
-except:
-    import simplejson as json
 
 try:
     import requests

contrib/inventory/vagrant.py

@@ -40,10 +40,7 @@ import re
 from paramiko import SSHConfig
 from optparse import OptionParser
 from collections import defaultdict
-try:
 import json
-except Exception:
-    import simplejson as json
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.six.moves import StringIO

contrib/inventory/vbox.py

@@ -18,10 +18,7 @@
 import sys
 from subprocess import Popen, PIPE
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class SetEncoder(json.JSONEncoder):

contrib/inventory/zabbix.py

@@ -49,10 +49,7 @@ except:
           file=sys.stderr)
     sys.exit(1)
 
-try:
 import json
-except:
-    import simplejson as json
 
 
 class ZabbixInventory(object):

docs/docsite/rst/user_guide/intro_bsd.rst

@@ -16,7 +16,8 @@ versions of sshpass do not deal particularly well with BSD login prompts, so whe
     [freebsd]
     mybsdhost1 ansible_connection=paramiko
 
-Ansible is agentless by default, however certain software is required on the target machines. Using Python 2.4 on the agents requires an additional py-simplejson package/library to be installed, however this library is already included in Python 2.5 and above.
+Ansible is agentless by default, however certain software is required on the target machines.
+
 Operating without Python is possible with the ``raw`` module. Although this module can be used to bootstrap Ansible and install Python on BSD variants (see below), it is very limited and the use of Python is required to make full use of Ansible's features.
 
 .. _bootstrap_bsd:

hacking/test-module

@@ -44,10 +44,7 @@ import ansible.constants as C
 from ansible.module_utils._text import to_native, to_text
 from ansible.template import Templar
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 def parse():

lib/ansible/module_utils/basic.py

@@ -109,26 +109,14 @@ NoneType = type(None)
 try:
     import json
     # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
     try:
         if not isinstance(json.loads, types.FunctionType) or not isinstance(json.dumps, types.FunctionType):
             raise ImportError
     except AttributeError:
         raise ImportError
 except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        print('\n{"msg": "Error: ansible requires the stdlib json or simplejson module, neither was found!", "failed": true}')
+    print('\n{"msg": "Error: ansible requires the stdlib json and was not found!", "failed": true}')
     sys.exit(1)
-    except SyntaxError:
-        print('\n{"msg": "SyntaxError: probably due to installed simplejson being for a different python version", "failed": true}')
-        sys.exit(1)
-    else:
-        sj_version = json.__version__.split('.')
-        if sj_version < ['1', '6']:
-            # Version 1.5 released 2007-01-18 does not have the encoding parameter which we need
-            print('\n{"msg": "Error: Ansible requires the stdlib json or simplejson >= 1.6.  Neither was found!", "failed": true}')
 
 AVAILABLE_HASH_ALGORITHMS = dict()
 try:

lib/ansible/module_utils/ipa.py

@@ -27,10 +27,7 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 import re
 from ansible.module_utils._text import to_bytes, to_native, to_text

lib/ansible/module_utils/json_utils.py

@@ -23,10 +23,8 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
-try:
+
 import json
-except ImportError:
-    import simplejson as json
 
 
 # NB: a copy of this function exists in ../../modules/core/async_wrapper.py. Ensure any

lib/ansible/module_utils/lxd.py

@@ -39,10 +39,7 @@ from ansible.module_utils._text import to_text
 HTTPConnection = http_client.HTTPConnection
 HTTPSConnection = http_client.HTTPSConnection
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 class UnixHTTPConnection(HTTPConnection):

lib/ansible/module_utils/netapp.py

@@ -27,10 +27,7 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 from ansible.module_utils.six.moves.urllib.error import HTTPError
 from ansible.module_utils.urls import open_url

lib/ansible/module_utils/network/f5/icontrol.py

@@ -19,10 +19,7 @@ from ansible.module_utils.urls import urllib_error
 from ansible.module_utils._text import to_native
 from ansible.module_utils.six import PY3
 
-try:
 import json as _json
-except ImportError:
-    import simplejson as _json
 
 try:
     from library.module_utils.network.f5.common import F5ModuleError

lib/ansible/modules/commands/raw.py

@@ -30,10 +30,9 @@ options:
     version_added: "1.0"
 description:
      - Executes a low-down and dirty SSH command, not going through the module
-       subsystem. This is useful and should only be done in two cases. The
-       first case is installing C(python-simplejson) on older (Python 2.4 and
-       before) hosts that need it as a dependency to run modules, since nearly
-       all core modules require it. Another is speaking to any devices such as
+       subsystem. This is useful and should only be done in a few cases. A common
+       case is installing C(python) on a system without python installed by default.
+       Another is speaking to any devices such as
        routers that do not have any Python installed. In any other case, using
        the M(shell) or M(command) module is much more appropriate. Arguments
        given to C(raw) are run directly through the configured remote shell.
@@ -58,9 +57,6 @@ author:
 '''
 
 EXAMPLES = '''
-- name: Bootstrap a legacy python 2.4 host
-  raw: yum -y install python-simplejson
-
 - name: Bootstrap a host without python2 installed
   raw: dnf install -y python2 python2-dnf libselinux-python
 
@@ -70,5 +66,5 @@ EXAMPLES = '''
     executable: /bin/bash
 
 - name: safely use templated variables. Always use quote filter to avoid injection issues.
-  raw: "{{package_mgr|quote}} {{pkg_flags|quote}} install {{python_simplejson|quote}}"
+  raw: "{{package_mgr|quote}} {{pkg_flags|quote}} install {{python|quote}}"
 '''

lib/ansible/modules/monitoring/logicmonitor.py

@@ -497,6 +497,7 @@ EXAMPLES = '''
 '''
 
 import datetime
+import json
 import os
 import platform
 import socket
@@ -508,28 +509,6 @@ from ansible.module_utils.six.moves.urllib.parse import urlencode
 from ansible.module_utils.urls import open_url
 
 
-HAS_LIB_JSON = True
-try:
-    import json
-    # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
-    try:
-        if (
-            not isinstance(json.loads, types.FunctionType) or
-            not isinstance(json.dumps, types.FunctionType)
-        ):
-            raise ImportError
-    except AttributeError:
-        raise ImportError
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        HAS_LIB_JSON = False
-    except SyntaxError:
-        HAS_LIB_JSON = False
-
-
 class LogicMonitor(object):
 
     def __init__(self, module, **params):
@@ -2148,9 +2127,6 @@ def main():
         supports_check_mode=True
     )
 
-    if HAS_LIB_JSON is not True:
-        module.fail_json(msg="Unable to load JSON library")
-
     selector(module)
 
 

lib/ansible/modules/monitoring/logicmonitor_facts.py

@@ -113,30 +113,10 @@ RETURN = '''
 ...
 '''
 
+import json
 import socket
 import types
 
-HAS_LIB_JSON = True
-try:
-    import json
-    # Detect the python-json library which is incompatible
-    # Look for simplejson if that's the case
-    try:
-        if (
-            not isinstance(json.loads, types.FunctionType) or
-            not isinstance(json.dumps, types.FunctionType)
-        ):
-            raise ImportError
-    except AttributeError:
-        raise ImportError
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        HAS_LIB_JSON = False
-    except SyntaxError:
-        HAS_LIB_JSON = False
-
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.six.moves.urllib.parse import urlencode
 from ansible.module_utils._text import to_native
@@ -577,9 +557,6 @@ def main():
         supports_check_mode=True
     )
 
-    if HAS_LIB_JSON is not True:
-        module.fail_json(msg="Unable to load JSON library")
-
     selector(module)
 
 

lib/ansible/modules/monitoring/sensu_silence.py

@@ -95,11 +95,7 @@ EXAMPLES = '''
 RETURN = '''
 '''
 
-
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.urls import fetch_url

lib/ansible/modules/monitoring/sensu_subscription.py

@@ -64,6 +64,8 @@ EXAMPLES = '''
 - name: unsubscribe from common checks
   sensu_subscription: name=common state=absent
 '''
+
+import json
 import traceback
 
 from ansible.module_utils.basic import AnsibleModule
@@ -74,11 +76,6 @@ def sensu_subscription(module, path, name, state='present', backup=False):
     changed = False
     reasons = []
 
-    try:
-        import json
-    except ImportError:
-        import simplejson as json
-
     try:
         config = json.load(open(path))
     except IOError as e:

lib/ansible/modules/packaging/language/npm.py

@@ -123,14 +123,7 @@ import re
 
 from ansible.module_utils.basic import AnsibleModule
 
-try:
 import json
-except ImportError:
-    try:
-        import simplejson as json
-    except ImportError:
-        # Let snippet from module_utils/basic.py return a proper error in this case
-        pass
 
 
 class Npm(object):

lib/ansible/modules/system/service.py

@@ -114,6 +114,7 @@ EXAMPLES = '''
 '''
 
 import glob
+import json
 import os
 import platform
 import re
@@ -124,11 +125,6 @@ import subprocess
 import tempfile
 import time
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
 # The distutils module is not shipped with SUNWPython on Solaris.
 # It's in the SUNWPython-devel package which also contains development files
 # that don't belong on production boxes.  Since our Solaris code doesn't

lib/ansible/modules/utilities/logic/async_wrapper.py

@@ -8,10 +8,7 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 import shlex
 import shutil
 import os

lib/ansible/parsing/utils/jsonify.py

@@ -19,10 +19,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 
 def jsonify(result, format=False):

lib/ansible/plugins/action/__init__.py

@@ -575,8 +575,8 @@ class ActionBase(with_metaclass(ABCMeta, object)):
                 x = "2"  # cannot read file
             elif errormsg.endswith(u'MODULE FAILURE'):
                 x = "4"  # python not found or module uncaught exception
-            elif 'json' in errormsg or 'simplejson' in errormsg:
-                x = "5"  # json or simplejson modules needed
+            elif 'json' in errormsg:
+                x = "5"  # json module needed
         finally:
             return x  # pylint: disable=lost-exception
 

lib/ansible/plugins/action/fetch.py

@@ -159,7 +159,7 @@ class ActionModule(ActionBase):
                 elif remote_checksum == '4':
                     result['msg'] = "python isn't present on the system.  Unable to compute checksum"
                 elif remote_checksum == '5':
-                    result['msg'] = "stdlib json or simplejson was not found on the remote machine. Only the raw module can work without those installed"
+                    result['msg'] = "stdlib json was not found on the remote machine. Only the raw module can work without those installed"
                 # Historically, these don't fail because you may want to transfer
                 # a log file that possibly MAY exist but keep going to fetch other
                 # log files. Today, this is better achieved by adding

lib/ansible/plugins/filter/core.py

@@ -58,6 +58,12 @@ from ansible.utils.hashing import md5s, checksum_s
 from ansible.utils.unicode import unicode_wrap
 from ansible.utils.vars import merge_hash
 
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
 UUID_NAMESPACE_ANSIBLE = uuid.UUID('361E6D51-FAEC-444A-9079-341386DA8E2E')
 
 
@@ -80,25 +86,11 @@ def to_json(a, *args, **kw):
 
 def to_nice_json(a, indent=4, *args, **kw):
     '''Make verbose, human readable JSON'''
-    # python-2.6's json encoder is buggy (can't encode hostvars)
-    if sys.version_info < (2, 7):
-        try:
-            import simplejson
-        except ImportError:
-            pass
-        else:
-            try:
-                major = int(simplejson.__version__.split('.')[0])
-            except Exception:
-                pass
-            else:
-                if major >= 2:
-                    return simplejson.dumps(a, default=AnsibleJSONEncoder.default, indent=indent, sort_keys=True, *args, **kw)
-
     try:
         return json.dumps(a, indent=indent, sort_keys=True, cls=AnsibleJSONEncoder, *args, **kw)
-    except Exception:
+    except Exception as e:
         # Fallback to the to_json filter
+        display.warning(u'Unable to convert data using to_nice_json, falling back to to_json: %s' % to_text(e))
         return to_json(a, *args, **kw)
 
 

lib/ansible/plugins/lookup/consul_kv.py

@@ -67,10 +67,7 @@ from ansible.module_utils.six.moves.urllib.parse import urlparse
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.plugins.lookup import LookupBase
 
-try:
 import json
-except ImportError:
-    import simplejson as json
 
 try:
     import consul

packaging/port/sysutils/ansible/Makefile

@@ -13,11 +13,10 @@ COMMENT=	Ansible ssh based config management framework
 LICENSE=	GPLv3
 LICENSE_FILE=	${WRKSRC}/COPYING
 
-RUN_DEPENDS=	python>2.5:${PORTSDIR}/lang/python \
+RUN_DEPENDS=	python>2.7:${PORTSDIR}/lang/python \
                ${PORTSDIR}/devel/py-Jinja2 \
                ${PORTSDIR}/devel/py-yaml \
                ${PORTSDIR}/security/py-paramiko \
-               ${PORTSDIR}/devel/py-simplejson \
                ${PORTSDIR}/security/py-pycrypto
 
 OPTIONS_DEFINE=	EXAMPLES

test/units/module_utils/basic/test_imports.py

@@ -66,18 +66,15 @@ class TestImports(ModuleTestCase):
         def _mock_import(name, *args, **kwargs):
             if name == 'json':
                 raise ImportError
-            elif name == 'simplejson':
-                sj = MagicMock()
-                sj.__version__ = '3.10.0'
-                return sj
             return realimport(name, *args, **kwargs)
 
         self.clear_modules(['json', 'ansible.module_utils.basic'])
-        mod = builtins.__import__('ansible.module_utils.basic')
+        builtins.__import__('ansible.module_utils.basic')
 
         self.clear_modules(['json', 'ansible.module_utils.basic'])
         mock_import.side_effect = _mock_import
-        mod = builtins.__import__('ansible.module_utils.basic')
+        with self.assertRaises(SystemExit):
+            builtins.__import__('ansible.module_utils.basic')
 
     # FIXME: doesn't work yet
     # @patch.object(builtins, 'bytes')