Python 3.8 Controller Minimum (#74013)

changelogs/fragments/74013-controller-py38-min.yml

@@ -0,0 +1,3 @@
+major_changes:
+- Python Controller Requirement - Python 3.8 or newer is required for the control node (the machine that runs Ansible)
+  (https://github.com/ansible/ansible/pull/74013)

docs/docsite/rst/installation_guide/intro_installation.rst

@@ -22,7 +22,7 @@ Before you install Ansible, review the requirements for a control node. Before y
 Control node requirements
 -------------------------
 
-For your control node (the machine that runs Ansible), you can use any machine with Python 2 (version 2.7) or Python 3 (versions 3.5 and higher) installed. ansible-core 2.11 and Ansible 4.0.0 will make Python 3.8 a soft dependency for the control node, but will function with the aforementioned requirements. ansible-core 2.12 and Ansible 5.0.0 will require Python 3.8 or newer to function on the control node. Starting with ansible-core 2.11, the project will only be packaged for Python 3.8 and newer.
+For your control node (the machine that runs Ansible), you can use any machine with Python 3.8 or newer installed.
 This includes Red Hat, Debian, CentOS, macOS, any of the BSDs, and so on.
 Windows is not supported for the control node, read more about this in `Matt Davis's blog post <http://blog.rolpdog.com/2020/03/why-no-ansible-controller-for-windows.html>`_.
 

docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst

@@ -1,15 +1,15 @@
 
-.. _porting_2.12_guide:
+.. _porting_2.12_guide_core:
 
-**************************
-Ansible 2.12 Porting Guide
-**************************
+*******************************
+Ansible-core 2.12 Porting Guide
+*******************************
 
-This section discusses the behavioral changes between Ansible 2.11 and Ansible 2.12.
+This section discusses the behavioral changes between ``ansible-core`` 2.11 and ``ansible-core`` 2.12.
 
 It is intended to assist in updating your playbooks, plugins and other parts of your Ansible infrastructure so they will work with this version of Ansible.
 
-We suggest you read this page along with `Ansible Changelog for 2.12 <https://github.com/ansible/ansible/blob/devel/changelogs/CHANGELOG-v2.12.rst>`_ to understand what updates you may need to make.
+We suggest you read this page along with `ansible-core Changelog for 2.12 <https://github.com/ansible/ansible/blob/devel/changelogs/CHANGELOG-v2.12.rst>`_ to understand what updates you may need to make.
 
 This document is part of a collection on porting. The complete list of porting guides can be found at :ref:`porting guides <porting_guides>`.
 
@@ -45,9 +45,9 @@ See :ref:`interpreter discovery documentation <interpreter_discovery>` for more
 Command Line
 ============
 
+* Python 3.8 on the controller node is a hard requirement for this release. The command line scripts will not function with a lower Python version.
 * ``ansible-vault`` no longer supports ``PyCrypto`` and requires ``cryptography``.
 
-
 Deprecated
 ==========
 

lib/ansible/cli/scripts/ansible_cli_stub.py

@@ -29,19 +29,23 @@ import shutil
 import sys
 import traceback
 
+# Used for determining if the system is running a new enough python version
+# and should only restrict on our documented minimum versions
+_PY38_MIN = sys.version_info[:2] >= (3, 8)
+if not _PY38_MIN:
+    raise SystemExit(
+        'ERROR: Ansible requires Python 3.8 or newer on the controller. '
+        'Current version: %s' % ''.join(sys.version.splitlines())
+    )
+
+
+# These lines appear after the PY38 check, to ensure the "friendly" error happens before
+# any invalid syntax appears in other files that may get imported
 from ansible import context
 from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
 from ansible.module_utils._text import to_text
 
-
-# Used for determining if the system is running a new enough python version
-# and should only restrict on our documented minimum versions
-_PY38_MIN = sys.version_info[:2] >= (3, 8)
-_PY3_MIN = sys.version_info[:2] >= (3, 5)
-_PY2_MIN = (2, 6) <= sys.version_info[:2] < (3,)
-_PY_MIN = _PY3_MIN or _PY2_MIN
-if not _PY_MIN:
-    raise SystemExit('ERROR: Ansible requires a minimum of Python2 version 2.6 or Python3 version 3.5. Current version: %s' % ''.join(sys.version.splitlines()))
+from pathlib import Path
 
 
 class LastResort(object):
@@ -67,19 +71,10 @@ if __name__ == '__main__':
     initialize_locale()
 
     cli = None
-    me = os.path.basename(sys.argv[0])
+    me = Path(sys.argv[0]).name
 
     try:
         display = Display()
-        if C.CONTROLLER_PYTHON_WARNING and not _PY38_MIN:
-            display.deprecated(
-                (
-                    'Ansible will require Python 3.8 or newer on the controller starting with Ansible 2.12. '
-                    'Current version: %s' % ''.join(sys.version.splitlines())
-                ),
-                version='2.12',
-                collection_name='ansible.builtin',
-            )
         display.debug("starting run")
 
         sub = None
@@ -111,16 +106,16 @@ if __name__ == '__main__':
             else:
                 raise
 
-        b_ansible_dir = os.path.expanduser(os.path.expandvars(b"~/.ansible"))
+        ansible_dir = Path("~/.ansible").expanduser()
         try:
-            os.mkdir(b_ansible_dir, 0o700)
+            ansible_dir.mkdir(mode=0o700)
         except OSError as exc:
             if exc.errno != errno.EEXIST:
-                display.warning("Failed to create the directory '%s': %s"
-                                % (to_text(b_ansible_dir, errors='surrogate_or_replace'),
-                                   to_text(exc, errors='surrogate_or_replace')))
+                display.warning(
+                    "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace'))
+                )
         else:
-            display.debug("Created the '%s' directory" % to_text(b_ansible_dir, errors='surrogate_or_replace'))
+            display.debug("Created the '%s' directory" % ansible_dir)
 
         try:
             args = [to_text(a, errors='surrogate_or_strict') for a in sys.argv]

lib/ansible/config/base.yml

@@ -558,15 +558,6 @@ CALLABLE_ACCEPT_LIST:
     section: defaults
     version_added: '2.11'
   type: list
-CONTROLLER_PYTHON_WARNING:
-  name: Running Older than Python 3.8 Warning
-  default: True
-  description: Toggle to control showing warnings related to running a Python version
-               older than Python 3.8 on the controller
-  env: [{name: ANSIBLE_CONTROLLER_PYTHON_WARNING}]
-  ini:
-  - {key: controller_python_warning, section: defaults}
-  type: boolean
 DEFAULT_CALLBACK_PLUGIN_PATH:
   name: Callback Plugins Path
   default: ~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback

lib/ansible/config/manager.py

@@ -19,7 +19,7 @@ from ansible.errors import AnsibleOptionsError, AnsibleError
 from ansible.module_utils._text import to_text, to_bytes, to_native
 from ansible.module_utils.common._collections_compat import Mapping, Sequence
 from ansible.module_utils.common.yaml import yaml_load
-from ansible.module_utils.six import PY3, string_types
+from ansible.module_utils.six import string_types
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.parsing.quoting import unquote
@@ -323,21 +323,14 @@ class ConfigManager(object):
         ftype = get_config_type(cfile)
         if cfile is not None:
             if ftype == 'ini':
-                kwargs = {}
-                if PY3:
-                    kwargs['inline_comment_prefixes'] = (';',)
-                self._parsers[cfile] = configparser.ConfigParser(**kwargs)
+                self._parsers[cfile] = configparser.ConfigParser(inline_comment_prefixes=(';',))
                 with open(to_bytes(cfile), 'rb') as f:
                     try:
                         cfg_text = to_text(f.read(), errors='surrogate_or_strict')
                     except UnicodeError as e:
                         raise AnsibleOptionsError("Error reading config file(%s) because the config file was not utf8 encoded: %s" % (cfile, to_native(e)))
                 try:
-                    if PY3:
-                        self._parsers[cfile].read_string(cfg_text)
-                    else:
-                        cfg_file = io.StringIO(cfg_text)
-                        self._parsers[cfile].readfp(cfg_file)
+                    self._parsers[cfile].read_string(cfg_text)
                 except configparser.Error as e:
                     raise AnsibleOptionsError("Error reading config file (%s): %s" % (cfile, to_native(e)))
             # FIXME: this should eventually handle yaml config files

lib/ansible/executor/task_queue_manager.py

@@ -32,7 +32,7 @@ from ansible.errors import AnsibleError
 from ansible.executor.play_iterator import PlayIterator
 from ansible.executor.stats import AggregateStats
 from ansible.executor.task_result import TaskResult
-from ansible.module_utils.six import PY3, string_types
+from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_text, to_native
 from ansible.playbook.play_context import PlayContext
 from ansible.playbook.task import Task
@@ -60,8 +60,7 @@ class CallbackSend:
 
 class FinalQueue(multiprocessing.queues.Queue):
     def __init__(self, *args, **kwargs):
-        if PY3:
-            kwargs['ctx'] = multiprocessing_context
+        kwargs['ctx'] = multiprocessing_context
         super(FinalQueue, self).__init__(*args, **kwargs)
 
     def send_callback(self, method_name, *args, **kwargs):

lib/ansible/parsing/vault/__init__.py

@@ -54,7 +54,7 @@ except ImportError:
 
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible import constants as C
-from ansible.module_utils.six import PY3, binary_type
+from ansible.module_utils.six import binary_type
 # Note: on py2, this zip is izip not the list based zip() builtin
 from ansible.module_utils.six.moves import zip
 from ansible.module_utils._text import to_bytes, to_text, to_native
@@ -1015,10 +1015,7 @@ class VaultEditor:
 
         try:
             if filename == '-':
-                if PY3:
-                    data = sys.stdin.buffer.read()
-                else:
-                    data = sys.stdin.read()
+                data = sys.stdin.buffer.read()
             else:
                 with open(filename, "rb") as fh:
                     data = fh.read()
@@ -1258,10 +1255,7 @@ class VaultAES256:
 
         result = 0
         for b_x, b_y in zip(b_a, b_b):
-            if PY3:
-                result |= b_x ^ b_y
-            else:
-                result |= ord(b_x) ^ ord(b_y)
+            result |= b_x ^ b_y
         return result == 0
 
     @classmethod

lib/ansible/parsing/yaml/dumper.py

@@ -21,7 +21,7 @@ __metaclass__ = type
 
 import yaml
 
-from ansible.module_utils.six import PY3, text_type, binary_type
+from ansible.module_utils.six import text_type, binary_type
 from ansible.module_utils.common.yaml import SafeDumper
 from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode
 from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes
@@ -46,18 +46,12 @@ def represent_vault_encrypted_unicode(self, data):
     return self.represent_scalar(u'!vault', data._ciphertext.decode(), style='|')
 
 
-if PY3:
-    def represent_unicode(self, data):
-        return yaml.representer.SafeRepresenter.represent_str(self, text_type(data))
+def represent_unicode(self, data):
+    return yaml.representer.SafeRepresenter.represent_str(self, text_type(data))
 
-    def represent_binary(self, data):
-        return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data))
-else:
-    def represent_unicode(self, data):
-        return yaml.representer.SafeRepresenter.represent_unicode(self, text_type(data))
 
-    def represent_binary(self, data):
-        return yaml.representer.SafeRepresenter.represent_str(self, binary_type(data))
+def represent_binary(self, data):
+    return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data))
 
 
 def represent_undefined(self, data):

lib/ansible/parsing/yaml/objects.py

@@ -58,17 +58,7 @@ class AnsibleBaseYAMLObject(object):
     ansible_pos = property(_get_ansible_position, _set_ansible_position)
 
 
-# try to always use orderddict with yaml, after py3.6 the dict type already does this
-odict = dict
-if sys.version_info[:2] < (3, 7):
-    # if python 2.7 or py3 < 3.7
-    try:
-        from collections import OrderedDict as odict
-    except ImportError:
-        pass
-
-
-class AnsibleMapping(AnsibleBaseYAMLObject, odict):
+class AnsibleMapping(AnsibleBaseYAMLObject, dict):
     ''' sub class for dictionaries '''
     pass
 
@@ -314,12 +304,7 @@ class AnsibleVaultEncryptedUnicode(Sequence, AnsibleBaseYAMLObject):
     def lstrip(self, chars=None):
         return self.data.lstrip(chars)
 
-    try:
-        # PY3
-        maketrans = str.maketrans
-    except AttributeError:
-        # PY2
-        maketrans = string.maketrans
+    maketrans = str.maketrans
 
     def partition(self, sep):
         return self.data.partition(sep)

lib/ansible/playbook/play_context.py

@@ -199,10 +199,8 @@ class PlayContext(Base):
         # loop through a subset of attributes on the task object and set
         # connection fields based on their values
         for attr in TASK_ATTRIBUTE_OVERRIDES:
-            if hasattr(task, attr):
-                attr_val = getattr(task, attr)
-                if attr_val is not None:
-                    setattr(new_info, attr, attr_val)
+            if (attr_val := getattr(task, attr, None)) is not None:
+                setattr(new_info, attr, attr_val)
 
         # next, use the MAGIC_VARIABLE_MAPPING dictionary to update this
         # connection info object with 'magic' variables from the variable list.

lib/ansible/plugins/callback/__init__.py

@@ -22,12 +22,11 @@ __metaclass__ = type
 import difflib
 import json
 import sys
-
+from collections import OrderedDict
 from copy import deepcopy
 
 from ansible import constants as C
 from ansible.module_utils.common._collections_compat import MutableMapping
-from ansible.module_utils.six import PY3
 from ansible.module_utils._text import to_text
 from ansible.parsing.ajson import AnsibleJSONEncoder
 from ansible.plugins import AnsiblePlugin, get_plugin_class
@@ -35,13 +34,6 @@ from ansible.utils.color import stringc
 from ansible.utils.display import Display
 from ansible.vars.clean import strip_internal_keys, module_response_deepcopy
 
-if PY3:
-    # OrderedDict is needed for a backwards compat shim on Python3.x only
-    # https://github.com/ansible/ansible/pull/49512
-    from collections import OrderedDict
-else:
-    OrderedDict = None
-
 global_display = Display()
 
 

lib/ansible/template/native_helpers.py

@@ -15,7 +15,7 @@ from jinja2.runtime import StrictUndefined
 from ansible.module_utils._text import to_text
 from ansible.module_utils.common.collections import is_sequence, Mapping
 from ansible.module_utils.common.text.converters import container_to_text
-from ansible.module_utils.six import PY2, text_type, string_types
+from ansible.module_utils.six import text_type, string_types
 from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
 from ansible.utils.native_jinja import NativeJinjaText
 
@@ -84,9 +84,6 @@ def ansible_native_concat(nodes):
 
     try:
         out = literal_eval(out)
-        if PY2:
-            # ensure bytes are not returned back into Ansible from templating
-            out = container_to_text(out)
         return out
     except (ValueError, SyntaxError, MemoryError):
         return out

lib/ansible/template/safe_eval.py

@@ -23,9 +23,8 @@ import sys
 
 from ansible import constants as C
 from ansible.module_utils.common.text.converters import container_to_text, to_native
-from ansible.module_utils.six import string_types, PY2
+from ansible.module_utils.six import string_types
 from ansible.module_utils.six.moves import builtins
-from ansible.plugins.loader import filter_loader, test_loader
 
 
 def safe_eval(expr, locals=None, include_exceptions=False):
@@ -64,6 +63,7 @@ def safe_eval(expr, locals=None, include_exceptions=False):
             ast.BinOp,
             # ast.Call,
             ast.Compare,
+            ast.Constant,
             ast.Dict,
             ast.Div,
             ast.Expression,
@@ -72,6 +72,7 @@ def safe_eval(expr, locals=None, include_exceptions=False):
             ast.Mult,
             ast.Num,
             ast.Name,
+            ast.Set,
             ast.Str,
             ast.Sub,
             ast.USub,
@@ -80,47 +81,7 @@ def safe_eval(expr, locals=None, include_exceptions=False):
         )
     )
 
-    # AST node types were expanded after 2.6
-    if sys.version_info[:2] >= (2, 7):
-        SAFE_NODES.update(
-            set(
-                (ast.Set,)
-            )
-        )
-
-    # And in Python 3.4 too
-    if sys.version_info[:2] >= (3, 4):
-        SAFE_NODES.update(
-            set(
-                (ast.NameConstant,)
-            )
-        )
-
-    # And in Python 3.6 too, although not encountered until Python 3.8, see https://bugs.python.org/issue32892
-    if sys.version_info[:2] >= (3, 6):
-        SAFE_NODES.update(
-            set(
-                (ast.Constant,)
-            )
-        )
-
-    filter_list = []
-    for filter_ in filter_loader.all():
-        try:
-            filter_list.extend(filter_.filters().keys())
-        except Exception:
-            # This is handled and displayed in JinjaPluginIntercept._load_ansible_plugins
-            continue
-
-    test_list = []
-    for test in test_loader.all():
-        try:
-            test_list.extend(test.tests().keys())
-        except Exception:
-            # This is handled and displayed in JinjaPluginIntercept._load_ansible_plugins
-            continue
-
-    CALL_ENABLED = C.CALLABLE_ACCEPT_LIST + filter_list + test_list
+    CALL_ENABLED = []
 
     class CleansingNodeVisitor(ast.NodeVisitor):
         def generic_visit(self, node, inside_call=False):
@@ -153,10 +114,6 @@ def safe_eval(expr, locals=None, include_exceptions=False):
         # callables (and other identifiers) are recognized.  this is in
         # addition to the filtering of builtins done in CleansingNodeVisitor
         result = eval(compiled, OUR_GLOBALS, dict(locals))
-        if PY2:
-            # On Python 2 u"{'key': 'value'}" is evaluated to {'key': 'value'},
-            # ensure it is converted to {u'key': u'value'}.
-            result = container_to_text(result)
 
         if include_exceptions:
             return (result, None)

setup.py

@@ -333,7 +333,7 @@ static_setup_params = dict(
     license='GPLv3+',
     # Ansible will also make use of a system copy of python-six and
     # python-selectors2 if installed but use a Bundled copy if it's not.
-    python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*',
+    python_requires='>=3.8',
     package_dir={'': 'lib',
                  'ansible_test': 'test/lib/ansible_test'},
     packages=find_packages('lib') + find_packages('test/lib'),
@@ -347,14 +347,10 @@ static_setup_params = dict(
         'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
         'Natural Language :: English',
         'Operating System :: POSIX',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'Topic :: System :: Installation/Setup',
         'Topic :: System :: Systems Administration',
         'Topic :: Utilities',

test/lib/ansible_test/_internal/ansible_util.py

@@ -101,7 +101,6 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_CONFIG=ansible_config,
         ANSIBLE_LIBRARY='/dev/null',
         ANSIBLE_DEVEL_WARNING='false',  # Don't show warnings that CI is running devel
-        ANSIBLE_CONTROLLER_PYTHON_WARNING='false',  # Don't show warnings in CI for old controller Python
         ANSIBLE_JINJA2_NATIVE_WARNING='false',  # Don't show warnings in CI for old Jinja for native
         PYTHONPATH=get_ansible_python_path(args),
         PAGER='/bin/cat',

test/sanity/ignore.txt

@@ -8,7 +8,6 @@ examples/scripts/my_test_info.py shebang # example module but not in a normal mo
 examples/scripts/upgrade_to_ps3.ps1 pslint:PSCustomUseLiteralPath
 examples/scripts/upgrade_to_ps3.ps1 pslint:PSUseApprovedVerbs
 lib/ansible/cli/console.py pylint:disallowed-name
-lib/ansible/cli/scripts/ansible_cli_stub.py pylint:ansible-deprecated-version
 lib/ansible/cli/scripts/ansible_cli_stub.py shebang
 lib/ansible/cli/scripts/ansible_connection_cli_stub.py shebang
 lib/ansible/config/base.yml no-unwanted-files