Overhaul ansible-test integration tests. (#76111)

* Overhaul ansible-test integration tests.
* ansible-test - Fix import test pyyaml usage.
* ansible-test - Remove unused import.
* ansible-test - Fix traceback when pip is unavailable.
* ansible-test - Fix typo in port forwarding message.
* ansible-test - Fix controller logic in requirements install.
* Fix unit tests in ansible-test integration test.

Unit tests are now run for available Python versions which
provide `virtualenv` (Python 2.x) or `venv` (Python 3.x).

changelogs/fragments/ansible-test-requirements-target.yaml

@@ -0,0 +1,3 @@
+bugfixes:
+  - ansible-test - Target integration test requirements are now correctly installed for target environments running on the controller.
+  - ansible-test - Automatic target requirements installation is now based on the target environment instead of the controller environment.

changelogs/fragments/ansible-test-sanity-import-no-pip.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Fix traceback in ``import`` sanity test on Python 2.7 when ``pip`` is not available.

changelogs/fragments/ansible-test-sanity-import.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Fix installation and usage of ``pyyaml`` requirement for the ``import`` sanity test for collections.

test/integration/targets/ansible-test/aliases

@@ -1,2 +1,3 @@
-shippable/posix/group1
+shippable/posix/group1  # runs in the distro test containers
+shippable/generic/group1  # runs in the default test container
 context/controller

test/integration/targets/ansible-test/collection-tests/constraints.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux -o pipefail
-
-cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
-cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
-
-# common args for all tests
-# each test will be run in a separate venv to verify that requirements have been properly specified
-common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
-
-# unit tests
-
-rm -rf "tests/output"
-ansible-test units "${common[@]}"
-
-# integration tests
-
-rm -rf "tests/output"
-ansible-test integration "${common[@]}"

test/integration/targets/ansible-test/collection-tests/coverage.sh

@@ -5,22 +5,13 @@ set -eux -o pipefail
 cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
 cd "${WORK_DIR}/ansible_collections/ns/col"
 
-# rename the sanity ignore file to match the current ansible version and update import ignores with the python version
+"${TEST_DIR}/collection-tests/update-ignore.py"
-ansible_version="$(python -c 'import ansible.release; print(".".join(ansible.release.__version__.split(".")[:2]))')"
-if [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^2\. ]] || [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^3\.[567] ]]; then
-    # Non-module/module_utils plugins are not checked on these remote-only Python versions
-    sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" | grep -v 'plugins/[^m].* import' > "tests/sanity/ignore-${ansible_version}.txt"
-else
-    sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" > "tests/sanity/ignore-${ansible_version}.txt"
-fi
-cat "tests/sanity/ignore-${ansible_version}.txt"
 
 # common args for all tests
 common=(--venv --color --truncate 0 "${@}")
-test_common=("${common[@]}" --python "${ANSIBLE_TEST_PYTHON_VERSION}")
 
 # run a lightweight test that generates code coverge output
-ansible-test sanity --test import "${test_common[@]}" --coverage
+ansible-test sanity --test import "${common[@]}" --coverage
 
 # report on code coverage in all supported formats
 ansible-test coverage report "${common[@]}"

test/integration/targets/ansible-test/collection-tests/integration-constraints.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
+
+ansible-test integration --venv --color --truncate 0 "${@}"

test/integration/targets/ansible-test/collection-tests/integration.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+ansible-test integration --venv --color --truncate 0 "${@}"

test/integration/targets/ansible-test/collection-tests/sanity.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+"${TEST_DIR}/collection-tests/update-ignore.py"
+
+ansible-test sanity --color --truncate 0 "${@}"

test/integration/targets/ansible-test/collection-tests/units-constraints.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
+
+options=$("${TEST_DIR}"/collection-tests/venv-pythons.py)
+IFS=', ' read -r -a pythons <<< "${options}"
+
+ansible-test units --color --truncate 0 "${pythons[@]}" "${@}"

test/integration/targets/ansible-test/collection-tests/units.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+options=$("${TEST_DIR}"/collection-tests/venv-pythons.py)
+IFS=', ' read -r -a pythons <<< "${options}"
+
+ansible-test units --color --truncate 0 "${pythons[@]}" "${@}"

test/integration/targets/ansible-test/collection-tests/update-ignore.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+"""Rewrite a sanity ignore file to expand Python versions for import ignores and write the file out with the correct Ansible version in the name."""
+
+import os
+import sys
+
+from ansible import release
+
+
+def main():
+    ansible_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(release.__file__))))
+    source_root = os.path.join(ansible_root, 'test', 'lib')
+
+    sys.path.insert(0, source_root)
+
+    from ansible_test._internal import constants
+
+    src_path = 'tests/sanity/ignore.txt'
+    directory = os.path.dirname(src_path)
+    name, ext = os.path.splitext(os.path.basename(src_path))
+    major_minor = '.'.join(release.__version__.split('.')[:2])
+    dst_path = os.path.join(directory, f'{name}-{major_minor}{ext}')
+
+    with open(src_path) as src_file:
+        src_lines = src_file.read().splitlines()
+
+    dst_lines = []
+
+    for line in src_lines:
+        path, rule = line.split(' ')
+
+        if rule != 'import':
+            dst_lines.append(line)
+            continue
+
+        if path.startswith('plugins/module'):
+            python_versions = constants.SUPPORTED_PYTHON_VERSIONS
+        else:
+            python_versions = constants.CONTROLLER_PYTHON_VERSIONS
+
+        for python_version in python_versions:
+            dst_lines.append(f'{line}-{python_version}')
+
+    ignores = '\n'.join(dst_lines) + '\n'
+
+    with open(dst_path, 'w') as dst_file:
+        dst_file.write(ignores)
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/ansible-test/collection-tests/venv-pythons.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+"""Return target Python options for use with ansible-test."""
+
+import os
+import shutil
+import subprocess
+import sys
+
+from ansible import release
+
+
+def main():
+    ansible_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(release.__file__))))
+    source_root = os.path.join(ansible_root, 'test', 'lib')
+
+    sys.path.insert(0, source_root)
+
+    from ansible_test._internal import constants
+
+    args = []
+
+    for python_version in constants.SUPPORTED_PYTHON_VERSIONS:
+        executable = shutil.which(f'python{python_version}')
+
+        if executable:
+            if python_version.startswith('2.'):
+                cmd = [executable, '-m', 'virtualenv', '--version']
+            else:
+                cmd = [executable, '-m', 'venv', '--help']
+
+            process = subprocess.run(cmd, capture_output=True, check=False)
+
+            print(f'{executable} - {"fail" if process.returncode else "pass"}', file=sys.stderr)
+
+            if not process.returncode:
+                args.extend(['--target-python', f'venv/{python_version}'])
+
+    print(' '.join(args))
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/ansible-test/collection-tests/venv.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux -o pipefail
-
-cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
-cd "${WORK_DIR}/ansible_collections/ns/col"
-
-# rename the sanity ignore file to match the current ansible version and update import ignores with the python version
-ansible_version="$(python -c 'import ansible.release; print(".".join(ansible.release.__version__.split(".")[:2]))')"
-if [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^2\. ]] || [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^3\.[567] ]]; then
-    # Non-module/module_utils plugins are not checked on these remote-only Python versions
-    sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" | grep -v 'plugins/[^m].* import' > "tests/sanity/ignore-${ansible_version}.txt"
-else
-    sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" > "tests/sanity/ignore-${ansible_version}.txt"
-fi
-cat "tests/sanity/ignore-${ansible_version}.txt"
-
-# common args for all tests
-# each test will be run in a separate venv to verify that requirements have been properly specified
-common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
-
-# sanity tests
-
-tests=()
-
-set +x
-
-while IFS='' read -r line; do
-  tests+=("$line");
-done < <(
-  ansible-test sanity --list-tests
-)
-
-set -x
-
-for test in "${tests[@]}"; do
-  rm -rf "tests/output"
-  ansible-test sanity "${common[@]}" --test "${test}"
-done
-
-# unit tests
-
-rm -rf "tests/output"
-ansible-test units "${common[@]}"
-
-# integration tests
-
-rm -rf "tests/output"
-ansible-test integration "${common[@]}"

test/lib/ansible_test/_data/requirements/sanity.import.txt

@@ -0,0 +1 @@
+pyyaml == 5.4.1  # needed for yaml_to_json.py

test/lib/ansible_test/_internal/commands/integration/__init__.py

@@ -440,9 +440,10 @@ def command_integration_filtered(
 
         if isinstance(target_profile, ControllerProfile):
             if host_state.controller_profile.python.path != target_profile.python.path:
-                install_requirements(args, target_python, command=True)  # integration
+                install_requirements(args, target_python, command=True, controller=False)  # integration
         elif isinstance(target_profile, SshTargetHostProfile):
-            install_requirements(args, target_python, command=True, connection=target_profile.get_controller_target_connections()[0])  # integration
+            connection = target_profile.get_controller_target_connections()[0]
+            install_requirements(args, target_python, command=True, controller=False, connection=connection)  # integration
 
     coverage_manager = CoverageManager(args, host_state, inventory_path)
     coverage_manager.setup()

test/lib/ansible_test/_internal/commands/sanity/import.py

@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 import os
-import tempfile
 import typing as t
 
 from . import (
@@ -34,6 +33,7 @@ from ...util import (
     display,
     parse_to_list_of_dict,
     is_subdir,
+    ANSIBLE_TEST_TOOLS_ROOT,
 )
 
 from ...util_common import (
@@ -45,6 +45,7 @@ from ...ansible_util import (
 )
 
 from ...python_requirements import (
+    PipUnavailableError,
     install_requirements,
 )
 
@@ -94,7 +95,10 @@ class ImportTest(SanityMultipleVersion):
         if python.version.startswith('2.'):
             # hack to make sure that virtualenv is available under Python 2.x
             # on Python 3.x we can use the built-in venv
-            install_requirements(args, python, virtualenv=True)  # sanity (import)
+            try:
+                install_requirements(args, python, virtualenv=True, controller=False)  # sanity (import)
+            except PipUnavailableError as ex:
+                display.warning(ex)
 
         temp_root = os.path.join(ResultType.TMP.path, 'sanity', 'import')
 
@@ -134,25 +138,25 @@ class ImportTest(SanityMultipleVersion):
             )
 
             if data_context().content.collection:
+                external_python = create_sanity_virtualenv(args, args.controller_python, self.name, context=self.name)
+
                 env.update(
                     SANITY_COLLECTION_FULL_NAME=data_context().content.collection.full_name,
-                    SANITY_EXTERNAL_PYTHON=python.path,
+                    SANITY_EXTERNAL_PYTHON=external_python.path,
+                    SANITY_YAML_TO_JSON=os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'yaml_to_json.py'),
                 )
 
             display.info(import_type + ': ' + data, verbosity=4)
 
             cmd = ['importer.py']
 
+            # add the importer to the path so it can be accessed through the coverage injector
+            env.update(
+                PATH=os.pathsep.join([os.path.join(TARGET_SANITY_ROOT, 'import'), env['PATH']]),
+            )
+
             try:
-                with tempfile.TemporaryDirectory(prefix='ansible-test', suffix='-import') as temp_dir:
+                stdout, stderr = cover_python(args, virtualenv_python, cmd, self.name, env, capture=True, data=data)
-                    # make the importer available in the temporary directory
-                    os.symlink(os.path.abspath(os.path.join(TARGET_SANITY_ROOT, 'import', 'importer.py')), os.path.join(temp_dir, 'importer.py'))
-                    os.symlink(os.path.abspath(os.path.join(TARGET_SANITY_ROOT, 'import', 'yaml_to_json.py')), os.path.join(temp_dir, 'yaml_to_json.py'))
-
-                    # add the importer to the path so it can be accessed through the coverage injector
-                    env['PATH'] = os.pathsep.join([temp_dir, env['PATH']])
-
-                    stdout, stderr = cover_python(args, virtualenv_python, cmd, self.name, env, capture=True, data=data)
 
                 if stdout or stderr:
                     raise SubprocessError(cmd, stdout=stdout, stderr=stderr)

test/lib/ansible_test/_internal/commands/units/__init__.py

@@ -227,7 +227,7 @@ def command_units(args):  # type: (UnitsConfig) -> None
         controller = any(test_context == TestContext.controller for test_context, python, paths, env in final_candidates)
 
         if args.requirements_mode != 'skip':
-            install_requirements(args, target_profile.python, ansible=controller, command=True)  # units
+            install_requirements(args, target_profile.python, ansible=controller, command=True, controller=False)  # units
 
         test_sets.extend(final_candidates)
 

test/lib/ansible_test/_internal/containers.py

@@ -829,7 +829,7 @@ def cleanup_ssh_ports(
         for process in ssh_processes:
             process.terminate()
 
-        display.info('Waiting for the %s host SSH port forwarding processs(es) to terminate.' % host_type, verbosity=1)
+        display.info('Waiting for the %s host SSH port forwarding process(es) to terminate.' % host_type, verbosity=1)
 
         for process in ssh_processes:
             process.wait()

test/lib/ansible_test/_internal/python_requirements.py

@@ -25,6 +25,7 @@ from .util import (
     ANSIBLE_TEST_DATA_ROOT,
     ANSIBLE_TEST_TARGET_ROOT,
     ANSIBLE_TEST_TOOLS_ROOT,
+    ApplicationError,
     SubprocessError,
     display,
     find_executable,
@@ -65,6 +66,12 @@ REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requ
 # Pip Abstraction
 
 
+class PipUnavailableError(ApplicationError):
+    """Exception raised when pip is not available."""
+    def __init__(self, python):  # type: (PythonConfig) -> None
+        super().__init__(f'Python {python.version} at "{python.path}" does not have pip available.')
+
+
 @dataclasses.dataclass(frozen=True)
 class PipCommand:
     """Base class for pip commands."""""
@@ -97,6 +104,11 @@ class PipUninstall(PipCommand):
     ignore_errors: bool
 
 
+@dataclasses.dataclass(frozen=True)
+class PipVersion(PipCommand):
+    """Details required to get the pip version."""
+
+
 # Entry Points
 
 
@@ -107,13 +119,12 @@ def install_requirements(
         command=False,  # type: bool
         coverage=False,  # type: bool
         virtualenv=False,  # type: bool
+        controller=True,  # type: bool
         connection=None,  # type: t.Optional[Connection]
 ):  # type: (...) -> None
     """Install requirements for the given Python using the specified arguments."""
     create_result_directories(args)
 
-    controller = not connection
-
     if not requirements_allowed(args, controller):
         return
 
@@ -220,7 +231,18 @@ def run_pip(
     script = prepare_pip_script(commands)
 
     if not args.explain:
-        connection.run([python.path], data=script)
+        try:
+            connection.run([python.path], data=script)
+        except SubprocessError:
+            script = prepare_pip_script([PipVersion()])
+
+            try:
+                connection.run([python.path], data=script, capture=True)
+            except SubprocessError as ex:
+                if 'pip is unavailable:' in ex.stdout + ex.stderr:
+                    raise PipUnavailableError(python)
+
+            raise
 
 
 # Collect

test/lib/ansible_test/_util/target/sanity/import/importer.py

@@ -23,7 +23,8 @@ def main():
 
     ansible_path = os.path.dirname(os.path.dirname(ansible.__file__))
     temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep
-    external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') or sys.executable
+    external_python = os.environ.get('SANITY_EXTERNAL_PYTHON')
+    yaml_to_json_path = os.environ.get('SANITY_YAML_TO_JSON')
     collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME')
     collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATH')
     import_type = os.environ.get('SANITY_IMPORTER_TYPE')
@@ -48,7 +49,6 @@ def main():
         from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
         from ansible.utils.collection_loader import _collection_finder
 
-        yaml_to_json_path = os.path.join(os.path.dirname(__file__), 'yaml_to_json.py')
         yaml_to_dict_cache = {}
 
         # unique ISO date marker matching the one present in yaml_to_json.py

test/lib/ansible_test/_util/target/setup/quiet_pip.py

@@ -5,6 +5,7 @@ __metaclass__ = type
 import logging
 import re
 import runpy
+import sys
 import warnings
 
 BUILTIN_FILTERER_FILTER = logging.Filterer.filter
@@ -51,7 +52,11 @@ def main():
         #   Python 2.7 cannot use the -W option to match warning text after a colon. This makes it impossible to match specific warning messages.
         warnings.filterwarnings('ignore', message_filter)
 
-    runpy.run_module('pip.__main__', run_name='__main__', alter_sys=True)
+    try:
+        runpy.run_module('pip.__main__', run_name='__main__', alter_sys=True)
+    except ImportError as ex:
+        print('pip is unavailable: %s' % ex)
+        sys.exit(1)
 
 
 if __name__ == '__main__':

test/lib/ansible_test/_util/target/setup/requirements.py

@@ -114,6 +114,18 @@ def uninstall(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
             raise
 
 
+# noinspection PyUnusedLocal
+def version(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
+    """Report the pip version."""
+    del options
+
+    options = common_pip_options()
+
+    command = [sys.executable, pip, '-V'] + options
+
+    execute_command(command, capture=True)
+
+
 def common_pip_options():  # type: () -> t.List[str]
     """Return a list of common pip options."""
     return [