Fix collection loader iterator for ext modules (#84758)

* Fix collection loader iterator for ext modules

Fixes the collection loader iter_modules logic to return compiled Python
extension modules. This loader is used when attempting to call
`pkgutil.iter_modules` on a package that is contained inside the
`ansible_collections` path. Before extension modules were skipped which
would break imports of 3rd party modules that used compiled extensions
if it was installed in a Python environment located inside a collection.

* Fix up pkg logic

* Added integration test

* Add newline to alias file

* Update lib/ansible/utils/collection_loader/_collection_finder.py

Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

---------

Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>

changelogs/fragments/collection-loader-extensions.yml

@@ -0,0 +1,5 @@
+bugfixes:
+  - >-
+    collection loader - Fix the collection loader logic to correctly return Python module when calling
+    ``pkgutil.iter_modules`` with a package that is inside a collection path and contains compiled Python extension
+    modules.

lib/ansible/utils/collection_loader/_collection_finder.py

@@ -6,6 +6,7 @@
 
 from __future__ import annotations
 
+import inspect
 import itertools
 import os
 import os.path
@@ -1237,23 +1238,33 @@ def _iter_modules_impl(paths, prefix=''):
         prefix = _to_text(prefix)
     # yield (module_loader, name, ispkg) for each module/pkg under path
     # TODO: implement ignore/silent catch for unreadable?
-    for b_path in map(_to_bytes, paths):
-        if not os.path.isdir(b_path):
+    # Mostly based off the logic in the builtin pkgutil.iter_modules importer.
+    # Only difference is we don't check if dirs contains __init__.
+    # https://github.com/python/cpython/blob/fda056e64bdfcac3dd3d13eebda0a24994d83cb8/Lib/pkgutil.py#L130-L168
+    for path in paths:
+        if not os.path.isdir(path):
             continue
-        for b_basename in sorted(os.listdir(b_path)):
-            b_candidate_module_path = os.path.join(b_path, b_basename)
-            if os.path.isdir(b_candidate_module_path):
+
+        yielded = {'__init__'}
+        for basename in sorted(os.listdir(path)):
+            modname = inspect.getmodulename(basename)
+            if modname in yielded:
+                continue
+
+            mod_path = os.path.join(path, basename)
+            ispkg = False
+            if not modname and os.path.isdir(mod_path) and '.' not in basename:
                 # exclude things that obviously aren't Python package dirs
                 # FIXME: this dir is adjustable in py3.8+, check for it
-                if b'.' in b_basename or b_basename == b'__pycache__':
+                if basename == '__pycache__':
                     continue
 
-                # TODO: proper string handling?
-                yield prefix + _to_text(b_basename), True
-            else:
-                # FIXME: match builtin ordering for package/dir/file, support compiled?
-                if b_basename.endswith(b'.py') and b_basename != b'__init__.py':
-                    yield prefix + _to_text(os.path.splitext(b_basename)[0]), False
+                modname = basename
+                ispkg = True
+
+            if modname and '.' not in modname:
+                yielded.add(modname)
+                yield prefix + modname, ispkg
 
 
 def _get_collection_metadata(collection_name):

test/integration/targets/collection_loader/aliases

@@ -0,0 +1,3 @@
+shippable/posix/group3
+context/controller
+needs/target/collection

test/integration/targets/collection_loader/ansible_collections/ns/col/plugins/action/test_collection_loader.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from ansible.plugins.action import ActionBase
+
+
+class ActionModule(ActionBase):
+    def run(self, tmp=None, task_vars=None):
+        import my_module
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        result['modules'] = my_module.run()
+
+        return result

test/integration/targets/collection_loader/ansible_collections/ns/col/temp_pythonpath/my_module/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import pkgutil
+
+from my_module import sub
+
+
+def run() -> list[str]:
+    return [m[1] for m in pkgutil.iter_modules(sub.__path__)]

test/integration/targets/collection_loader/main.yml

@@ -0,0 +1,11 @@
+- name: run collection loader tests
+  hosts: localhost
+  tasks:
+  - name: run module which imports Python library inside collection dir
+    ns.col.test_collection_loader:
+    register: result
+
+  - name: assert module loader results
+    assert:
+      that:
+      - result.modules == ['foo']

test/integration/targets/collection_loader/runme.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eux
+
+PLAYBOOK_DIR="${PWD}"
+
+source ../collection/setup.sh
+
+EXT=$( python -c 'import importlib.machinery as i; print(next(iter([s for s in i.all_suffixes() if s != ".py"])))' )
+touch "temp_pythonpath/my_module/sub/foo${EXT}"
+
+ANSIBLE_COLLECTIONS_PATH="${PWD}../../../" \
+    ANSIBLE_LOCALHOST_WARNING=false \
+    PYTHONPATH="./temp_pythonpath:${PYTHONPATH}" \
+    ansible-playbook "${PLAYBOOK_DIR}/main.yml" "$@"