[stable-2.19] Add AnsiballZ debugging support with debugpy (#85476) (#85528)

* Add AnsiballZ debugging support with debugpy

Adds support for debugging AnsiballZ modules with debugpy which is used
by VSCode as its Python debugger DAP. Debugging can either be done
through a manual Debugpy listening server through a launch.json
configuration or through the new ansible-test --dev-debug-on-deman
argument.

* Fix up integration test

* Simplify config option and move mypy ignore

* Use new API if available and fix typo

* Guard the import of debugpy

* Fix sanity import issue

* Minor cosmetic adjustments

* Simplify debugger setup

* ansible-test - Refactor debugging interface

* Add ansible-test debug integration tests

* Fix ansible-test shell when in unsupported dir

---------
(cherry picked from commit 3882366585)

Co-authored-by: Jordan Borean <jborean93@gmail.com>

test/integration/targets/ansiballz_debugging/tasks/main.yml

@@ -1,7 +1,7 @@
 - name: Run a module with remote debugging configured to use a bogus debugger module
   ping:
   vars:
-    _ansible_ansiballz_debugger_config:
+    _ansible_ansiballz_pydevd_config:
       module: not_a_valid_debugger_module
   register: result
   ignore_errors: yes

test/integration/targets/ansible-test-debugging-env/aliases

@@ -0,0 +1,2 @@
+shippable/generic/group1  # runs in the default test container
+context/controller

test/integration/targets/ansible-test-debugging-env/runme.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Used to support the ansible-test-debugging integration test.
+
+env

test/integration/targets/ansible-test-debugging-inventory/aliases

@@ -0,0 +1,2 @@
+shippable/generic/group1  # runs in the default test container
+context/controller

test/integration/targets/ansible-test-debugging-inventory/runme.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Used to support the ansible-test-debugging integration test.
+
+cat "${INVENTORY_PATH}"

test/integration/targets/ansible-test-debugging/aliases

@@ -0,0 +1,5 @@
+shippable/generic/group1  # runs in the default test container
+needs/target/ansible-test-debugging-env  # indirectly used by ansible-test, included here for change detection
+needs/target/ansible-test-debugging-inventory  # indirectly used by ansible-test, included here for change detection
+context/controller
+gather_facts/no

test/integration/targets/ansible-test-debugging/tasks/main.yml

@@ -0,0 +1,98 @@
+- name: Define supported debuggers
+  set_fact:
+    debuggers:
+      - pydevd
+      - debugpy
+
+- name: Run without debugging features enabled
+  command: ansible-test shell -v -- env
+  register: result
+
+- assert:
+    that:
+      - result.stderr_lines is not contains 'Debugging'
+
+- name: Run a command which does not support debugging
+  command: ansible-test env -v
+  register: result
+
+- assert:
+    that:
+      - result.stderr_lines is not contains 'Debugging'
+
+- name: Verify on-demand debugging gracefully handles not running under a debugger
+  command: ansible-test shell -v --dev-debug-on-demand -- env
+  register: result
+
+- assert:
+    that:
+      - result.stderr_lines is contains 'Debugging disabled because no debugger was detected.'
+
+- name: Verify manual debugging gracefully handles lack of configuration
+  command: ansible-test shell -v --dev-debug-cli -- env
+  register: result
+
+- assert:
+    that:
+      - result.stderr_lines is contains 'Debugging disabled because no debugger configuration was provided.'
+
+- name: Verify invalid debugger configuration is handled
+  command: ansible-test shell --dev-debug-cli -- env
+  environment: >
+    {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": "{\"invalid_key\": true}"}') | from_json }}
+  register: result
+  loop: "{{ debuggers }}"
+  ignore_errors: yes
+
+- assert:
+    that:
+      - item.stderr is search("Invalid " + item.item + " settings.*invalid_key")
+  loop: "{{ result.results }}"
+
+- name: Verify CLI debugger can be manually enabled (shell)
+  command: ansible-test shell --dev-debug-cli -- env
+  environment: >
+    {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }}
+  register: result
+  loop: "{{ debuggers }}"
+
+- assert:
+    that:
+      - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG"
+  loop: "{{ result.results }}"
+
+- name: Verify CLI debugger can be manually enabled (integration)
+  command: ansible-test integration ansible-test-debugging-env --dev-debug-cli
+  environment: >
+    {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }}
+  register: result
+  loop: "{{ debuggers }}"
+
+- assert:
+    that:
+      - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG"
+  loop: "{{ result.results }}"
+
+- name: Verify AnsiballZ debugger can be manually enabled (shell)
+  command: ansible-test shell --dev-debug-ansiballz -- env
+  environment: >
+    {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }}
+  register: result
+  loop: "{{ debuggers }}"
+
+- assert:
+    that:
+      - item.stdout is contains("_ANSIBLE_ANSIBALLZ_" + item.item.upper() + "_CONFIG")
+  loop: "{{ result.results }}"
+
+- name: Verify AnsiballZ debugger can be manually enabled (integration)
+  command: ansible-test integration ansible-test-debugging-inventory --dev-debug-ansiballz
+  environment: >
+    {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }}
+  register: result
+  loop: "{{ debuggers }}"
+
+- assert:
+    that:
+      - item.stdout is contains("_ansible_ansiballz_" + item.item + "_config")
+  loop: "{{ result.results }}"

test/lib/ansible_test/_internal/commands/shell/__init__.py

@@ -2,11 +2,16 @@
 
 from __future__ import annotations
 
+import contextlib
 import dataclasses
 import os
 import sys
 import typing as t
 
+from ...data import (
+    data_context,
+)
+
 from ...util import (
     ApplicationError,
     OutputStream,
@@ -101,19 +106,33 @@ def command_shell(args: ShellConfig) -> None:
     if args.export:
         return
 
-    if args.cmd:
-        # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive.
-        # If we want to support interactive commands in the future, we'll need an `--interactive` command line option.
-        # Command stderr output is allowed to mix with our own output, which is all sent to stderr.
-        con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL)
-        return
-
     if isinstance(con, LocalConnection) and isinstance(target_profile, DebuggableProfile) and target_profile.debugging_enabled:
-        # HACK: ensure the pydevd port visible in the shell is the forwarded port, not the original
-        args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.pydevd_port)
+        # HACK: ensure the debugger port visible in the shell is the forwarded port, not the original
+        args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.debugger_port)
 
-    with metadata_context(args):
-        interactive_shell(args, target_profile, con)
+    with contextlib.nullcontext() if data_context().content.unsupported else metadata_context(args):
+        if args.cmd:
+            non_interactive_shell(args, target_profile, con)
+        else:
+            interactive_shell(args, target_profile, con)
+
+
+def non_interactive_shell(
+    args: ShellConfig,
+    target_profile: SshTargetHostProfile,
+    con: Connection,
+) -> None:
+    """Run a non-interactive shell command."""
+    if isinstance(target_profile, PosixProfile):
+        env = get_environment_variables(args, target_profile, con)
+        cmd = get_env_command(env) + args.cmd
+    else:
+        cmd = args.cmd
+
+    # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive.
+    # If we want to support interactive commands in the future, we'll need an `--interactive` command line option.
+    # Command stderr output is allowed to mix with our own output, which is all sent to stderr.
+    con.run(cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL)
 
 
 def interactive_shell(
@@ -135,23 +154,8 @@ def interactive_shell(
             python = target_profile.python  # make sure the python interpreter has been initialized before opening a shell
             display.info(f'Target Python {python.version} is at: {python.path}')
 
-            optional_vars = (
-                'TERM',  # keep backspace working
-            )
-
-            env = {name: os.environ[name] for name in optional_vars if name in os.environ}
-
-            if isinstance(con, LocalConnection):  # configure the controller environment
-                env.update(ansible_environment(args))
-                env.update(get_injector_env(target_profile.python, env))
-                env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path))
-
-                if isinstance(target_profile, DebuggableProfile):
-                    env.update(target_profile.get_ansiballz_environment_variables())
-                    env.update(target_profile.get_ansible_cli_environment_variables())
-
-            if env:
-                cmd = ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()]
+            env = get_environment_variables(args, target_profile, con)
+            cmd = get_env_command(env)
 
         cmd += [shell, '-i']
     else:
@@ -175,3 +179,38 @@ def interactive_shell(
             raise HostConnectionError(f'SSH shell connection failed for host {target_profile.config}: {ex}', callback) from ex
 
         raise
+
+
+def get_env_command(env: dict[str, str]) -> list[str]:
+    """Get an `env` command to set the given environment variables, if any."""
+    if not env:
+        return []
+
+    return ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()]
+
+
+def get_environment_variables(
+    args: ShellConfig,
+    target_profile: PosixProfile,
+    con: Connection,
+) -> dict[str, str]:
+    """Get the environment variables to expose to the shell."""
+    if data_context().content.unsupported:
+        return {}
+
+    optional_vars = (
+        'TERM',  # keep backspace working
+    )
+
+    env = {name: os.environ[name] for name in optional_vars if name in os.environ}
+
+    if isinstance(con, LocalConnection):  # configure the controller environment
+        env.update(ansible_environment(args))
+        env.update(get_injector_env(target_profile.python, env))
+        env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path))
+
+        if isinstance(target_profile, DebuggableProfile):
+            env.update(target_profile.get_ansiballz_environment_variables())
+            env.update(target_profile.get_ansible_cli_environment_variables())
+
+    return env

test/lib/ansible_test/_internal/debugging.py

@@ -2,16 +2,25 @@
 
 from __future__ import annotations
 
+import abc
 import dataclasses
+import importlib
 import json
 import os
 import re
+import sys
+import typing as t
 
 from .util import (
     cache,
     display,
     raw_command,
     ApplicationError,
+    get_subclasses,
+)
+
+from .util_common import (
+    CommonConfig,
 )
 
 from .processes import (
@@ -24,16 +33,282 @@ from .config import (
 )
 
 from .metadata import (
-    DebuggerSettings,
     DebuggerFlags,
 )
 
-from . import (
+from .data import (
     data_context,
-    CommonConfig,
 )
 
 
+class DebuggerProfile(t.Protocol):
+    """Protocol for debugger profiles."""
+
+    @property
+    def debugger_host(self) -> str:
+        """The hostname to expose to the debugger."""
+
+    @property
+    def debugger_port(self) -> int:
+        """The port to expose to the debugger."""
+
+    def get_source_mapping(self) -> dict[str, str]:
+        """The source mapping to expose to the debugger."""
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class DebuggerSettings(metaclass=abc.ABCMeta):
+    """Common debugger settings."""
+
+    port: int = 5678
+    """
+    The port on the origin host which is listening for incoming connections from the debugger.
+    SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed.
+    """
+
+    def as_dict(self) -> dict[str, object]:
+        """Convert this instance to a dict."""
+        data = dataclasses.asdict(self)
+        data.update(__type__=self.__class__.__name__)
+
+        return data
+
+    @classmethod
+    def from_dict(cls, value: dict[str, t.Any]) -> t.Self:
+        """Load an instance from a dict."""
+        debug_cls = globals()[value.pop('__type__')]
+
+        return debug_cls(**value)
+
+    @classmethod
+    def get_debug_type(cls) -> str:
+        """Return the name for this debugger."""
+        return cls.__name__.removesuffix('Settings').lower()
+
+    @classmethod
+    def get_config_env_var_name(cls) -> str:
+        """Return the name of the environment variable used to customize settings for this debugger."""
+        return f'ANSIBLE_TEST_REMOTE_DEBUGGER_{cls.get_debug_type().upper()}'
+
+    @classmethod
+    def parse(cls, value: str) -> t.Self:
+        """Parse debugger settings from the given JSON and apply defaults."""
+        try:
+            settings = cls(**json.loads(value))
+        except Exception as ex:
+            raise ApplicationError(f"Invalid {cls.get_debug_type()} settings: {ex}") from ex
+
+        return cls.apply_defaults(settings)
+
+    @classmethod
+    @abc.abstractmethod
+    def is_active(cls) -> bool:
+        """Detect if the debugger is active."""
+
+    @classmethod
+    @abc.abstractmethod
+    def apply_defaults(cls, settings: t.Self) -> t.Self:
+        """Apply defaults to the given settings."""
+
+    @abc.abstractmethod
+    def get_python_package(self) -> str:
+        """The Python package to install for debugging."""
+
+    @abc.abstractmethod
+    def activate_debugger(self, profile: DebuggerProfile) -> None:
+        """Activate the debugger in ansible-test after delegation."""
+
+    @abc.abstractmethod
+    def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]:
+        """Gets the extra configuration data for the AnsiballZ extension module."""
+
+    @abc.abstractmethod
+    def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]:
+        """Get command line arguments for the debugger when running Ansible CLI programs."""
+
+    @abc.abstractmethod
+    def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]:
+        """Get environment variables needed to configure the debugger for debugging."""
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class PydevdSettings(DebuggerSettings):
+    """Settings for the pydevd debugger."""
+
+    package: str | None = None
+    """
+    The Python package to install for debugging.
+    If `None` then the package will be auto-detected.
+    If an empty string, then no package will be installed.
+    """
+
+    module: str | None = None
+    """
+    The Python module to import for debugging.
+    This should be pydevd or a derivative.
+    If not provided it will be auto-detected.
+    """
+
+    settrace: dict[str, object] = dataclasses.field(default_factory=dict)
+    """
+    Options to pass to the `{module}.settrace` method.
+    Used for running AnsiballZ modules only.
+    The `host` and `port` options will be provided by ansible-test.
+    The `suspend` option defaults to `False`.
+    """
+
+    args: list[str] = dataclasses.field(default_factory=list)
+    """
+    Arguments to pass to `pydevd` on the command line.
+    Used for running Ansible CLI programs only.
+    The `--client` and `--port` options will be provided by ansible-test.
+    """
+
+    @classmethod
+    def is_active(cls) -> bool:
+        return detect_pydevd_port() is not None
+
+    @classmethod
+    def apply_defaults(cls, settings: t.Self) -> t.Self:
+        if not settings.module:
+            if not settings.package or 'pydevd-pycharm' in settings.package:
+                module = 'pydevd_pycharm'
+            else:
+                module = 'pydevd'
+
+            settings = dataclasses.replace(settings, module=module)
+
+        if settings.package is None:
+            if settings.module == 'pydevd_pycharm':
+                if pycharm_version := detect_pycharm_version():
+                    package = f'pydevd-pycharm~={pycharm_version}'
+                else:
+                    package = None
+            else:
+                package = 'pydevd'
+
+            settings = dataclasses.replace(settings, package=package)
+
+        settings.settrace.setdefault('suspend', False)
+
+        if port := detect_pydevd_port():
+            settings = dataclasses.replace(settings, port=port)
+
+            if detect_pycharm_process():
+                # This only works with the default PyCharm debugger.
+                # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers.
+                # Further investigation is required to understand the cause.
+                settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess'])
+
+        return settings
+
+    def get_python_package(self) -> str:
+        if self.package is None and self.module == 'pydevd_pycharm':
+            display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version was not detected.')
+
+        return self.package
+
+    def activate_debugger(self, profile: DebuggerProfile) -> None:
+        debugging_module = importlib.import_module(self.module)
+        debugging_module.settrace(**self._get_settrace_arguments(profile))
+
+    def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]:
+        return dict(
+            module=self.module,
+            settrace=self._get_settrace_arguments(profile),
+            source_mapping=profile.get_source_mapping(),
+        )
+
+    def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]:
+        # Although `pydevd_pycharm` can be used to invoke `settrace`, it cannot be used to run the debugger on the command line.
+        return ['-m', 'pydevd', '--client', profile.debugger_host, '--port', str(profile.debugger_port)] + self.args + ['--file']
+
+    def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]:
+        return dict(
+            PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())),
+            PYDEVD_DISABLE_FILE_VALIDATION="1",
+        )
+
+    def _get_settrace_arguments(self, profile: DebuggerProfile) -> dict[str, object]:
+        """Get settrace arguments for pydevd."""
+        return self.settrace | dict(
+            host=profile.debugger_host,
+            port=profile.debugger_port,
+        )
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class DebugpySettings(DebuggerSettings):
+    """Settings for the debugpy debugger."""
+
+    connect: dict[str, object] = dataclasses.field(default_factory=dict)
+    """
+    Options to pass to the `debugpy.connect` method.
+    Used for running AnsiballZ modules and ansible-test after delegation.
+    The endpoint addr, `access_token`, and `parent_session_pid` options will be provided by ansible-test.
+    """
+
+    args: list[str] = dataclasses.field(default_factory=list)
+    """
+    Arguments to pass to `debugpy` on the command line.
+    Used for running Ansible CLI programs only.
+    The `--connect`, `--adapter-access-token`, and `--parent-session-pid` options will be provided by ansible-test.
+    """
+
+    @classmethod
+    def is_active(cls) -> bool:
+        return detect_debugpy_options() is not None
+
+    @classmethod
+    def apply_defaults(cls, settings: t.Self) -> t.Self:
+        if options := detect_debugpy_options():
+            settings = dataclasses.replace(settings, port=options.port)
+            settings.connect.update(
+                access_token=options.adapter_access_token,
+                parent_session_pid=os.getpid(),
+            )
+        else:
+            display.warning('Debugging will be limited to the first connection. Run ansible-test under debugpy to support multiple connections.')
+
+        return settings
+
+    def get_python_package(self) -> str:
+        return 'debugpy'
+
+    def activate_debugger(self, profile: DebuggerProfile) -> None:
+        import debugpy  # pylint: disable=import-error
+
+        debugpy.connect((profile.debugger_host, profile.debugger_port), **self.connect)
+
+    def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]:
+        return dict(
+            host=profile.debugger_host,
+            port=profile.debugger_port,
+            connect=self.connect,
+            source_mapping=profile.get_source_mapping(),
+        )
+
+    def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]:
+        cli_args = ['-m', 'debugpy', '--connect', f"{profile.debugger_host}:{profile.debugger_port}"]
+
+        if access_token := self.connect.get('access_token'):
+            cli_args += ['--adapter-access-token', str(access_token)]
+
+        if session_pid := self.connect.get('parent_session_pid'):
+            cli_args += ['--parent-session-pid', str(session_pid)]
+
+        if self.args:
+            cli_args += self.args
+
+        return cli_args
+
+    def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]:
+        return dict(
+            PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())),
+            PYDEVD_DISABLE_FILE_VALIDATION="1",
+        )
+
+
 def initialize_debugger(args: CommonConfig) -> None:
     """Initialize the debugger settings before delegation."""
     if not isinstance(args, EnvironmentConfig):
@@ -48,52 +323,18 @@ def initialize_debugger(args: CommonConfig) -> None:
     load_debugger_settings(args)
 
 
-def parse_debugger_settings(value: str) -> DebuggerSettings:
-    """Parse remote debugger settings and apply defaults."""
-    try:
-        settings = DebuggerSettings(**json.loads(value))
-    except Exception as ex:
-        raise ApplicationError(f"Invalid debugger settings: {ex}") from ex
-
-    if not settings.module:
-        if not settings.package or 'pydevd-pycharm' in settings.package:
-            module = 'pydevd_pycharm'
-        else:
-            module = 'pydevd'
-
-        settings = dataclasses.replace(settings, module=module)
-
-    if settings.package is None:
-        if settings.module == 'pydevd_pycharm':
-            if pycharm_version := detect_pycharm_version():
-                package = f'pydevd-pycharm~={pycharm_version}'
-            else:
-                package = None
-        else:
-            package = 'pydevd'
-
-        settings = dataclasses.replace(settings, package=package)
-
-    settings.settrace.setdefault('suspend', False)
-
-    if port := detect_pydevd_port():
-        settings = dataclasses.replace(settings, port=port)
-
-        if detect_pycharm_process():
-            # This only works with the default PyCharm debugger.
-            # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers.
-            # Further investigation is required to understand the cause.
-            settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess'])
-
-    return settings
-
-
 def load_debugger_settings(args: EnvironmentConfig) -> None:
     """Load the remote debugger settings."""
+    use_debugger: type[DebuggerSettings] | None = None
+
     if args.metadata.debugger_flags.on_demand:
         # On-demand debugging only enables debugging if we're running under a debugger, otherwise it's a no-op.
 
-        if not detect_pydevd_port():
+        for candidate_debugger in get_subclasses(DebuggerSettings):
+            if candidate_debugger.is_active():
+                use_debugger = candidate_debugger
+                break
+        else:
             display.info('Debugging disabled because no debugger was detected.', verbosity=1)
             args.metadata.debugger_flags = DebuggerFlags.all(False)
             return
@@ -107,13 +348,22 @@ def load_debugger_settings(args: EnvironmentConfig) -> None:
     if not args.metadata.debugger_flags.enable:
         return
 
-    value = os.environ.get('ANSIBLE_TEST_REMOTE_DEBUGGER') or '{}'
-    settings = parse_debugger_settings(value)
-
-    display.info(f'>>> Debugger Settings\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3)
+    if not use_debugger:  # detect debug type based on env var
+        for candidate_debugger in get_subclasses(DebuggerSettings):
+            if candidate_debugger.get_config_env_var_name() in os.environ:
+                use_debugger = candidate_debugger
+                break
+        else:
+            display.info('Debugging disabled because no debugger configuration was provided.', verbosity=1)
+            args.metadata.debugger_flags = DebuggerFlags.all(False)
+            return
 
+    config = os.environ.get(use_debugger.get_config_env_var_name()) or '{}'
+    settings = use_debugger.parse(config)
     args.metadata.debugger_settings = settings
 
+    display.info(f'>>> Debugger Settings ({use_debugger.get_debug_type()})\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3)
+
 
 @cache
 def detect_pydevd_port() -> int | None:
@@ -140,8 +390,6 @@ def detect_pycharm_version() -> str | None:
             display.info(f'Detected PyCharm version {version}.', verbosity=1)
             return version
 
-    display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version could not be detected.')
-
     return None
 
 
@@ -164,3 +412,43 @@ def detect_pycharm_process() -> Process | None:
 def get_current_process_cached() -> Process:
     """Return the current process. The result is cached."""
     return get_current_process()
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class DebugpyOptions:
+    """Options detected from the debugpy instance hosting this process."""
+
+    port: int
+    adapter_access_token: str | None
+
+
+@cache
+def detect_debugpy_options() -> DebugpyOptions | None:
+    """Return the options for the debugpy instance hosting this process, or `None` if not detected."""
+    if "debugpy" not in sys.modules:
+        return None
+
+    import debugpy  # pylint: disable=import-error
+
+    # get_cli_options is the new public API introduced after debugpy 1.8.15.
+    # We should remove the debugpy.server cli fallback once the new version is
+    # released.
+    if hasattr(debugpy, 'get_cli_options'):
+        opts = debugpy.get_cli_options()
+    else:
+        from debugpy.server import cli  # pylint: disable=import-error
+        opts = cli.options
+
+    # address can be None if the debugger is not configured through the CLI as
+    # we expected.
+    if not opts.address:
+        return None
+
+    port = opts.address[1]
+
+    display.info(f'Detected debugpy debugger port {port}.', verbosity=1)
+
+    return DebugpyOptions(
+        port=port,
+        adapter_access_token=opts.adapter_access_token,
+    )

test/lib/ansible_test/_internal/host_profiles.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import abc
 import dataclasses
-import importlib
 import json
 import os
 import pathlib
@@ -140,6 +139,11 @@ from .dev.container_probe import (
     check_container_cgroup_status,
 )
 
+from .debugging import (
+    DebuggerProfile,
+    DebuggerSettings,
+)
+
 TControllerHostConfig = t.TypeVar('TControllerHostConfig', bound=ControllerHostConfig)
 THostConfig = t.TypeVar('THostConfig', bound=HostConfig)
 TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig)
@@ -292,12 +296,17 @@ class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta):
         return f'{self.__class__.__name__}: {self.name}'
 
 
-class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
+class DebuggableProfile(HostProfile[THostConfig], DebuggerProfile, metaclass=abc.ABCMeta):
     """Base class for profiles remote debugging."""
 
-    __PYDEVD_PORT_KEY = 'pydevd_port'
+    __DEBUGGING_PORT_KEY = 'debugging_port'
     __DEBUGGING_FORWARDER_KEY = 'debugging_forwarder'
 
+    @property
+    def debugger(self) -> DebuggerSettings | None:
+        """The debugger settings for this host if present and enabled, otherwise None."""
+        return self.args.metadata.debugger_settings
+
     @property
     def debugging_enabled(self) -> bool:
         """Returns `True` if debugging is enabled for this profile, otherwise `False`."""
@@ -307,9 +316,14 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
         return self.args.metadata.debugger_flags.ansiballz
 
     @property
-    def pydevd_port(self) -> int:
-        """The pydevd port to use."""
-        return self.state.get(self.__PYDEVD_PORT_KEY) or self.origin_pydev_port
+    def debugger_host(self) -> str:
+        """The debugger host to use."""
+        return 'localhost'
+
+    @property
+    def debugger_port(self) -> int:
+        """The debugger port to use."""
+        return self.state.get(self.__DEBUGGING_PORT_KEY) or self.origin_debugger_port
 
     @property
     def debugging_forwarder(self) -> SshProcess | None:
@@ -322,23 +336,23 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
         self.cache[self.__DEBUGGING_FORWARDER_KEY] = value
 
     @property
-    def origin_pydev_port(self) -> int:
-        """The pydevd port on the origin."""
-        return self.args.metadata.debugger_settings.port
+    def origin_debugger_port(self) -> int:
+        """The debugger port on the origin."""
+        return self.debugger.port
 
     def enable_debugger_forwarding(self, ssh: SshConnectionDetail) -> None:
-        """Enable pydevd port forwarding from the origin."""
+        """Enable debugger port forwarding from the origin."""
         if not self.debugging_enabled:
             return
 
-        endpoint = ('localhost', self.origin_pydev_port)
+        endpoint = ('localhost', self.origin_debugger_port)
         forwards = [endpoint]
 
         self.debugging_forwarder = create_ssh_port_forwards(self.args, ssh, forwards)
 
         port_forwards = self.debugging_forwarder.collect_port_forwards()
 
-        self.state[self.__PYDEVD_PORT_KEY] = port = port_forwards[endpoint]
+        self.state[self.__DEBUGGING_PORT_KEY] = port = port_forwards[endpoint]
 
         display.info(f'Remote debugging of {self.name!r} is available on port {port}.', verbosity=1)
 
@@ -355,19 +369,6 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
 
         self.debugging_forwarder.wait()
 
-    def get_pydevd_settrace_arguments(self) -> dict[str, object]:
-        """Get settrace arguments for pydevd."""
-        return self.args.metadata.debugger_settings.settrace | dict(
-            host="localhost",
-            port=self.pydevd_port,
-        )
-
-    def get_pydevd_environment_variables(self) -> dict[str, str]:
-        """Get environment variables needed to configure pydevd for debugging."""
-        return dict(
-            PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(self.get_source_mapping().items())),
-        )
-
     def get_source_mapping(self) -> dict[str, str]:
         """Get the source mapping from the given metadata."""
         from . import data_context
@@ -396,10 +397,9 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
 
         display.info('Activating remote debugging of ansible-test.', verbosity=1)
 
-        os.environ.update(self.get_pydevd_environment_variables())
+        os.environ.update(self.debugger.get_environment_variables(self))
 
-        debugging_module = importlib.import_module(self.args.metadata.debugger_settings.module)
-        debugging_module.settrace(**self.get_pydevd_settrace_arguments())
+        self.debugger.activate_debugger(self)
 
         pass  # pylint: disable=unnecessary-pass  # when suspend is True, execution pauses here -- it's also a convenient place to put a breakpoint
 
@@ -411,9 +411,11 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
         if not self.args.metadata.debugger_flags.ansiballz:
             return {}
 
-        return dict(
-            _ansible_ansiballz_debugger_config=json.dumps(self.get_ansiballz_debugger_config()),
-        )
+        debug_type = self.debugger.get_debug_type()
+
+        return {
+            f"_ansible_ansiballz_{debug_type}_config": json.dumps(self.get_ansiballz_debugger_config()),
+        }
 
     def get_ansiballz_environment_variables(self) -> dict[str, t.Any]:
         """
@@ -423,20 +425,18 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
         if not self.args.metadata.debugger_flags.ansiballz:
             return {}
 
-        return dict(
-            _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG=json.dumps(self.get_ansiballz_debugger_config()),
-        )
+        debug_type = self.debugger.get_debug_type().upper()
+
+        return {
+            f"_ANSIBLE_ANSIBALLZ_{debug_type}_CONFIG": json.dumps(self.get_ansiballz_debugger_config()),
+        }
 
     def get_ansiballz_debugger_config(self) -> dict[str, t.Any]:
         """
         Return config for remote debugging of AnsiballZ modules.
         When delegating, this function must be called after delegation.
         """
-        debugger_config = dict(
-            module=self.args.metadata.debugger_settings.module,
-            settrace=self.get_pydevd_settrace_arguments(),
-            source_mapping=self.get_source_mapping(),
-        )
+        debugger_config = self.debugger.get_ansiballz_config(self)
 
         display.info(f'>>> Debugger Config ({self.name} AnsiballZ)\n{json.dumps(debugger_config, indent=4)}', verbosity=3)
 
@@ -451,8 +451,8 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta):
             return {}
 
         debugger_config = dict(
-            args=['-m', 'pydevd', '--client', 'localhost', '--port', str(self.pydevd_port)] + self.args.metadata.debugger_settings.args + ['--file'],
-            env=self.get_pydevd_environment_variables(),
+            args=self.debugger.get_cli_arguments(self),
+            env=self.debugger.get_environment_variables(self),
         )
 
         display.info(f'>>> Debugger Config ({self.name} Ansible CLI)\n{json.dumps(debugger_config, indent=4)}', verbosity=3)
@@ -597,9 +597,9 @@ class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[Con
         return self.controller_profile.name
 
     @property
-    def pydevd_port(self) -> int:
+    def debugger_port(self) -> int:
         """The pydevd port to use."""
-        return self.controller_profile.pydevd_port
+        return self.controller_profile.debugger_port
 
     def get_controller_target_connections(self) -> list[SshConnection]:
         """Return SSH connection(s) for accessing the host as a target from the controller."""

test/lib/ansible_test/_internal/metadata.py

@@ -22,6 +22,9 @@ from .diff import (
     FileDiff,
 )
 
+if t.TYPE_CHECKING:
+    from .debugging import DebuggerSettings
+
 
 class Metadata:
     """Metadata object for passing data to delegated tests."""
@@ -71,7 +74,7 @@ class Metadata:
             ansible_test_root=self.ansible_test_root,
             collection_root=self.collection_root,
             debugger_flags=dataclasses.asdict(self.debugger_flags),
-            debugger_settings=dataclasses.asdict(self.debugger_settings) if self.debugger_settings else None,
+            debugger_settings=self.debugger_settings.as_dict() if self.debugger_settings else None,
         )
 
     def to_file(self, path: str) -> None:
@@ -91,6 +94,8 @@ class Metadata:
     @staticmethod
     def from_dict(data: dict[str, t.Any]) -> Metadata:
         """Return metadata loaded from the specified dictionary."""
+        from .debugging import DebuggerSettings
+
         metadata = Metadata(
             debugger_flags=DebuggerFlags(**data['debugger_flags']),
         )
@@ -103,7 +108,7 @@ class Metadata:
         metadata.ansible_lib_root = data['ansible_lib_root']
         metadata.ansible_test_root = data['ansible_test_root']
         metadata.collection_root = data['collection_root']
-        metadata.debugger_settings = DebuggerSettings(**data['debugger_settings']) if data['debugger_settings'] else None
+        metadata.debugger_settings = DebuggerSettings.from_dict(data['debugger_settings']) if data['debugger_settings'] else None
         metadata.loaded = True
 
         return metadata
@@ -155,46 +160,6 @@ class ChangeDescription:
         return changes
 
 
-@dataclasses.dataclass(frozen=True, kw_only=True)
-class DebuggerSettings:
-    """Settings for remote debugging."""
-
-    module: str | None = None
-    """
-    The Python module to import.
-    This should be pydevd or a derivative.
-    If not provided it will be auto-detected.
-    """
-
-    package: str | None = None
-    """
-    The Python package to install for debugging.
-    If `None` then the package will be auto-detected.
-    If an empty string, then no package will be installed.
-    """
-
-    settrace: dict[str, object] = dataclasses.field(default_factory=dict)
-    """
-    Options to pass to the `{module}.settrace` method.
-    Used for running AnsiballZ modules only.
-    The `host` and `port` options will be provided by ansible-test.
-    The `suspend` option defaults to `False`.
-    """
-
-    args: list[str] = dataclasses.field(default_factory=list)
-    """
-    Arguments to pass to `pydevd` on the command line.
-    Used for running Ansible CLI programs only.
-    The `--client` and `--port` options will be provided by ansible-test.
-    """
-
-    port: int = 5678
-    """
-    The port on the origin host which is listening for incoming connections from pydevd.
-    SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed.
-    """
-
-
 @dataclasses.dataclass(frozen=True, kw_only=True)
 class DebuggerFlags:
     """Flags for enabling specific debugging features."""

test/lib/ansible_test/_internal/python_requirements.py

@@ -170,11 +170,11 @@ def install_requirements(
 
     from .host_profiles import DebuggableProfile
 
-    if isinstance(host_profile, DebuggableProfile) and host_profile.debugging_enabled and args.metadata.debugger_settings.package:
+    if isinstance(host_profile, DebuggableProfile) and host_profile.debugger and host_profile.debugger.get_python_package():
         commands.append(PipInstall(
             requirements=[],
             constraints=[],
-            packages=[args.metadata.debugger_settings.package],
+            packages=[host_profile.debugger.get_python_package()],
         ))
 
     if not commands:

test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg

@@ -21,6 +21,7 @@ disable=
     broad-exception-raised,  # many exceptions with no need for a custom type
     too-few-public-methods,
     too-many-public-methods,
+    too-many-ancestors,
     too-many-arguments,
     too-many-branches,
     too-many-instance-attributes,

test/sanity/code-smell/mypy/ansible-core.ini

@@ -133,3 +133,6 @@ ignore_missing_imports = True
 
 [mypy-jinja2.nativetypes]
 ignore_missing_imports = True
+
+[mypy-debugpy]
+ignore_missing_imports = True

test/sanity/code-smell/mypy/ansible-test.ini

@@ -77,3 +77,9 @@ ignore_missing_imports = True
 
 [mypy-py._path.local]
 ignore_missing_imports = True
+
+[mypy-debugpy]
+ignore_missing_imports = True
+
+[mypy-debugpy.server]
+ignore_missing_imports = True