winrm - quota retry handling (#83656)

* winrm - quota retry handling

Add a retry attempt when receiving ERROR_WSMAN_QUOTA_MAX_OPERATIONS when
starting a command. This can occur when running a loop with multiple
iterations or an action plugin that runs multiple commands.

* Update pywinrm constraint for test

* Add verbose hint and mark test as destructive

changelogs/fragments/winrm-quota.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - winrm - Add retry after exceeding commands per user quota that can occur in loops and action plugins running
+    multiple commands.

lib/ansible/plugins/connection/winrm.py

@@ -207,6 +207,14 @@ except ImportError as e:
     HAS_WINRM = False
     WINRM_IMPORT_ERR = e
 
+try:
+    from winrm.exceptions import WSManFaultError
+except ImportError:
+    # This was added in pywinrm 0.5.0, we just use our no-op exception for
+    # older versions which won't be able to handle this scenario.
+    class WSManFaultError(Exception):  # type: ignore[no-redef]
+        pass
+
 try:
     import xmltodict
     HAS_XMLTODICT = True
@@ -633,7 +641,11 @@ class Connection(ConnectionBase):
         command_id = None
         try:
             stdin_push_failed = False
-            command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args), console_mode_stdin=(stdin_iterator is None))
+            command_id = self._winrm_run_command(
+                to_bytes(command),
+                tuple(map(to_bytes, args)),
+                console_mode_stdin=(stdin_iterator is None),
+            )
 
             try:
                 if stdin_iterator:
@@ -697,6 +709,39 @@ class Connection(ConnectionBase):
 
                     display.warning("Failed to cleanup running WinRM command, resources might still be in use on the target server")
 
+    def _winrm_run_command(
+        self,
+        command: bytes,
+        args: tuple[bytes, ...],
+        console_mode_stdin: bool = False,
+    ) -> str:
+        """Starts a command with handling when the WSMan quota is exceeded."""
+        try:
+            return self.protocol.run_command(
+                self.shell_id,
+                command,
+                args,
+                console_mode_stdin=console_mode_stdin,
+            )
+        except WSManFaultError as fault_error:
+            if fault_error.wmierror_code != 0x803381A6:
+                raise
+
+            # 0x803381A6 == ERROR_WSMAN_QUOTA_MAX_OPERATIONS
+            # WinRS does not decrement the operation count for commands,
+            # only way to avoid this is to re-create the shell. This is
+            # important for action plugins that might be running multiple
+            # processes in the same connection.
+            display.vvvvv("Shell operation quota exceeded, re-creating shell", host=self._winrm_host)
+            self.close()
+            self._connect()
+            return self.protocol.run_command(
+                self.shell_id,
+                command,
+                args,
+                console_mode_stdin=console_mode_stdin,
+            )
+
     def _connect(self) -> Connection:
 
         if not HAS_WINRM:

test/integration/targets/connection_winrm/aliases

@@ -1,3 +1,4 @@
+destructive
 windows
 shippable/windows/group1
 shippable/windows/smoketest

test/integration/targets/connection_winrm/tests.yml

@@ -41,3 +41,20 @@
   - assert:
       that:
       - timeout_cmd.msg == 'The win_shell action failed to execute in the expected time frame (5) and was terminated'
+
+  - name: get WinRM quota value
+    win_shell: (Get-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser).Value
+    changed_when: false
+    register: winrm_quota
+
+  - block:
+    - name: set WinRM quota to lower value
+      win_shell: Set-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser 3
+
+    - name: run ping with loop to exceed quota
+      win_ping:
+      loop: '{{ range(0, 4) }}'
+
+    always:
+    - name: reset WinRM quota value
+      win_shell: Set-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser {{ winrm_quota.stdout | trim }}

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -1,7 +1,7 @@
 # do not add a cryptography or pyopenssl constraint to this file, they require special handling, see get_cryptography_requirements in python_requirements.py
 # do not add a coverage constraint to this file, it is handled internally by ansible-test
 pypsrp < 1.0.0  # in case the next major version is too big of a change
-pywinrm >= 0.4.3  # support for Python 3.11
+pywinrm >= 0.5.0  # support for WSManFaultError and type annotation
 pytest >= 4.5.0  # pytest 4.5.0 added support for --strict-markers
 ntlm-auth >= 1.3.0 # message encryption support using cryptography
 requests-ntlm >= 1.1.0 # message encryption support