Ensure delegate vars calculation has correct loop context. Fixes #37132 (#71477)

changelogs/fragments/37132-delegate-to-loop-context.yml

@@ -0,0 +1,4 @@
+bugfixes:
+- delegate_to - Ensure that calculating ``delegate_to`` vars with a loop
+  uses the correct context to correctly evaluate the loop
+  (https://github.com/ansible/ansible/issues/37132)

lib/ansible/vars/manager.py

@@ -517,6 +517,12 @@ class VariableManager:
         return variables
 
     def _get_delegated_vars(self, play, task, existing_variables):
+        # This method has a lot of code copied from ``TaskExecutor._get_loop_items``
+        # if this is failing, and ``TaskExecutor._get_loop_items`` is not
+        # then more will have to be copied here.
+        # TODO: dedupe code here and with ``TaskExecutor._get_loop_items``
+        #       this may be possible once we move pre-processing pre fork
+
         if not hasattr(task, 'loop'):
             # This "task" is not a Task, so we need to skip it
             return {}, None
@@ -525,16 +531,41 @@ class VariableManager:
         # as we're fetching vars before post_validate has been called on
         # the task that has been passed in
         vars_copy = existing_variables.copy()
+
+        # get search path for this task to pass to lookup plugins
+        vars_copy['ansible_search_path'] = task.get_search_path()
+
+        # ensure basedir is always in (dwim already searches here but we need to display it)
+        if self._loader.get_basedir() not in vars_copy['ansible_search_path']:
+            vars_copy['ansible_search_path'].append(self._loader.get_basedir())
+
         templar = Templar(loader=self._loader, variables=vars_copy)
 
         items = []
         has_loop = True
         if task.loop_with is not None:
             if task.loop_with in lookup_loader:
+                fail = True
+                if task.loop_with == 'first_found':
+                    # first_found loops are special. If the item is undefined then we want to fall through to the next
+                    fail = False
                 try:
                     loop_terms = listify_lookup_plugin_terms(terms=task.loop, templar=templar,
-                                                             loader=self._loader, fail_on_undefined=True, convert_bare=False)
+                                                             loader=self._loader, fail_on_undefined=fail, convert_bare=False)
-                    items = wrap_var(lookup_loader.get(task.loop_with, loader=self._loader, templar=templar).run(terms=loop_terms, variables=vars_copy))
+
+                    if not fail:
+                        loop_terms = [t for t in loop_terms if not templar.is_template(t)]
+
+                    mylookup = lookup_loader.get(task.loop_with, loader=self._loader, templar=templar)
+
+                    # give lookup task 'context' for subdir (mostly needed for first_found)
+                    for subdir in ['template', 'var', 'file']:  # TODO: move this to constants?
+                        if subdir in task.action:
+                            break
+                    setattr(mylookup, '_subdir', subdir + 's')
+
+                    items = wrap_var(mylookup.run(terms=loop_terms, variables=vars_copy))
+
                 except AnsibleTemplateError:
                     # This task will be skipped later due to this, so we just setup
                     # a dummy array for the later code so it doesn't fail

test/integration/targets/delegate_to/delegate_to_lookup_context.yml

@@ -0,0 +1,4 @@
+- hosts: localhost
+  gather_facts: false
+  roles:
+    - delegate_to_lookup_context

test/integration/targets/delegate_to/roles/delegate_to_lookup_context/tasks/main.yml

@@ -0,0 +1,5 @@
+- name: sends SQL template files to mysql host(s)
+  debug:
+    msg: "{{ item }}"
+  with_fileglob: ../templates/*.j2
+  delegate_to: localhost

test/integration/targets/delegate_to/roles/delegate_to_lookup_context/templates/one.j2

@@ -0,0 +1 @@
+{{ inventory_hostname }}

test/integration/targets/delegate_to/roles/delegate_to_lookup_context/templates/two.j2

@@ -0,0 +1 @@
+{{ inventory_hostname }}

test/integration/targets/delegate_to/runme.sh

@@ -71,5 +71,5 @@ ln -s python secondpython
 )
 ansible-playbook verify_interpreter.yml -i inventory_interpreters -v "$@"
 ansible-playbook discovery_applied.yml -i inventory -v "$@"
-
 ansible-playbook resolve_vars.yml -i inventory -v "$@"
+ansible-playbook test_delegate_to_lookup_context.yml -i inventory -v "$@"

test/integration/targets/delegate_to/test_delegate_to_lookup_context.yml

@@ -0,0 +1,12 @@
+- hosts: localhost
+  gather_facts: false
+  vars:
+    verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verbosity) }}"
+  tasks:
+    - command: ansible-playbook {{ verbosity }} delegate_to_lookup_context.yml
+      register: result
+
+    - assert:
+        that:
+          - >
+            '[WARNING]: Unable to find' not in result.stderr