Fix marker handling in templating (#85690)

* allow markers to pass through template lookup
* avoid tripping markers within Jinja generated code

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/template_lookup_skip_finalize.yml

@@ -0,0 +1,6 @@
+bugfixes:
+  - template lookup - Skip finalization on the internal templating operation to allow markers to be returned and handled by, e.g. the ``default`` filter.
+    Previously, finalization tripped markers, causing an exception to end processing of the current template pipeline.
+    (https://github.com/ansible/ansible/issues/85674)
+  - templating - Avoid tripping markers within Jinja generated code.
+    (https://github.com/ansible/ansible/issues/85674)

lib/ansible/_internal/_templating/_engine.py

@@ -44,7 +44,7 @@ from ._jinja_bits import (
     _finalize_template_result,
     FinalizeMode,
 )
-from ._jinja_common import _TemplateConfig, MarkerError, ExceptionMarker
+from ._jinja_common import _TemplateConfig, MarkerError, ExceptionMarker, JinjaCallContext
 from ._lazy_containers import _AnsibleLazyTemplateMixin
 from ._marker_behaviors import MarkerBehavior, FAIL_ON_UNDEFINED
 from ._transform import _type_transform_mapping
@@ -260,6 +260,7 @@ class TemplateEngine:
             with (
                 TemplateContext(template_value=variable, templar=self, options=options, stop_on_template=stop_on_template) as ctx,
                 DeprecatedAccessAuditContext.when(ctx.is_top_level),
+                JinjaCallContext(accept_lazy_markers=True),  # let default Jinja marker behavior apply, since we're descending into a new template
             ):
                 try:
                     if not value_is_str:

lib/ansible/plugins/lookup/template.py

@@ -107,6 +107,7 @@ from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.template import trust_as_template
 from ansible._internal._templating import _template_vars
+from ansible._internal._templating._engine import TemplateOptions, TemplateOverrides
 from ansible.utils.display import Display
 
 
@@ -174,7 +175,11 @@ class LookupModule(LookupBase):
                 )
 
                 data_templar = templar.copy_with_new_env(available_variables=vars, searchpath=searchpath)
-                res = data_templar.template(template_data, escape_backslashes=False, overrides=overrides)
+                # use the internal template API to avoid forced top-level finalization behavior imposed by the public API
+                res = data_templar._engine.template(template_data, options=TemplateOptions(
+                    escape_backslashes=False,
+                    overrides=TemplateOverrides.from_kwargs(overrides),
+                ))
 
                 ret.append(res)
             else:

test/units/_internal/templating/test_jinja_bits.py

@@ -9,10 +9,11 @@ from contextlib import nullcontext
 import pytest
 import pytest_mock
 
+from ansible._internal._templating._access import NotifiableAccessContextBase
 from ansible.errors import AnsibleUndefinedVariable, AnsibleTemplateError
 from ansible._internal._templating._errors import AnsibleTemplatePluginRuntimeError
 from ansible.module_utils._internal._datatag import AnsibleTaggedObject
-from ansible._internal._templating._jinja_common import CapturedExceptionMarker, MarkerError, Marker, UndefinedMarker, JinjaCallContext
+from ansible._internal._templating._jinja_common import CapturedExceptionMarker, MarkerError, Marker, UndefinedMarker
 from ansible._internal._templating._utils import TemplateContext
 from ansible._internal._datatag._tags import TrustedAsTemplate
 from ansible._internal._templating._jinja_bits import (AnsibleEnvironment, TemplateOverrides, _TEMPLATE_OVERRIDE_FIELD_NAMES, defer_template_error,
@@ -445,6 +446,15 @@ def test_mutation_methods(template: str, result: object) -> None:
     assert TemplateEngine().template(TRUST.tag(template)) == result
 
 
+class ExampleMarkerAccessTracker(NotifiableAccessContextBase):
+    def __init__(self) -> None:
+        self._type_interest = frozenset(Marker._concrete_subclasses)
+        self._markers: list[Marker] = []
+
+    def _notify(self, o: Marker) -> None:
+        self._markers.append(o)
+
+
 @pytest.mark.parametrize("template", (
     '{{ adict["bogus"] | default("ok") }}',
     '{{ adict.bogus | default("ok") }}',
@@ -454,6 +464,7 @@ def test_marker_access_getattr_and_getitem(template: str) -> None:
     # the absence of a JinjaCallContext should cause the access done by getattr and getitem not to trip when a marker is encountered
     assert TemplateEngine(variables=dict(adict={})).template(TRUST.tag(template)) == "ok"
 
-    with pytest.raises(AnsibleUndefinedVariable):
-        with JinjaCallContext(accept_lazy_markers=False):  # the access done by getattr and getitem should immediately trip when a marker is encountered
-            TemplateEngine(variables=dict(adict={})).template(TRUST.tag(template))
+    with ExampleMarkerAccessTracker() as tracker:  # the access done by getattr and getitem should immediately trip when a marker is encountered
+        TemplateEngine(variables=dict(adict={})).template(TRUST.tag(template))
+
+    assert type(tracker._markers[0]) is UndefinedMarker  # pylint: disable=unidiomatic-typecheck

test/units/_internal/templating/test_templar.py

@@ -1111,3 +1111,15 @@ def test_filter_generator() -> None:
     te = TemplateEngine(variables=variables)
     te.template(TRUST.tag("{{ bar }}"))
     te.template(TRUST.tag("{{ lookup('vars', 'bar') }}"))
+
+
+def test_call_context_reset() -> None:
+    """Ensure that new template invocations do not inherit trip behavior from running Jinja plugins."""
+    templar = TemplateEngine(variables=dict(
+        somevar=TRUST.tag("{{ somedict.somekey | default('ok') }}"),
+        somedict=dict(
+            somekey=TRUST.tag("{{ not_here }}"),
+        )
+    ))
+
+    assert templar.template(TRUST.tag("{{ lookup('vars', 'somevar') }}")) == 'ok'

test/units/plugins/lookup/test_template.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import pathlib
+
+from ansible._internal._templating._utils import Omit
+from ansible.parsing.dataloader import DataLoader
+from ansible.template import Templar, trust_as_template
+
+
+def test_no_finalize_marker_passthru(tmp_path: pathlib.Path) -> None:
+    """Return an Undefined marker from a template lookup to ensure that the internal templating operation does not finalize its result."""
+    template_path = tmp_path / 'template.txt'
+    template_path.write_text("{{ bogusvar }}")
+
+    templar = Templar(loader=DataLoader(), variables=dict(template_path=str(template_path)))
+
+    assert templar.template(trust_as_template('{{ lookup("template", template_path) | default("pass") }}')) == "pass"
+
+
+def test_no_finalize_omit_passthru(tmp_path: pathlib.Path) -> None:
+    """Return an Omit scalar from a template lookup to ensure that the internal templating operation does not finalize its result."""
+    template_path = tmp_path / 'template.txt'
+    template_path.write_text("{{ omitted }}")
+
+    data = dict(omitted=trust_as_template("{{ omit }}"), template_path=str(template_path))
+
+    # The result from the lookup should be an Omit value, since the result of the template lookup's internal templating call should not be finalized.
+    # If it were, finalize would trip the Omit and raise an error about a top-level template result resolving to an Omit scalar.
+    res = Templar(loader=DataLoader(), variables=data).template(trust_as_template("{{ lookup('template', template_path) | type_debug }}"))
+
+    assert res == type(Omit).__name__