Fix certbot tests after updating pytest to 9.0.2 (#10545)

Fixes  #10518.

`tools/pinning/current/repin.sh` is not run; only pytest version is
updated. This is because `pypinning` had a bunch of syntax changes that
seem simply but I believe should be in a separate PR, which I think
should be done after this to collect all repin changes.

As discussed further in #10518, these issues were caused by pytest's
internalization of pytest-subtest, which had several implementation
changes.

To fix these, we simply no longer use subtest in the failing tests. The
test in acme is now parametrized instead, and the tests in apache only
ever had a single parameter.

To use parametrization in the acme test, I converted `DNSTest` from
unittest to pytest style, which was pretty straightforward. The only
note there is that while it would be nice to make `ec_secp384r1_key` a
fixture, you [can't use fixtures in
parameters](https://github.com/pytest-dev/pytest/issues/349). You could
use requests, but that seemed less clear and messier, because then you'd
be checking the value of the parameter and only sometimes loading it.
Could also make it a global variable, but that didn't really seem
necessary, as it's only called twice. Happy to consider other options,
not strongly tied to this one, just seemed nicest to me.

acme/src/acme/_internal/tests/challenges_test.py

@@ -1,5 +1,6 @@
 """Tests for acme.challenges."""
 import sys
+from typing import TYPE_CHECKING
 import unittest
 from unittest import mock
 import urllib.parse as urllib_parse
@@ -267,71 +268,82 @@ class HTTP01Test(unittest.TestCase):
         assert not self.msg.update(token=b'..').good_token
 
 
-class DNSTest(unittest.TestCase):
+class TestDNS:
 
-    def setUp(self):
+    if TYPE_CHECKING:
         from acme.challenges import DNS
-        self.msg = DNS(token=jose.b64decode(
-            b'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))
-        self.jmsg = {
+
+    @pytest.fixture
+    def jmsg(self) -> dict:
+        jmsg = {
             'type': 'dns',
             'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
         }
+        return jmsg
 
-    def test_to_partial_json(self):
-        assert self.jmsg == self.msg.to_partial_json()
-
-    def test_from_json(self):
+    @pytest.fixture
+    def msg(self) -> 'DNS':
         from acme.challenges import DNS
-        assert self.msg == DNS.from_json(self.jmsg)
+        msg = DNS(token=jose.b64decode(
+            b'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))
+        return msg
 
-    def test_from_json_hashable(self):
+    def test_to_partial_json(self, msg: 'DNS', jmsg: dict):
+        assert jmsg == msg.to_partial_json()
+
+    def test_from_json(self, msg: 'DNS', jmsg: dict):
         from acme.challenges import DNS
-        hash(DNS.from_json(self.jmsg))
+        assert msg == DNS.from_json(jmsg)
 
-    def test_gen_check_validation(self):
-        ec_key_secp384r1 = JWKEC(key=test_util.load_ecdsa_private_key('ec_secp384r1_key.pem'))
-        for key, alg in [(KEY, jose.RS256), (ec_key_secp384r1, jose.ES384)]:
-            with self.subTest(key=key, alg=alg):
-                assert self.msg.check_validation(
-                    self.msg.gen_validation(key, alg=alg), key.public_key())
+    def test_from_json_hashable(self, jmsg: dict):
+        from acme.challenges import DNS
+        hash(DNS.from_json(jmsg))
 
-    def test_gen_check_validation_wrong_key(self):
+    # Using fixtures in parametrize is an open issue
+    # https://github.com/pytest-dev/pytest/issues/349
+    @pytest.mark.parametrize("key, alg", [
+        (KEY, jose.RS256),
+        (JWKEC(key=test_util.load_ecdsa_private_key('ec_secp384r1_key.pem')), jose.ES384)])
+    def test_gen_check_validation(self, key, alg, msg: 'DNS'):
+        assert msg.check_validation(
+            msg.gen_validation(key, alg=alg), key.public_key())
+
+    def test_gen_check_validation_wrong_key(self, msg: 'DNS'):
         key2 = jose.JWKRSA.load(test_util.load_vector('rsa1024_key.pem'))
-        assert not self.msg.check_validation(
-            self.msg.gen_validation(KEY), key2.public_key())
+        assert not msg.check_validation(
+            msg.gen_validation(KEY), key2.public_key())
 
-    def test_check_validation_wrong_payload(self):
+    def test_check_validation_wrong_payload(self, msg: 'DNS'):
         validations = tuple(
             jose.JWS.sign(payload=payload, alg=jose.RS256, key=KEY)
             for payload in (b'', b'{}')
         )
         for validation in validations:
-            assert not self.msg.check_validation(
+            assert not msg.check_validation(
                 validation, KEY.public_key())
 
-    def test_check_validation_wrong_fields(self):
+    def test_check_validation_wrong_fields(self, msg: 'DNS'):
         bad_validation = jose.JWS.sign(
-            payload=self.msg.update(
+            payload=msg.update(
                 token=b'x' * 20).json_dumps().encode('utf-8'),
             alg=jose.RS256, key=KEY)
-        assert not self.msg.check_validation(bad_validation, KEY.public_key())
+        assert not msg.check_validation(bad_validation, KEY.public_key())
 
-    def test_gen_response(self):
+    def test_gen_response(self, msg: 'DNS'):
         with mock.patch('acme.challenges.DNS.gen_validation') as mock_gen:
             mock_gen.return_value = mock.sentinel.validation
-            response = self.msg.gen_response(KEY)
+            response = msg.gen_response(KEY)
         from acme.challenges import DNSResponse
         assert isinstance(response, DNSResponse)
         assert response.validation == mock.sentinel.validation
 
-    def test_validation_domain_name(self):
-        assert '_acme-challenge.le.wtf' == self.msg.validation_domain_name('le.wtf')
+    def test_validation_domain_name(self, msg: 'DNS'):
+        assert '_acme-challenge.le.wtf' == msg.validation_domain_name('le.wtf')
 
-    def test_validation_domain_name_ecdsa(self):
+    def test_validation_domain_name_ecdsa(self, msg: 'DNS'):
         ec_key_secp384r1 = JWKEC(key=test_util.load_ecdsa_private_key('ec_secp384r1_key.pem'))
-        assert self.msg.check_validation(
-            self.msg.gen_validation(ec_key_secp384r1, alg=jose.ES384),
+        assert msg.check_validation(
+            msg.gen_validation(ec_key_secp384r1, alg=jose.ES384),
             ec_key_secp384r1.public_key()) is True
 
 

certbot-apache/src/certbot_apache/_internal/tests/dualnode_test.py

@@ -337,10 +337,10 @@ class DualParserNodeTest(unittest.TestCase):  # pylint: disable=too-many-public-
         self.block.secondary.find_blocks = find_blocks_secondary
 
         blocks = self.block.find_blocks("anything")
+        assert len(blocks) == 1
         for block in blocks:
-            with self.subTest(block=block):
-                assert block.primary == block.secondary
-                assert block.primary is not block.secondary
+            assert block.primary == block.secondary
+            assert block.primary is not block.secondary
 
     def test_find_dirs_no_pass_equal(self):
         notpassing1 = [augeasparser.AugeasDirectiveNode(name="notpassing",
@@ -357,10 +357,10 @@ class DualParserNodeTest(unittest.TestCase):  # pylint: disable=too-many-public-
         self.block.secondary.find_directives = find_dirs_secondary
 
         directives = self.block.find_directives("anything")
+        assert len(directives) == 1
         for directive in directives:
-            with self.subTest(directive=directive):
-                assert directive.primary == directive.secondary
-                assert directive.primary is not directive.secondary
+            assert directive.primary == directive.secondary
+            assert directive.primary is not directive.secondary
 
     def test_find_comments_no_pass_equal(self):
         notpassing1 = [augeasparser.AugeasCommentNode(comment="notpassing",
@@ -377,10 +377,10 @@ class DualParserNodeTest(unittest.TestCase):  # pylint: disable=too-many-public-
         self.block.secondary.find_comments = find_coms_secondary
 
         comments = self.block.find_comments("anything")
+        assert len(comments) == 1
         for comment in comments:
-            with self.subTest(comment=comment):
-                assert comment.primary == comment.secondary
-                assert comment.primary is not comment.secondary
+            assert comment.primary == comment.secondary
+            assert comment.primary is not comment.secondary
 
     def test_find_blocks_no_pass_notequal(self):
         notpassing1 = [augeasparser.AugeasBlockNode(name="notpassing",

tools/requirements.txt

@@ -132,7 +132,7 @@ pyproject-hooks==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
 pyrfc3339==2.1.0 ; python_version >= "3.10" and python_version < "4.0"
 pytest-cov==7.0.0 ; python_version >= "3.10" and python_version < "4.0"
 pytest-xdist==3.8.0 ; python_version >= "3.10" and python_version < "4.0"
-pytest==8.4.2 ; python_version >= "3.10" and python_version < "4.0"
+pytest==9.0.2 ; python_version >= "3.10" and python_version < "4.0"
 python-augeas==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0"
 python-digitalocean==1.17.0 ; python_version >= "3.10" and python_version < "4.0"