Rewrite tox config (#9666)

* rewrite tox config

* fix apacheconftest-with-pebble deps

* more fixes

* more fixes

* move comment up

* fix mock location

* bump cffi

* update oldest constraints

* Revert "fix mock location"

This reverts commit 561037bfad.

* fix apache test

* fix server cleanup

* fix some leaky sockets

* stop leaking sockets

* change less

* Update tox.ini

Co-authored-by: alexzorin <alex@zorin.id.au>

* Update tox.ini

Co-authored-by: alexzorin <alex@zorin.id.au>

* tweak contributing doc

---------

Co-authored-by: alexzorin <alex@zorin.id.au>

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -17,6 +17,8 @@ jobs:
         linux-py310:
           PYTHON_VERSION: 3.10
           TOXENV: py310
+        linux-isolated:
+          TOXENV: 'isolated-{acme,certbot,apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}'
         linux-boulder-v2-integration-certbot-oldest:
           PYTHON_VERSION: 3.7
           TOXENV: integration-certbot-oldest

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -14,7 +14,7 @@ jobs:
         windows-py37:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.7
-          TOXENV: py37-win
+          TOXENV: py-win
         windows-py39-cover:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
@@ -23,14 +23,10 @@ jobs:
           IMAGE_NAME: windows-2019
           PYTHON_VERSION: 3.9
           TOXENV: integration-certbot
-        linux-oldest-tests-1:
+        linux-oldest:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.7
-          TOXENV: '{acme,apache,apache-v2,certbot}-oldest'
+          TOXENV: oldest
-        linux-oldest-tests-2:
-          IMAGE_NAME: ubuntu-22.04
-          PYTHON_VERSION: 3.7
-          TOXENV: '{dns,nginx}-oldest'
         linux-py37:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.7
@@ -43,7 +39,7 @@ jobs:
           TOXENV: lint-posix
         linux-mypy:
           IMAGE_NAME: ubuntu-22.04
-          TOXENV: mypy-posix
+          TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.8

acme/acme/_internal/tests/crypto_util_test.py

@@ -29,11 +29,9 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         from acme.crypto_util import SSLSocket
 
         class _TestServer(socketserver.TCPServer):
-
+            def __init__(self, *args, **kwargs):
-            def server_bind(self):  # pylint: disable=missing-docstring
+                super().__init__(*args, **kwargs)
-                self.socket = SSLSocket(socket.socket(),
+                self.socket = SSLSocket(self.socket, certs)
-                        certs)
-                socketserver.TCPServer.server_bind(self)
 
         self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
         self.port = self.server.socket.getsockname()[1]
@@ -44,6 +42,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         if self.server_thread.is_alive():
             # The thread may have already terminated.
             self.server_thread.join()  # pragma: no cover
+        self.server.server_close()
 
     def _probe(self, name):
         from acme.crypto_util import probe_sni

acme/acme/_internal/tests/standalone_test.py

@@ -55,6 +55,7 @@ class HTTP01ServerTest(unittest.TestCase):
     def tearDown(self):
         self.server.shutdown()
         self.thread.join()
+        self.server.server_close()
 
     def test_index(self):
         response = requests.get(
@@ -88,25 +89,25 @@ class HTTP01ServerTest(unittest.TestCase):
 
     def test_timely_shutdown(self):
         from acme.standalone import HTTP01Server
-        server = HTTP01Server(('', 0), resources=set(), timeout=0.05)
+        with HTTP01Server(('', 0), resources=set(), timeout=0.05) as server:
-        server_thread = threading.Thread(target=server.serve_forever)
+            server_thread = threading.Thread(target=server.serve_forever)
-        server_thread.start()
+            server_thread.start()
 
-        client = socket.socket()
+            with socket.socket() as client:
-        client.connect(('localhost', server.socket.getsockname()[1]))
+                client.connect(('localhost', server.socket.getsockname()[1]))
 
-        stop_thread = threading.Thread(target=server.shutdown)
+                stop_thread = threading.Thread(target=server.shutdown)
-        stop_thread.start()
+                stop_thread.start()
-        server_thread.join(5.)
+                server_thread.join(5.)
 
-        is_hung = server_thread.is_alive()
+                is_hung = server_thread.is_alive()
-        try:
+                try:
-            client.shutdown(socket.SHUT_RDWR)
+                    client.shutdown(socket.SHUT_RDWR)
-        except: # pragma: no cover, pylint: disable=bare-except
+                except: # pragma: no cover, pylint: disable=bare-except
-            # may raise error because socket could already be closed
+                    # may raise error because socket could already be closed
-            pass
+                    pass
 
-        assert not is_hung, 'Server shutdown should not be hung'
+                assert not is_hung, 'Server shutdown should not be hung'
 
 
 @unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old")
@@ -133,6 +134,7 @@ class TLSALPN01ServerTest(unittest.TestCase):
     def tearDown(self):
         self.server.shutdown()  # pylint: disable=no-member
         self.thread.join()
+        self.server.server_close()
 
     # TODO: This is not implemented yet, see comments in standalone.py
     # def test_certs(self):
@@ -214,6 +216,8 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
             if prev_port:
                 assert prev_port == port
             prev_port = port
+        for server in servers.servers:
+            server.server_close()
 
 
 class HTTP01DualNetworkedServersTest(unittest.TestCase):

acme/acme/crypto_util.py

@@ -136,27 +136,33 @@ class SSLSocket:  # pylint: disable=too-few-public-methods
     def accept(self) -> Tuple[FakeConnection, Any]:  # pylint: disable=missing-function-docstring
         sock, addr = self.sock.accept()
 
-        context = SSL.Context(self.method)
-        context.set_options(SSL.OP_NO_SSLv2)
-        context.set_options(SSL.OP_NO_SSLv3)
-        context.set_tlsext_servername_callback(self._pick_certificate_cb)
-        if self.alpn_selection is not None:
-            context.set_alpn_select_callback(self.alpn_selection)
-
-        ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
-        ssl_sock.set_accept_state()
-
-        # This log line is especially desirable because without it requests to
-        # our standalone TLSALPN server would not be logged.
-        logger.debug("Performing handshake with %s", addr)
         try:
-            ssl_sock.do_handshake()
+            context = SSL.Context(self.method)
-        except SSL.Error as error:
+            context.set_options(SSL.OP_NO_SSLv2)
-            # _pick_certificate_cb might have returned without
+            context.set_options(SSL.OP_NO_SSLv3)
-            # creating SSL context (wrong server name)
+            context.set_tlsext_servername_callback(self._pick_certificate_cb)
-            raise socket.error(error)
+            if self.alpn_selection is not None:
+                context.set_alpn_select_callback(self.alpn_selection)
 
-        return ssl_sock, addr
+            ssl_sock = self.FakeConnection(SSL.Connection(context, sock))
+            ssl_sock.set_accept_state()
+
+            # This log line is especially desirable because without it requests to
+            # our standalone TLSALPN server would not be logged.
+            logger.debug("Performing handshake with %s", addr)
+            try:
+                ssl_sock.do_handshake()
+            except SSL.Error as error:
+                # _pick_certificate_cb might have returned without
+                # creating SSL context (wrong server name)
+                raise socket.error(error)
+
+            return ssl_sock, addr
+        except:
+            # If we encounter any error, close the new socket before reraising
+            # the exception.
+            sock.close()
+            raise
 
 
 def probe_sni(name: bytes, host: bytes, port: int = 443, timeout: int = 300,  # pylint: disable=too-many-arguments

certbot-apache/certbot_apache/_internal/tests/autohsts_test.py

@@ -128,11 +128,11 @@ class AutoHSTSTest(util.ApacheTest):
                           max_val
 
     def test_autohsts_update_noop(self):
-        with mock.patch("time.time") as mock_time:
+        with mock.patch("certbot_apache._internal.configurator.time") as mock_time_module:
             # Time mock is used to make sure that the execution does not
             # continue when no autohsts entries exist in pluginstorage
             self.config.update_autohsts(mock.MagicMock())
-            assert mock_time.called is False
+            assert not mock_time_module.time.called
 
     def test_autohsts_make_permanent_noop(self):
         self.config.storage.put = mock.MagicMock()

certbot/docs/contributing.rst

@@ -515,8 +515,8 @@ Steps:
    virtualenv. You can do this by following the instructions in the
    :ref:`Getting Started <getting_started>` section.
 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors.
-4. Run ``tox --skip-missing-interpreters`` to run the entire test suite
+4. Run ``tox --skip-missing-interpreters`` to run all the tests we recommend
-   including coverage. The ``--skip-missing-interpreters`` argument ignores
+   developers run locally. The ``--skip-missing-interpreters`` argument ignores
    missing versions of Python needed for running the tests. Fix any errors.
 5. If any documentation should be added or updated as part of the changes you
    have made, please include the documentation changes in your PR.

tools/install_and_test.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. Packages are installed using
-# pinned versions of all of our dependencies. See pip_install.py for more
-# information on the versions pinned to.
-import os
-import re
-import subprocess
-import sys
-
-
-def call_with_print(command):
-    print(command)
-    subprocess.check_call(command, shell=True)
-
-
-def main(args):
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
-
-    for requirement in args:
-        current_command = command[:]
-        current_command.append(requirement)
-        call_with_print(' '.join(current_command))
-        pkg = re.sub(r'\[\w+\]', '', requirement)
-
-        call_with_print(' '.join([
-            sys.executable, '-m', 'pytest', pkg]))
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

tools/oldest_constraints.txt

@@ -2,69 +2,69 @@
 # that script.
 apacheconfig==0.3.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 asn1crypto==0.24.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-astroid==2.12.13 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+astroid==2.15.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-attrs==22.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+attrs==22.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 boto3==1.15.15 ; python_full_version < "3.8.0" and python_version >= "3.7"
 botocore==1.18.15 ; python_full_version < "3.8.0" and python_version >= "3.7"
 certifi==2022.12.7 ; python_full_version < "3.8.0" and python_version >= "3.7"
-cffi==1.9.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
+cffi==1.11.5 ; python_full_version < "3.8.0" and python_version >= "3.7"
 chardet==3.0.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 cloudflare==1.5.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 colorama==0.4.6 ; python_full_version < "3.8.0" and sys_platform == "win32" and python_version >= "3.7"
 configargparse==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 configobj==5.0.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-coverage==6.5.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+coverage==7.2.3 ; python_version >= "3.7" and python_full_version < "3.8.0"
 cryptography==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
-cython==0.29.32 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
+cython==0.29.34 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
 dill==0.3.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 distlib==0.3.6 ; python_version >= "3.7" and python_full_version < "3.8.0"
 distro==1.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 dns-lexicon==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 dnspython==1.15.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
-exceptiongroup==1.0.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+exceptiongroup==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
 execnet==1.9.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-filelock==3.8.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
+filelock==3.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 funcsigs==0.4 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-future==0.18.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
+future==0.18.3 ; python_full_version < "3.8.0" and python_version >= "3.7"
 google-api-python-client==1.5.5 ; python_full_version < "3.8.0" and python_version >= "3.7"
 httplib2==0.9.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 idna==2.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-importlib-metadata==5.1.0 ; python_version >= "3.7" and python_version < "3.8"
+importlib-metadata==6.1.0 ; python_version >= "3.7" and python_version < "3.8"
-iniconfig==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+iniconfig==2.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ipaddress==1.0.16 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-isort==5.10.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+isort==5.11.5 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 jmespath==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 josepy==1.13.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-lazy-object-proxy==1.8.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+lazy-object-proxy==1.9.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 logger==1.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 mccabe==0.7.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-mypy-extensions==0.4.3 ; python_version >= "3.7" and python_full_version < "3.8.0"
+mypy-extensions==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-mypy==0.991 ; python_version >= "3.7" and python_full_version < "3.8.0"
+mypy==1.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ndg-httpsclient==0.3.2 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
 oauth2client==4.0.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
-packaging==22.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+packaging==23.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 parsedatetime==2.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pbr==1.8.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0"
-pip==22.3.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+pip==23.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
-platformdirs==2.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
+platformdirs==3.2.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pluggy==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 ply==3.4 ; python_full_version < "3.8.0" and python_version >= "3.7"
 py==1.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 pyasn1-modules==0.0.10 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyasn1==0.1.9 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pycparser==2.14 ; python_full_version < "3.8.0" and python_version >= "3.7"
-pylint==2.15.8 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+pylint==2.17.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 pyopenssl==17.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyparsing==2.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pyrfc3339==1.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pytest-cov==4.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-pytest-xdist==3.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+pytest-xdist==3.2.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
-pytest==7.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
+pytest==7.2.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
 python-augeas==0.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 python-dateutil==2.8.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
 python-digitalocean==1.11 ; python_full_version < "3.8.0" and python_version >= "3.7"
 pytz==2019.3 ; python_full_version < "3.8.0" and python_version >= "3.7"
-pywin32==305 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32"
+pywin32==306 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32"
 pyyaml==6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 requests-file==1.5.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
 requests==2.20.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
@@ -74,22 +74,22 @@ setuptools==41.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 six==1.11.0 ; python_full_version < "3.8.0" and python_version >= "3.7"
 tldextract==3.4.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 tomli==2.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
-tomlkit==0.11.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+tomlkit==0.11.7 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
 tox==1.9.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
 typed-ast==1.5.4 ; python_version < "3.8" and python_version >= "3.7"
 types-cryptography==3.3.23.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pyopenssl==22.1.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-pyopenssl==23.0.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pyrfc3339==1.1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-pyrfc3339==1.1.1.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-python-dateutil==2.8.19.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-python-dateutil==2.8.19.12 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-pytz==2022.6.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-pytz==2023.3.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-requests==2.28.11.5 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-requests==2.28.11.17 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-setuptools==65.6.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-setuptools==67.6.0.7 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-six==1.16.21.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-six==1.16.21.8 ; python_version >= "3.7" and python_full_version < "3.8.0"
-types-urllib3==1.26.25.4 ; python_version >= "3.7" and python_full_version < "3.8.0"
+types-urllib3==1.26.25.10 ; python_version >= "3.7" and python_full_version < "3.8.0"
-typing-extensions==4.4.0 ; python_version >= "3.7" and python_version < "3.8"
+typing-extensions==4.5.0 ; python_version < "3.8" and python_version >= "3.7"
 uritemplate==3.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7"
 urllib3==1.24.2 ; python_full_version < "3.8.0" and python_version >= "3.7"
-virtualenv==20.17.1 ; python_version >= "3.7" and python_full_version < "3.8.0"
+virtualenv==20.21.0 ; python_version >= "3.7" and python_full_version < "3.8.0"
 wheel==0.33.6 ; python_full_version < "3.8.0" and python_version >= "3.7"
-wrapt==1.14.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
+wrapt==1.15.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0"
-zipp==3.11.0 ; python_version >= "3.7" and python_version < "3.8"
+zipp==3.15.0 ; python_version >= "3.7" and python_version < "3.8"

tools/pinning/oldest/pyproject.toml

@@ -48,7 +48,7 @@ apacheconfig = "0.3.2"
 asn1crypto = "0.24.0"
 boto3 = "1.15.15"
 botocore = "1.18.15"
-cffi = "1.9.1"
+cffi = "1.11.5"
 chardet = "3.0.4"
 cloudflare = "1.5.1"
 configobj = "5.0.6"

tools/pip_install_editable.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-# pip installs packages in editable mode using pip_install.py
-import sys
-
-import pip_install
-
-
-def main(args):
-    new_args = []
-    for arg in args:
-        new_args.append('-e')
-        new_args.append(arg)
-
-    pip_install.main(new_args)
-
-if __name__ == '__main__':
-    main(sys.argv[1:])

tox.ini

@@ -1,42 +1,66 @@
-# Tox (https://tox.readthedocs.io/) is a tool for running tests in
-# multiple virtualenvs. To use it, "pip install tox" and then run
-# "tox" from this directory.
-
 [tox]
-skipsdist = true
 # mypy doesn't current pass for us on Windows. Fixing that is being tracked by
 # https://github.com/certbot/certbot/issues/7803.
-envlist = {cover,lint}-{win,posix},mypy-posix
+envlist = {cover,lint}-{win,posix},mypy
+skipsdist = true
 
 [base]
-# pip installs the requested packages in editable mode
+pytest = python -m pytest {posargs}
-pip_install = python {toxinidir}/tools/pip_install_editable.py
+# Paths are listed on one line because tox seems to have inconsistent
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. All dependencies are pinned
-# to a specific version for increased stability for developers.
-install_and_test = python {toxinidir}/tools/install_and_test.py
-# Packages are listed on one line because tox seems to have inconsistent
 # behavior with substitutions that contain line continuations, see
 # https://github.com/tox-dev/tox/issues/2069 for more info.
-dns_packages = certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud
-win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx
-all_packages = {[base]win_all_packages} certbot-apache
 source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx
 
 [testenv]
 platform =
     win: win32
     posix: ^(?!.*win32).*$
-commands =
-    win: {[base]install_and_test} {[base]win_all_packages}
-    !win: {[base]install_and_test} {[base]all_packages}
-# We always recreate the virtual environment to avoid problems like
-# https://github.com/certbot/certbot/issues/7745.
-recreate = true
 setenv =
     PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto}
     PYTHONHASHSEED = 0
+# The default install command is python -I -m pip install {opts} {packages}
+install_command = python -I {toxinidir}/tools/pip_install.py {opts} {packages}
+deps =
+    -e acme[test]
+    -e certbot[test]
+    !win: -e certbot-apache[dev]
+    -e certbot-dns-cloudflare
+    -e certbot-dns-digitalocean
+    -e certbot-dns-dnsimple
+    -e certbot-dns-dnsmadeeasy
+    -e certbot-dns-gehirn
+    -e certbot-dns-google
+    -e certbot-dns-linode
+    -e certbot-dns-luadns
+    -e certbot-dns-nsone
+    -e certbot-dns-ovh
+    -e certbot-dns-rfc2136
+    -e certbot-dns-route53
+    -e certbot-dns-sakuracloud
+    -e certbot-nginx
+whitelist_externals =
+    echo
+    false
+# This and the next few testenvs are a workaround for
+# https://github.com/tox-dev/tox/issues/2858.
+commands =
+    echo "Unrecognized environment name {envname}"
+    false
+
+[testenv:py-win]
+commands =
+    {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx
+
+[testenv:py{,-posix}]
+# We want to test everything we do on Windows plus the Apache plugin.
+commands =
+    {[testenv:py-win]commands} certbot-apache
+
+[testenv:py3{,7,8,9,10,11}]
+commands = {[testenv:py]commands}
+
+[testenv:py3.{7,8,9,10,11}]
+commands = {[testenv:py]commands}
 
 [testenv:oldest]
 # Setting basepython allows the tests to fail fast if that version of Python
@@ -46,117 +70,108 @@ setenv =
 # This version should be kept in sync with the one declared in
 # tools/pinning/oldest/pyproject.toml.
 basepython = python3.7
-commands =
+setenv = CERTBOT_OLDEST=1
-    {[testenv]commands}
+commands = {[testenv:py]commands}
-setenv =
-    {[testenv]setenv}
-    CERTBOT_OLDEST=1
-
-[testenv:acme-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]install_and_test} acme[test]
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:apache-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-apache
-    pytest certbot-apache
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:apache-v2-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-apache[dev]
-    pytest certbot-apache
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:certbot-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test]
-    pytest certbot
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:dns-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] {[base]dns_packages}
-    pytest {[base]dns_packages}
-setenv =
-    {[testenv:oldest]setenv}
-
-[testenv:nginx-oldest]
-basepython =
-    {[testenv:oldest]basepython}
-commands =
-    {[base]pip_install} acme[test] certbot[test] certbot-nginx
-    pytest certbot-nginx
-setenv =
-    {[testenv:oldest]setenv}
 
 [testenv:cover{,-win,-posix}]
-commands =
+commands = python tox.cover.py
-    win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages} certbot-apache[dev]
-    python tox.cover.py
 
 [testenv:lint{,-win,-posix}]
-basepython = python3
+commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
-# separating into multiple invocations disables cross package
-# duplicate code checking; if one of the commands fails, others will
-# continue, but tox return code will reflect previous error
-commands =
-    win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages}
-    python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
 
-[testenv:mypy{,-win,-posix}]
+[testenv:mypy]
-basepython = python3
+deps =
+    {[testenv]deps}
+    -e certbot-ci
+commands = mypy {[base]source_paths}
+
+[testenv:isolated-acme]
+description = Tests acme without any Certbot components installed
+deps = -e acme[test]
+commands = {[base]pytest} acme
+
+[testenv:isolated-certbot]
+description = Tests Certbot without any additional plugins installed
+deps =
+    {[testenv:isolated-acme]deps}
+    -e certbot[test]
+commands = {[base]pytest} certbot
+
+[testenv:isolated-{apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}]
+description = Tests the plugin without installing any other plugins
+deps =
+    {[testenv:isolated-certbot]deps}
+    apache: -e certbot-apache[dev]
+    cloudflare: -e certbot-dns-cloudflare
+    digitalocean: -e certbot-dns-digitalocean
+    dnsimple: -e certbot-dns-dnsimple
+    dnsmadeeasy: -e certbot-dns-dnsmadeeasy
+    gehirn: -e certbot-dns-gehirn
+    google: -e certbot-dns-google
+    linode: -e certbot-dns-linode
+    luadns: -e certbot-dns-luadns
+    nsone: -e certbot-dns-nsone
+    ovh: -e certbot-dns-ovh
+    rfc2136: -e certbot-dns-rfc2136
+    route53: -e certbot-dns-route53
+    sakuracloud: -e certbot-dns-sakuracloud
+    nginx: -e certbot-nginx
 commands =
-    win: {[base]pip_install} {[base]win_all_packages}
+    apache: {[base]pytest} certbot-apache
-    !win: {[base]pip_install} {[base]all_packages} certbot-ci
+    cloudflare: {[base]pytest} certbot-dns-cloudflare
-    mypy {[base]source_paths}
+    digitalocean: {[base]pytest} certbot-dns-digitalocean
+    dnsimple: {[base]pytest} certbot-dns-dnsimple
+    dnsmadeeasy: {[base]pytest} certbot-dns-dnsmadeeasy
+    gehirn: {[base]pytest} certbot-dns-gehirn
+    google: {[base]pytest} certbot-dns-google
+    linode: {[base]pytest} certbot-dns-linode
+    luadns: {[base]pytest} certbot-dns-luadns
+    nsone: {[base]pytest} certbot-dns-nsone
+    ovh: {[base]pytest} certbot-dns-ovh
+    rfc2136: {[base]pytest} certbot-dns-rfc2136
+    route53: {[base]pytest} certbot-dns-route53
+    sakuracloud: {[base]pytest} certbot-dns-sakuracloud
+    nginx: {[base]pytest} certbot-nginx
 
 [testenv:apacheconftest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-apache
 commands =
-    {[base]pip_install} acme certbot certbot-apache
     {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test --debian-modules
 passenv =
     SERVER
 
 [testenv:apacheconftest-external-with-pebble]
-# Run apacheconftest with pebble and Certbot outside of tox's virtual
+description = Run apacheconftest with pebble and Certbot outside of the tox virtual environment.
-# environment.
+deps =
+    -e certbot-ci
 commands =
-    {[base]pip_install} certbot-ci
     {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules
 
 [testenv:apacheconftest-with-pebble]
-commands =
+deps =
-    {[base]pip_install} acme certbot certbot-apache
+    {[testenv:apacheconftest]deps}
-    {[testenv:apacheconftest-external-with-pebble]commands}
+    {[testenv:apacheconftest-external-with-pebble]deps}
+commands = {[testenv:apacheconftest-external-with-pebble]commands}
 
 [testenv:nginxroundtrip]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-apache
+    -e certbot-nginx
 commands =
-    {[base]pip_install} acme certbot certbot-apache certbot-nginx
     python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata
 
 [testenv:modification]
+deps =
 commands =
     python {toxinidir}/tests/modification-check.py
 
 [testenv:apache_compat]
+deps =
 commands =
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache .
@@ -167,6 +182,7 @@ passenv =
     DOCKER_*
 
 [testenv:nginx_compat]
+deps =
 commands =
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
@@ -177,8 +193,8 @@ passenv =
     DOCKER_*
 
 [testenv:docker_dev]
-# tests the Dockerfile-dev file to ensure development with it works
+description = Tests the Dockerfile-dev file to ensure development with it works as expected
-# as expected
+deps =
 commands =
     docker-compose run --rm --service-ports development bash -c 'tox -e lint'
 whitelist_externals =
@@ -186,9 +202,13 @@ whitelist_externals =
 passenv = DOCKER_*
 
 [testenv:integration]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-nginx
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-nginx certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests \
-    pytest certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
@@ -197,18 +217,25 @@ commands =
 passenv = DOCKER_*
 
 [testenv:integration-certbot]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble} \
         --cov=acme --cov=certbot --cov-report= \
         --cov-config=certbot-ci/certbot_integration_tests/.coveragerc
     coverage report --include 'certbot/*' --show-missing --fail-under=62
 
 [testenv:integration-dns-rfc2136]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-dns-rfc2136
+    -e certbot-ci
 commands =
-    {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests/rfc2136_tests \
-    pytest certbot-ci/certbot_integration_tests/rfc2136_tests \
         --acme-server=pebble --dns-server=bind \
         --numprocesses=1 \
         --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \
@@ -217,46 +244,45 @@ commands =
     coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=86
 
 [testenv:integration-external]
-# Run integration tests with Certbot outside of tox's virtual environment.
+description = Run integration tests with Certbot outside of the tox virtual environment.
+deps =
+    -e certbot-ci
 commands =
-    {[base]pip_install} certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests \
-    pytest certbot-ci/certbot_integration_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 
 [testenv:integration-certbot-oldest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-ci
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme certbot certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \
-    pytest certbot-ci/certbot_integration_tests/certbot_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
 [testenv:integration-nginx-oldest]
+deps =
+    -e acme
+    -e certbot
+    -e certbot-nginx
+    -e certbot-ci
 basepython =
     {[testenv:oldest]basepython}
 commands =
-    {[base]pip_install} acme certbot certbot-nginx certbot-ci
+    {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests \
-    pytest certbot-ci/certbot_integration_tests/nginx_tests \
         --acme-server={env:ACME_SERVER:pebble}
 passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
-[testenv:test-farm-tests-base]
+[testenv:test-farm-apache2]
-changedir = letstest
-# The package to install is in the current working directory because of the
-# value of changedir.
-commands = {[base]pip_install} .
 passenv =
     AWS_*
 setenv = AWS_DEFAULT_REGION=us-east-1
-
+changedir = letstest
-[testenv:test-farm-apache2]
+deps = -e {toxinidir}/letstest
-changedir = {[testenv:test-farm-tests-base]changedir}
+commands = {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
-commands =
-    {[testenv:test-farm-tests-base]commands}
-    {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
-passenv = {[testenv:test-farm-tests-base]passenv}
-setenv = {[testenv:test-farm-tests-base]setenv}