From 4740e20725af5a8639d8bce08b1d25241922deca Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Sat, 15 Apr 2023 17:30:59 -0700 Subject: [PATCH] Rewrite tox config (#9666) * rewrite tox config * fix apacheconftest-with-pebble deps * more fixes * more fixes * move comment up * fix mock location * bump cffi * update oldest constraints * Revert "fix mock location" This reverts commit 561037bfadd1f19675d03cefaf1214a52c433816. * fix apache test * fix server cleanup * fix some leaky sockets * stop leaking sockets * change less * Update tox.ini Co-authored-by: alexzorin * Update tox.ini Co-authored-by: alexzorin * tweak contributing doc --------- Co-authored-by: alexzorin --- .../templates/jobs/extended-tests-jobs.yml | 2 + .../templates/jobs/standard-tests-jobs.yml | 12 +- acme/acme/_internal/tests/crypto_util_test.py | 9 +- acme/acme/_internal/tests/standalone_test.py | 34 +- acme/acme/crypto_util.py | 44 +-- .../_internal/tests/autohsts_test.py | 4 +- certbot/docs/contributing.rst | 4 +- tools/install_and_test.py | 32 -- tools/oldest_constraints.txt | 68 ++-- tools/pinning/oldest/pyproject.toml | 2 +- tools/pip_install_editable.py | 17 - tox.ini | 294 ++++++++++-------- 12 files changed, 253 insertions(+), 269 deletions(-) delete mode 100755 tools/install_and_test.py delete mode 100755 tools/pip_install_editable.py diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index d7c1a8e55..c233d3c43 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -17,6 +17,8 @@ jobs: linux-py310: PYTHON_VERSION: 3.10 TOXENV: py310 + linux-isolated: + TOXENV: 'isolated-{acme,certbot,apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}' linux-boulder-v2-integration-certbot-oldest: PYTHON_VERSION: 3.7 TOXENV: integration-certbot-oldest diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 1e6d7a352..29afa2a98 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -14,7 +14,7 @@ jobs: windows-py37: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.7 - TOXENV: py37-win + TOXENV: py-win windows-py39-cover: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.9 @@ -23,14 +23,10 @@ jobs: IMAGE_NAME: windows-2019 PYTHON_VERSION: 3.9 TOXENV: integration-certbot - linux-oldest-tests-1: + linux-oldest: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.7 - TOXENV: '{acme,apache,apache-v2,certbot}-oldest' - linux-oldest-tests-2: - IMAGE_NAME: ubuntu-22.04 - PYTHON_VERSION: 3.7 - TOXENV: '{dns,nginx}-oldest' + TOXENV: oldest linux-py37: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.7 @@ -43,7 +39,7 @@ jobs: TOXENV: lint-posix linux-mypy: IMAGE_NAME: ubuntu-22.04 - TOXENV: mypy-posix + TOXENV: mypy linux-integration: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.8 diff --git a/acme/acme/_internal/tests/crypto_util_test.py b/acme/acme/_internal/tests/crypto_util_test.py index ae1821794..84142a012 100644 --- a/acme/acme/_internal/tests/crypto_util_test.py +++ b/acme/acme/_internal/tests/crypto_util_test.py @@ -29,11 +29,9 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): from acme.crypto_util import SSLSocket class _TestServer(socketserver.TCPServer): - - def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), - certs) - socketserver.TCPServer.server_bind(self) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.socket = SSLSocket(self.socket, certs) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) self.port = self.server.socket.getsockname()[1] @@ -44,6 +42,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): if self.server_thread.is_alive(): # The thread may have already terminated. self.server_thread.join() # pragma: no cover + self.server.server_close() def _probe(self, name): from acme.crypto_util import probe_sni diff --git a/acme/acme/_internal/tests/standalone_test.py b/acme/acme/_internal/tests/standalone_test.py index 130d8dc7d..ba58e6989 100644 --- a/acme/acme/_internal/tests/standalone_test.py +++ b/acme/acme/_internal/tests/standalone_test.py @@ -55,6 +55,7 @@ class HTTP01ServerTest(unittest.TestCase): def tearDown(self): self.server.shutdown() self.thread.join() + self.server.server_close() def test_index(self): response = requests.get( @@ -88,25 +89,25 @@ class HTTP01ServerTest(unittest.TestCase): def test_timely_shutdown(self): from acme.standalone import HTTP01Server - server = HTTP01Server(('', 0), resources=set(), timeout=0.05) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() + with HTTP01Server(('', 0), resources=set(), timeout=0.05) as server: + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() - client = socket.socket() - client.connect(('localhost', server.socket.getsockname()[1])) + with socket.socket() as client: + client.connect(('localhost', server.socket.getsockname()[1])) - stop_thread = threading.Thread(target=server.shutdown) - stop_thread.start() - server_thread.join(5.) + stop_thread = threading.Thread(target=server.shutdown) + stop_thread.start() + server_thread.join(5.) - is_hung = server_thread.is_alive() - try: - client.shutdown(socket.SHUT_RDWR) - except: # pragma: no cover, pylint: disable=bare-except - # may raise error because socket could already be closed - pass + is_hung = server_thread.is_alive() + try: + client.shutdown(socket.SHUT_RDWR) + except: # pragma: no cover, pylint: disable=bare-except + # may raise error because socket could already be closed + pass - assert not is_hung, 'Server shutdown should not be hung' + assert not is_hung, 'Server shutdown should not be hung' @unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") @@ -133,6 +134,7 @@ class TLSALPN01ServerTest(unittest.TestCase): def tearDown(self): self.server.shutdown() # pylint: disable=no-member self.thread.join() + self.server.server_close() # TODO: This is not implemented yet, see comments in standalone.py # def test_certs(self): @@ -214,6 +216,8 @@ class BaseDualNetworkedServersTest(unittest.TestCase): if prev_port: assert prev_port == port prev_port = port + for server in servers.servers: + server.server_close() class HTTP01DualNetworkedServersTest(unittest.TestCase): diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 754f70e46..f35b634b5 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -136,27 +136,33 @@ class SSLSocket: # pylint: disable=too-few-public-methods def accept(self) -> Tuple[FakeConnection, Any]: # pylint: disable=missing-function-docstring sock, addr = self.sock.accept() - context = SSL.Context(self.method) - context.set_options(SSL.OP_NO_SSLv2) - context.set_options(SSL.OP_NO_SSLv3) - context.set_tlsext_servername_callback(self._pick_certificate_cb) - if self.alpn_selection is not None: - context.set_alpn_select_callback(self.alpn_selection) - - ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) - ssl_sock.set_accept_state() - - # This log line is especially desirable because without it requests to - # our standalone TLSALPN server would not be logged. - logger.debug("Performing handshake with %s", addr) try: - ssl_sock.do_handshake() - except SSL.Error as error: - # _pick_certificate_cb might have returned without - # creating SSL context (wrong server name) - raise socket.error(error) + context = SSL.Context(self.method) + context.set_options(SSL.OP_NO_SSLv2) + context.set_options(SSL.OP_NO_SSLv3) + context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) - return ssl_sock, addr + ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) + ssl_sock.set_accept_state() + + # This log line is especially desirable because without it requests to + # our standalone TLSALPN server would not be logged. + logger.debug("Performing handshake with %s", addr) + try: + ssl_sock.do_handshake() + except SSL.Error as error: + # _pick_certificate_cb might have returned without + # creating SSL context (wrong server name) + raise socket.error(error) + + return ssl_sock, addr + except: + # If we encounter any error, close the new socket before reraising + # the exception. + sock.close() + raise def probe_sni(name: bytes, host: bytes, port: int = 443, timeout: int = 300, # pylint: disable=too-many-arguments diff --git a/certbot-apache/certbot_apache/_internal/tests/autohsts_test.py b/certbot-apache/certbot_apache/_internal/tests/autohsts_test.py index 4d7d2ec68..f6d5b2957 100644 --- a/certbot-apache/certbot_apache/_internal/tests/autohsts_test.py +++ b/certbot-apache/certbot_apache/_internal/tests/autohsts_test.py @@ -128,11 +128,11 @@ class AutoHSTSTest(util.ApacheTest): max_val def test_autohsts_update_noop(self): - with mock.patch("time.time") as mock_time: + with mock.patch("certbot_apache._internal.configurator.time") as mock_time_module: # Time mock is used to make sure that the execution does not # continue when no autohsts entries exist in pluginstorage self.config.update_autohsts(mock.MagicMock()) - assert mock_time.called is False + assert not mock_time_module.time.called def test_autohsts_make_permanent_noop(self): self.config.storage.put = mock.MagicMock() diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 532f6df6c..552a338ec 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -515,8 +515,8 @@ Steps: virtualenv. You can do this by following the instructions in the :ref:`Getting Started ` section. 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors. -4. Run ``tox --skip-missing-interpreters`` to run the entire test suite - including coverage. The ``--skip-missing-interpreters`` argument ignores +4. Run ``tox --skip-missing-interpreters`` to run all the tests we recommend + developers run locally. The ``--skip-missing-interpreters`` argument ignores missing versions of Python needed for running the tests. Fix any errors. 5. If any documentation should be added or updated as part of the changes you have made, please include the documentation changes in your PR. diff --git a/tools/install_and_test.py b/tools/install_and_test.py deleted file mode 100755 index 439e5dce3..000000000 --- a/tools/install_and_test.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. Packages are installed using -# pinned versions of all of our dependencies. See pip_install.py for more -# information on the versions pinned to. -import os -import re -import subprocess -import sys - - -def call_with_print(command): - print(command) - subprocess.check_call(command, shell=True) - - -def main(args): - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] - - for requirement in args: - current_command = command[:] - current_command.append(requirement) - call_with_print(' '.join(current_command)) - pkg = re.sub(r'\[\w+\]', '', requirement) - - call_with_print(' '.join([ - sys.executable, '-m', 'pytest', pkg])) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index da361cb4d..bf30a8c56 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -2,69 +2,69 @@ # that script. apacheconfig==0.3.2 ; python_full_version < "3.8.0" and python_version >= "3.7" asn1crypto==0.24.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" -astroid==2.12.13 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" -attrs==22.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +astroid==2.15.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +attrs==22.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0" boto3==1.15.15 ; python_full_version < "3.8.0" and python_version >= "3.7" botocore==1.18.15 ; python_full_version < "3.8.0" and python_version >= "3.7" certifi==2022.12.7 ; python_full_version < "3.8.0" and python_version >= "3.7" -cffi==1.9.1 ; python_full_version < "3.8.0" and python_version >= "3.7" +cffi==1.11.5 ; python_full_version < "3.8.0" and python_version >= "3.7" chardet==3.0.4 ; python_full_version < "3.8.0" and python_version >= "3.7" cloudflare==1.5.1 ; python_full_version < "3.8.0" and python_version >= "3.7" colorama==0.4.6 ; python_full_version < "3.8.0" and sys_platform == "win32" and python_version >= "3.7" configargparse==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7" configobj==5.0.6 ; python_full_version < "3.8.0" and python_version >= "3.7" -coverage==6.5.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +coverage==7.2.3 ; python_version >= "3.7" and python_full_version < "3.8.0" cryptography==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7" -cython==0.29.32 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" +cython==0.29.34 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" dill==0.3.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" distlib==0.3.6 ; python_version >= "3.7" and python_full_version < "3.8.0" distro==1.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7" dns-lexicon==3.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7" dnspython==1.15.0 ; python_full_version < "3.8.0" and python_version >= "3.7" -exceptiongroup==1.0.4 ; python_version >= "3.7" and python_full_version < "3.8.0" +exceptiongroup==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0" execnet==1.9.0 ; python_version >= "3.7" and python_full_version < "3.8.0" -filelock==3.8.2 ; python_version >= "3.7" and python_full_version < "3.8.0" +filelock==3.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0" funcsigs==0.4 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" -future==0.18.2 ; python_full_version < "3.8.0" and python_version >= "3.7" +future==0.18.3 ; python_full_version < "3.8.0" and python_version >= "3.7" google-api-python-client==1.5.5 ; python_full_version < "3.8.0" and python_version >= "3.7" httplib2==0.9.2 ; python_full_version < "3.8.0" and python_version >= "3.7" idna==2.6 ; python_full_version < "3.8.0" and python_version >= "3.7" -importlib-metadata==5.1.0 ; python_version >= "3.7" and python_version < "3.8" -iniconfig==1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0" +importlib-metadata==6.1.0 ; python_version >= "3.7" and python_version < "3.8" +iniconfig==2.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" ipaddress==1.0.16 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" -isort==5.10.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +isort==5.11.5 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" jmespath==0.10.0 ; python_full_version < "3.8.0" and python_version >= "3.7" josepy==1.13.0 ; python_version >= "3.7" and python_full_version < "3.8.0" -lazy-object-proxy==1.8.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +lazy-object-proxy==1.9.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" logger==1.4 ; python_full_version < "3.8.0" and python_version >= "3.7" mccabe==0.7.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" -mypy-extensions==0.4.3 ; python_version >= "3.7" and python_full_version < "3.8.0" -mypy==0.991 ; python_version >= "3.7" and python_full_version < "3.8.0" +mypy-extensions==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +mypy==1.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0" ndg-httpsclient==0.3.2 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" oauth2client==4.0.0 ; python_full_version < "3.8.0" and python_version >= "3.7" -packaging==22.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +packaging==23.0 ; python_version >= "3.7" and python_full_version < "3.8.0" parsedatetime==2.4 ; python_full_version < "3.8.0" and python_version >= "3.7" pbr==1.8.0 ; python_full_version >= "3.7.0" and python_full_version < "3.8.0" -pip==22.3.1 ; python_version >= "3.7" and python_full_version < "3.8.0" -platformdirs==2.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7" +pip==23.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0" +platformdirs==3.2.0 ; python_full_version < "3.8.0" and python_version >= "3.7" pluggy==1.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" ply==3.4 ; python_full_version < "3.8.0" and python_version >= "3.7" py==1.11.0 ; python_version >= "3.7" and python_full_version < "3.8.0" pyasn1-modules==0.0.10 ; python_full_version < "3.8.0" and python_version >= "3.7" pyasn1==0.1.9 ; python_full_version < "3.8.0" and python_version >= "3.7" pycparser==2.14 ; python_full_version < "3.8.0" and python_version >= "3.7" -pylint==2.15.8 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +pylint==2.17.2 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" pyopenssl==17.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7" pyparsing==2.2.1 ; python_full_version < "3.8.0" and python_version >= "3.7" pyrfc3339==1.0 ; python_full_version < "3.8.0" and python_version >= "3.7" pytest-cov==4.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" -pytest-xdist==3.1.0 ; python_version >= "3.7" and python_full_version < "3.8.0" -pytest==7.2.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +pytest-xdist==3.2.1 ; python_version >= "3.7" and python_full_version < "3.8.0" +pytest==7.2.2 ; python_version >= "3.7" and python_full_version < "3.8.0" python-augeas==0.5.0 ; python_full_version < "3.8.0" and python_version >= "3.7" python-dateutil==2.8.2 ; python_full_version < "3.8.0" and python_version >= "3.7" python-digitalocean==1.11 ; python_full_version < "3.8.0" and python_version >= "3.7" pytz==2019.3 ; python_full_version < "3.8.0" and python_version >= "3.7" -pywin32==305 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32" +pywin32==306 ; python_version >= "3.7" and python_full_version < "3.8.0" and sys_platform == "win32" pyyaml==6.0 ; python_full_version < "3.8.0" and python_version >= "3.7" requests-file==1.5.1 ; python_version >= "3.7" and python_full_version < "3.8.0" requests==2.20.0 ; python_full_version < "3.8.0" and python_version >= "3.7" @@ -74,22 +74,22 @@ setuptools==41.6.0 ; python_full_version < "3.8.0" and python_version >= "3.7" six==1.11.0 ; python_full_version < "3.8.0" and python_version >= "3.7" tldextract==3.4.0 ; python_version >= "3.7" and python_full_version < "3.8.0" tomli==2.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7" -tomlkit==0.11.6 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +tomlkit==0.11.7 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" tox==1.9.2 ; python_version >= "3.7" and python_full_version < "3.8.0" typed-ast==1.5.4 ; python_version < "3.8" and python_version >= "3.7" types-cryptography==3.3.23.2 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-pyopenssl==22.1.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-pyrfc3339==1.1.1.1 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-python-dateutil==2.8.19.4 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-pytz==2022.6.0.1 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-requests==2.28.11.5 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-setuptools==65.6.0.2 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-six==1.16.21.4 ; python_version >= "3.7" and python_full_version < "3.8.0" -types-urllib3==1.26.25.4 ; python_version >= "3.7" and python_full_version < "3.8.0" -typing-extensions==4.4.0 ; python_version >= "3.7" and python_version < "3.8" +types-pyopenssl==23.0.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-pyrfc3339==1.1.1.4 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-python-dateutil==2.8.19.12 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-pytz==2023.3.0.0 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-requests==2.28.11.17 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-setuptools==67.6.0.7 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-six==1.16.21.8 ; python_version >= "3.7" and python_full_version < "3.8.0" +types-urllib3==1.26.25.10 ; python_version >= "3.7" and python_full_version < "3.8.0" +typing-extensions==4.5.0 ; python_version < "3.8" and python_version >= "3.7" uritemplate==3.0.1 ; python_full_version < "3.8.0" and python_version >= "3.7" urllib3==1.24.2 ; python_full_version < "3.8.0" and python_version >= "3.7" -virtualenv==20.17.1 ; python_version >= "3.7" and python_full_version < "3.8.0" +virtualenv==20.21.0 ; python_version >= "3.7" and python_full_version < "3.8.0" wheel==0.33.6 ; python_full_version < "3.8.0" and python_version >= "3.7" -wrapt==1.14.1 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" -zipp==3.11.0 ; python_version >= "3.7" and python_version < "3.8" +wrapt==1.15.0 ; python_full_version >= "3.7.2" and python_full_version < "3.8.0" +zipp==3.15.0 ; python_version >= "3.7" and python_version < "3.8" diff --git a/tools/pinning/oldest/pyproject.toml b/tools/pinning/oldest/pyproject.toml index 6ab7c628c..ce9453d69 100644 --- a/tools/pinning/oldest/pyproject.toml +++ b/tools/pinning/oldest/pyproject.toml @@ -48,7 +48,7 @@ apacheconfig = "0.3.2" asn1crypto = "0.24.0" boto3 = "1.15.15" botocore = "1.18.15" -cffi = "1.9.1" +cffi = "1.11.5" chardet = "3.0.4" cloudflare = "1.5.1" configobj = "5.0.6" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py deleted file mode 100755 index 87be1b8f5..000000000 --- a/tools/pip_install_editable.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -# pip installs packages in editable mode using pip_install.py -import sys - -import pip_install - - -def main(args): - new_args = [] - for arg in args: - new_args.append('-e') - new_args.append(arg) - - pip_install.main(new_args) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/tox.ini b/tox.ini index de98c3e25..33308b70d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,42 +1,66 @@ -# Tox (https://tox.readthedocs.io/) is a tool for running tests in -# multiple virtualenvs. To use it, "pip install tox" and then run -# "tox" from this directory. - [tox] -skipsdist = true # mypy doesn't current pass for us on Windows. Fixing that is being tracked by # https://github.com/certbot/certbot/issues/7803. -envlist = {cover,lint}-{win,posix},mypy-posix +envlist = {cover,lint}-{win,posix},mypy +skipsdist = true [base] -# pip installs the requested packages in editable mode -pip_install = python {toxinidir}/tools/pip_install_editable.py -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. All dependencies are pinned -# to a specific version for increased stability for developers. -install_and_test = python {toxinidir}/tools/install_and_test.py -# Packages are listed on one line because tox seems to have inconsistent +pytest = python -m pytest {posargs} +# Paths are listed on one line because tox seems to have inconsistent # behavior with substitutions that contain line continuations, see # https://github.com/tox-dev/tox/issues/2069 for more info. -dns_packages = certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud -win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx -all_packages = {[base]win_all_packages} certbot-apache source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx [testenv] platform = win: win32 posix: ^(?!.*win32).*$ -commands = - win: {[base]install_and_test} {[base]win_all_packages} - !win: {[base]install_and_test} {[base]all_packages} -# We always recreate the virtual environment to avoid problems like -# https://github.com/certbot/certbot/issues/7745. -recreate = true setenv = PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto} PYTHONHASHSEED = 0 +# The default install command is python -I -m pip install {opts} {packages} +install_command = python -I {toxinidir}/tools/pip_install.py {opts} {packages} +deps = + -e acme[test] + -e certbot[test] + !win: -e certbot-apache[dev] + -e certbot-dns-cloudflare + -e certbot-dns-digitalocean + -e certbot-dns-dnsimple + -e certbot-dns-dnsmadeeasy + -e certbot-dns-gehirn + -e certbot-dns-google + -e certbot-dns-linode + -e certbot-dns-luadns + -e certbot-dns-nsone + -e certbot-dns-ovh + -e certbot-dns-rfc2136 + -e certbot-dns-route53 + -e certbot-dns-sakuracloud + -e certbot-nginx +whitelist_externals = + echo + false +# This and the next few testenvs are a workaround for +# https://github.com/tox-dev/tox/issues/2858. +commands = + echo "Unrecognized environment name {envname}" + false + +[testenv:py-win] +commands = + {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx + +[testenv:py{,-posix}] +# We want to test everything we do on Windows plus the Apache plugin. +commands = + {[testenv:py-win]commands} certbot-apache + +[testenv:py3{,7,8,9,10,11}] +commands = {[testenv:py]commands} + +[testenv:py3.{7,8,9,10,11}] +commands = {[testenv:py]commands} [testenv:oldest] # Setting basepython allows the tests to fail fast if that version of Python @@ -46,117 +70,108 @@ setenv = # This version should be kept in sync with the one declared in # tools/pinning/oldest/pyproject.toml. basepython = python3.7 -commands = - {[testenv]commands} -setenv = - {[testenv]setenv} - CERTBOT_OLDEST=1 - -[testenv:acme-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]install_and_test} acme[test] -setenv = - {[testenv:oldest]setenv} - -[testenv:apache-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-apache - pytest certbot-apache -setenv = - {[testenv:oldest]setenv} - -[testenv:apache-v2-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-apache[dev] - pytest certbot-apache -setenv = - {[testenv:oldest]setenv} - -[testenv:certbot-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] - pytest certbot -setenv = - {[testenv:oldest]setenv} - -[testenv:dns-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] {[base]dns_packages} - pytest {[base]dns_packages} -setenv = - {[testenv:oldest]setenv} - -[testenv:nginx-oldest] -basepython = - {[testenv:oldest]basepython} -commands = - {[base]pip_install} acme[test] certbot[test] certbot-nginx - pytest certbot-nginx -setenv = - {[testenv:oldest]setenv} +setenv = CERTBOT_OLDEST=1 +commands = {[testenv:py]commands} [testenv:cover{,-win,-posix}] -commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} certbot-apache[dev] - python tox.cover.py +commands = python tox.cover.py [testenv:lint{,-win,-posix}] -basepython = python3 -# separating into multiple invocations disables cross package -# duplicate code checking; if one of the commands fails, others will -# continue, but tox return code will reflect previous error -commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} - python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} +commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} -[testenv:mypy{,-win,-posix}] -basepython = python3 +[testenv:mypy] +deps = + {[testenv]deps} + -e certbot-ci +commands = mypy {[base]source_paths} + +[testenv:isolated-acme] +description = Tests acme without any Certbot components installed +deps = -e acme[test] +commands = {[base]pytest} acme + +[testenv:isolated-certbot] +description = Tests Certbot without any additional plugins installed +deps = + {[testenv:isolated-acme]deps} + -e certbot[test] +commands = {[base]pytest} certbot + +[testenv:isolated-{apache,cloudflare,digitalocean,dnsimple,dnsmadeeasy,gehirn,google,linode,luadns,nsone,ovh,rfc2136,route53,sakuracloud,nginx}] +description = Tests the plugin without installing any other plugins +deps = + {[testenv:isolated-certbot]deps} + apache: -e certbot-apache[dev] + cloudflare: -e certbot-dns-cloudflare + digitalocean: -e certbot-dns-digitalocean + dnsimple: -e certbot-dns-dnsimple + dnsmadeeasy: -e certbot-dns-dnsmadeeasy + gehirn: -e certbot-dns-gehirn + google: -e certbot-dns-google + linode: -e certbot-dns-linode + luadns: -e certbot-dns-luadns + nsone: -e certbot-dns-nsone + ovh: -e certbot-dns-ovh + rfc2136: -e certbot-dns-rfc2136 + route53: -e certbot-dns-route53 + sakuracloud: -e certbot-dns-sakuracloud + nginx: -e certbot-nginx commands = - win: {[base]pip_install} {[base]win_all_packages} - !win: {[base]pip_install} {[base]all_packages} certbot-ci - mypy {[base]source_paths} + apache: {[base]pytest} certbot-apache + cloudflare: {[base]pytest} certbot-dns-cloudflare + digitalocean: {[base]pytest} certbot-dns-digitalocean + dnsimple: {[base]pytest} certbot-dns-dnsimple + dnsmadeeasy: {[base]pytest} certbot-dns-dnsmadeeasy + gehirn: {[base]pytest} certbot-dns-gehirn + google: {[base]pytest} certbot-dns-google + linode: {[base]pytest} certbot-dns-linode + luadns: {[base]pytest} certbot-dns-luadns + nsone: {[base]pytest} certbot-dns-nsone + ovh: {[base]pytest} certbot-dns-ovh + rfc2136: {[base]pytest} certbot-dns-rfc2136 + route53: {[base]pytest} certbot-dns-route53 + sakuracloud: {[base]pytest} certbot-dns-sakuracloud + nginx: {[base]pytest} certbot-nginx [testenv:apacheconftest] +deps = + -e acme + -e certbot + -e certbot-apache commands = - {[base]pip_install} acme certbot certbot-apache {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test --debian-modules passenv = SERVER [testenv:apacheconftest-external-with-pebble] -# Run apacheconftest with pebble and Certbot outside of tox's virtual -# environment. +description = Run apacheconftest with pebble and Certbot outside of the tox virtual environment. +deps = + -e certbot-ci commands = - {[base]pip_install} certbot-ci {toxinidir}/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules [testenv:apacheconftest-with-pebble] -commands = - {[base]pip_install} acme certbot certbot-apache - {[testenv:apacheconftest-external-with-pebble]commands} +deps = + {[testenv:apacheconftest]deps} + {[testenv:apacheconftest-external-with-pebble]deps} +commands = {[testenv:apacheconftest-external-with-pebble]commands} [testenv:nginxroundtrip] +deps = + -e acme + -e certbot + -e certbot-apache + -e certbot-nginx commands = - {[base]pip_install} acme certbot certbot-apache certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata [testenv:modification] +deps = commands = python {toxinidir}/tests/modification-check.py [testenv:apache_compat] +deps = commands = docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile . docker build -t apache-compat -f certbot-compatibility-test/Dockerfile-apache . @@ -167,6 +182,7 @@ passenv = DOCKER_* [testenv:nginx_compat] +deps = commands = docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile . docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx . @@ -177,8 +193,8 @@ passenv = DOCKER_* [testenv:docker_dev] -# tests the Dockerfile-dev file to ensure development with it works -# as expected +description = Tests the Dockerfile-dev file to ensure development with it works as expected +deps = commands = docker-compose run --rm --service-ports development bash -c 'tox -e lint' whitelist_externals = @@ -186,9 +202,13 @@ whitelist_externals = passenv = DOCKER_* [testenv:integration] +deps = + -e acme + -e certbot + -e certbot-nginx + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-nginx certbot-ci - pytest certbot-ci/certbot_integration_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests \ --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ --cov-config=certbot-ci/certbot_integration_tests/.coveragerc @@ -197,18 +217,25 @@ commands = passenv = DOCKER_* [testenv:integration-certbot] +deps = + -e acme + -e certbot + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-ci - pytest certbot-ci/certbot_integration_tests/certbot_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \ --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov-report= \ --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=62 [testenv:integration-dns-rfc2136] +deps = + -e acme + -e certbot + -e certbot-dns-rfc2136 + -e certbot-ci commands = - {[base]pip_install} acme certbot certbot-dns-rfc2136 certbot-ci - pytest certbot-ci/certbot_integration_tests/rfc2136_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests/rfc2136_tests \ --acme-server=pebble --dns-server=bind \ --numprocesses=1 \ --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \ @@ -217,46 +244,45 @@ commands = coverage report --include 'certbot-dns-rfc2136/*' --show-missing --fail-under=86 [testenv:integration-external] -# Run integration tests with Certbot outside of tox's virtual environment. +description = Run integration tests with Certbot outside of the tox virtual environment. +deps = + -e certbot-ci commands = - {[base]pip_install} certbot-ci - pytest certbot-ci/certbot_integration_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* [testenv:integration-certbot-oldest] +deps = + -e acme + -e certbot + -e certbot-ci basepython = {[testenv:oldest]basepython} commands = - {[base]pip_install} acme certbot certbot-ci - pytest certbot-ci/certbot_integration_tests/certbot_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* setenv = {[testenv:oldest]setenv} [testenv:integration-nginx-oldest] +deps = + -e acme + -e certbot + -e certbot-nginx + -e certbot-ci basepython = {[testenv:oldest]basepython} commands = - {[base]pip_install} acme certbot certbot-nginx certbot-ci - pytest certbot-ci/certbot_integration_tests/nginx_tests \ + {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests \ --acme-server={env:ACME_SERVER:pebble} passenv = DOCKER_* setenv = {[testenv:oldest]setenv} -[testenv:test-farm-tests-base] -changedir = letstest -# The package to install is in the current working directory because of the -# value of changedir. -commands = {[base]pip_install} . +[testenv:test-farm-apache2] passenv = AWS_* setenv = AWS_DEFAULT_REGION=us-east-1 - -[testenv:test-farm-apache2] -changedir = {[testenv:test-farm-tests-base]changedir} -commands = - {[testenv:test-farm-tests-base]commands} - {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir} -passenv = {[testenv:test-farm-tests-base]passenv} -setenv = {[testenv:test-farm-tests-base]setenv} +changedir = letstest +deps = -e {toxinidir}/letstest +commands = {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}