From 873f979a25cc29bba93627eb08697baaa894d640 Mon Sep 17 00:00:00 2001 From: Will Greenberg Date: Thu, 2 May 2024 12:24:00 -0700 Subject: [PATCH] Replace boulder tests with pebble (#9918) Pebble 2.5.1 supports OCSP stapling, so we can finally replace all boulder tests/harnesses with the much simpler pebble setup. Closes #9898 * Remove unused `--acme-server` argument Since this argument is never set and always defaults to 'pebble', just remove it to simplify assumptions about which test server's being used. * Remove boulder option from integration tests Now that pebble supports all of our test cases, we can move off of the much more complicated boulder test harness. * pebble_artifacts: bump to latest pebble release * pebble_artifacts: fix download path * certbot-ci: unzip pebble assets * CI: rip out windows tests/jobs * tox.ini: rm outdated Windows comment Co-authored-by: Brad Warren * ci: rm redundant integration test Co-authored-by: Brad Warren * acme_server: raise error if proxy and http-01 port are both set * acme_server: rm vestigial preterimate commands stuff --------- Co-authored-by: Brad Warren --- .../templates/jobs/extended-tests-jobs.yml | 24 +-- .../templates/jobs/packaging-jobs.yml | 59 -------- .../templates/jobs/standard-tests-jobs.yml | 13 -- .../apache-conf-test-pebble.py | 2 +- .../assets/boulder-rate-limit-policies.yml | 55 ------- .../certbot_tests/context.py | 1 - .../certbot_tests/test_main.py | 56 +++---- .../certbot_integration_tests/conftest.py | 25 +--- .../utils/acme_server.py | 137 ++---------------- .../utils/constants.py | 3 - .../certbot_integration_tests/utils/misc.py | 7 +- .../utils/pebble_artifacts.py | 34 +++-- tox.ini | 39 ++--- 13 files changed, 79 insertions(+), 376 deletions(-) delete mode 100644 certbot-ci/certbot_integration_tests/assets/boulder-rate-limit-policies.yml diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml index 09d0e22c7..0a72601dd 100644 --- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml @@ -19,34 +19,26 @@ jobs: TOXENV: py311 linux-isolated: TOXENV: 'isolated-acme,isolated-certbot,isolated-apache,isolated-cloudflare,isolated-digitalocean,isolated-dnsimple,isolated-dnsmadeeasy,isolated-gehirn,isolated-google,isolated-linode,isolated-luadns,isolated-nsone,isolated-ovh,isolated-rfc2136,isolated-route53,isolated-sakuracloud,isolated-nginx' - linux-boulder-v2-integration-certbot-oldest: + linux-integration-certbot-oldest: PYTHON_VERSION: 3.8 TOXENV: integration-certbot-oldest - ACME_SERVER: boulder-v2 - linux-boulder-v2-integration-nginx-oldest: + linux-integration-nginx-oldest: PYTHON_VERSION: 3.8 TOXENV: integration-nginx-oldest - ACME_SERVER: boulder-v2 - linux-boulder-v2-py38-integration: - PYTHON_VERSION: 3.8 - TOXENV: integration - ACME_SERVER: boulder-v2 - linux-boulder-v2-py39-integration: + # python 3.8 integration tests are not run here because they're run as + # part of the standard test suite + linux-py39-integration: PYTHON_VERSION: 3.9 TOXENV: integration - ACME_SERVER: boulder-v2 - linux-boulder-v2-py310-integration: + linux-py310-integration: PYTHON_VERSION: 3.10 TOXENV: integration - ACME_SERVER: boulder-v2 - linux-boulder-v2-py311-integration: + linux-py311-integration: PYTHON_VERSION: 3.11 TOXENV: integration - ACME_SERVER: boulder-v2 - linux-boulder-v2-py312-integration: + linux-py312-integration: PYTHON_VERSION: 3.12 TOXENV: integration - ACME_SERVER: boulder-v2 nginx-compat: TOXENV: nginx_compat linux-integration-rfc2136: diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index d26f1d4d6..cdf8393c2 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -55,65 +55,6 @@ jobs: - bash: | set -e && tools/docker/test.sh $(dockerTag) $DOCKER_ARCH displayName: Run integration tests for Docker images - - job: installer_build - pool: - vmImage: windows-2019 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.9 - architecture: x64 - addToPath: true - - script: | - python -m venv venv - venv\Scripts\python tools\pip_install.py -e windows-installer - displayName: Prepare Windows installer build environment - - script: | - venv\Scripts\construct-windows-installer - displayName: Build Certbot installer - - task: CopyFiles@2 - inputs: - sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis - contents: '*.exe' - targetFolder: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@1 - inputs: - path: $(Build.ArtifactStagingDirectory) - # If we change the artifact's name, it should also be changed in tools/create_github_release.py - artifact: windows-installer - displayName: Publish Windows installer - - job: installer_run - dependsOn: installer_build - strategy: - matrix: - win2019: - imageName: windows-2019 - pool: - vmImage: $(imageName) - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.9 - addToPath: true - - task: DownloadPipelineArtifact@2 - inputs: - artifact: windows-installer - path: $(Build.SourcesDirectory)/bin - displayName: Retrieve Windows installer - - script: | - python -m venv venv - venv\Scripts\python tools\pip_install.py -e certbot-ci - env: - PIP_NO_BUILD_ISOLATION: no - displayName: Prepare Certbot-CI - - script: | - set PATH=%ProgramFiles%\Certbot\bin;%PATH% - venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win_amd64.exe - displayName: Run windows installer integration tests - - script: | - set PATH=%ProgramFiles%\Certbot\bin;%PATH% - venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4 - displayName: Run certbot integration tests - job: snaps_build pool: vmImage: ubuntu-22.04 diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index 30833e891..65e6bac94 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -16,18 +16,6 @@ jobs: TOXENV: cover # See explanation under macos-py38-cover. PIP_USE_PEP517: "true" - windows-py38: - IMAGE_NAME: windows-2019 - PYTHON_VERSION: 3.8 - TOXENV: py-win - windows-py39-cover: - IMAGE_NAME: windows-2019 - PYTHON_VERSION: 3.9 - TOXENV: cover-win - windows-integration-certbot: - IMAGE_NAME: windows-2019 - PYTHON_VERSION: 3.9 - TOXENV: integration-certbot linux-oldest: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.8 @@ -49,7 +37,6 @@ jobs: IMAGE_NAME: ubuntu-22.04 PYTHON_VERSION: 3.8 TOXENV: integration - ACME_SERVER: pebble apache-compat: IMAGE_NAME: ubuntu-22.04 TOXENV: apache_compat diff --git a/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py b/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py index 383c652f8..837c67b9e 100755 --- a/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py +++ b/certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py @@ -14,7 +14,7 @@ SCRIPT_DIRNAME = os.path.dirname(__file__) def main() -> int: args = sys.argv[1:] - with acme_server.ACMEServer('pebble', [], False) as acme_xdist: + with acme_server.ACMEServer([], False) as acme_xdist: environ = os.environ.copy() environ['SERVER'] = acme_xdist['directory_url'] command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')] diff --git a/certbot-ci/certbot_integration_tests/assets/boulder-rate-limit-policies.yml b/certbot-ci/certbot_integration_tests/assets/boulder-rate-limit-policies.yml deleted file mode 100644 index 289de7021..000000000 --- a/certbot-ci/certbot_integration_tests/assets/boulder-rate-limit-policies.yml +++ /dev/null @@ -1,55 +0,0 @@ -# See https://github.com/letsencrypt/boulder/blob/main/cmd/shell.go for -# definitions of these rate limits. -certificatesPerName: - window: 2160h - threshold: 99 - overrides: - ratelimit.me: 1 - lim.it: 0 - # Hostnames used by the letsencrypt client integration test. - le.wtf: 9999 - le1.wtf: 9999 - le2.wtf: 9999 - le3.wtf: 9999 - le4.wtf: 9999 - nginx.wtf: 9999 - good-caa-reserved.com: 9999 - bad-caa-reserved.com: 9999 - ecdsa.le.wtf: 9999 - must-staple.le.wtf: 9999 - registrationOverrides: - 101: 1000 -registrationsPerIP: - window: 168h # 1 week - threshold: 9999 - overrides: - 127.0.0.1: 999990 -registrationsPerIPRange: - window: 168h # 1 week - threshold: 99999 - overrides: - 127.0.0.1: 1000000 -pendingAuthorizationsPerAccount: - window: 168h # 1 week, should match pending authorization lifetime. - threshold: 999 -newOrdersPerAccount: - window: 3h - threshold: 9999 -certificatesPerFQDNSet: - window: 168h - threshold: 99999 - overrides: - le.wtf: 9999 - le1.wtf: 9999 - le2.wtf: 9999 - le3.wtf: 9999 - le.wtf,le1.wtf: 9999 - good-caa-reserved.com: 9999 - nginx.wtf: 9999 - ecdsa.le.wtf: 9999 - must-staple.le.wtf: 9999 -certificatesPerFQDNSetFast: - window: 2h - threshold: 20 - overrides: - le.wtf: 9 diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index 6b7407e50..bc893d8d9 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -23,7 +23,6 @@ class IntegrationTestsContext: self.worker_id = 'primary' acme_xdist = request.config.acme_xdist # type: ignore[attr-defined] - self.acme_server = acme_xdist['acme_server'] self.directory_url = acme_xdist['directory_url'] self.tls_alpn_01_port = acme_xdist['https_port'][self.worker_id] self.http_01_port = acme_xdist['http_port'][self.worker_id] diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 8ed1fbf1e..54ff2d588 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -7,7 +7,6 @@ import shutil import subprocess import time from typing import Generator -from typing import Iterable from typing import Tuple from typing import Type @@ -82,11 +81,9 @@ def test_registration_override(context: IntegrationTestsContext) -> None: context.certbot(['update_account', '--email', 'ex1@domain.org,ex2@domain.org']) stdout2, _ = context.certbot(['show_account']) - # https://github.com/letsencrypt/boulder/issues/6144 - if context.acme_server != 'boulder-v2': - assert 'example@domain.org' in stdout1, "New email should be present" - assert 'example@domain.org' not in stdout2, "Old email should not be present" - assert 'ex1@domain.org, ex2@domain.org' in stdout2, "New emails should be present" + assert 'example@domain.org' in stdout1, "New email should be present" + assert 'example@domain.org' not in stdout2, "Old email should not be present" + assert 'ex1@domain.org, ex2@domain.org' in stdout2, "New emails should be present" def test_prepare_plugins(context: IntegrationTestsContext) -> None: @@ -566,19 +563,15 @@ def test_default_rsa_size(context: IntegrationTestsContext) -> None: assert_rsa_key(key1, 2048) -@pytest.mark.parametrize('curve,curve_cls,skip_servers', [ +@pytest.mark.parametrize('curve,curve_cls', [ # Curve name, Curve class, ACME servers to skip - ('secp256r1', SECP256R1, []), - ('secp384r1', SECP384R1, []), - ('secp521r1', SECP521R1, ['boulder-v2'])] + ('secp256r1', SECP256R1), + ('secp384r1', SECP384R1), + ('secp521r1', SECP521R1)] ) -def test_ecdsa_curves(context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve], - skip_servers: Iterable[str]) -> None: +def test_ecdsa_curves(context: IntegrationTestsContext, curve: str, + curve_cls: Type[EllipticCurve]) -> None: """Test issuance for each supported ECDSA curve""" - if context.acme_server in skip_servers: - pytest.skip('ACME server {} does not support ECDSA curve {}' - .format(context.acme_server, curve)) - domain = context.get_domain('curve') context.certbot([ 'certonly', @@ -640,9 +633,6 @@ def test_renew_with_ec_keys(context: IntegrationTestsContext) -> None: def test_ocsp_must_staple(context: IntegrationTestsContext) -> None: """Test that OCSP Must-Staple is correctly set in the generated certificate.""" - if context.acme_server == 'pebble': - pytest.skip('Pebble does not support OCSP Must-Staple.') - certname = context.get_domain('must-staple') context.certbot(['auth', '--must-staple', '--domains', certname]) @@ -710,17 +700,14 @@ def test_revoke_and_unregister(context: IntegrationTestsContext) -> None: assert cert3 in stdout -@pytest.mark.parametrize('curve,curve_cls,skip_servers', [ - ('secp256r1', SECP256R1, []), - ('secp384r1', SECP384R1, []), - ('secp521r1', SECP521R1, ['boulder-v2'])] +@pytest.mark.parametrize('curve,curve_cls', [ + ('secp256r1', SECP256R1), + ('secp384r1', SECP384R1), + ('secp521r1', SECP521R1)] ) def test_revoke_ecdsa_cert_key( - context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve], - skip_servers: Iterable[str]) -> None: + context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve]) -> None: """Test revoking a certificate """ - if context.acme_server in skip_servers: - pytest.skip(f'ACME server {context.acme_server} does not support ECDSA curve {curve}') cert: str = context.get_domain('curve') context.certbot([ 'certonly', @@ -738,17 +725,14 @@ def test_revoke_ecdsa_cert_key( assert stdout.count('INVALID: REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert) -@pytest.mark.parametrize('curve,curve_cls,skip_servers', [ - ('secp256r1', SECP256R1, []), - ('secp384r1', SECP384R1, []), - ('secp521r1', SECP521R1, ['boulder-v2'])] +@pytest.mark.parametrize('curve,curve_cls', [ + ('secp256r1', SECP256R1), + ('secp384r1', SECP384R1), + ('secp521r1', SECP521R1)] ) def test_revoke_ecdsa_cert_key_delete( - context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve], - skip_servers: Iterable[str]) -> None: + context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve]) -> None: """Test revoke and deletion for each supported curve type""" - if context.acme_server in skip_servers: - pytest.skip(f'ACME server {context.acme_server} does not support ECDSA curve {curve}') cert: str = context.get_domain('curve') context.certbot([ 'certonly', @@ -913,7 +897,7 @@ def test_dry_run_deactivate_authzs(context: IntegrationTestsContext) -> None: def test_preferred_chain(context: IntegrationTestsContext) -> None: """Test that --preferred-chain results in the correct chain.pem being produced""" try: - issuers = misc.get_acme_issuers(context) + issuers = misc.get_acme_issuers() except NotImplementedError: pytest.skip('This ACME server does not support alternative issuers.') diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py index 176d0fb51..ca226ac97 100644 --- a/certbot-ci/certbot_integration_tests/conftest.py +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -8,7 +8,6 @@ for a directory a specific configuration using built-in pytest hooks. See https://docs.pytest.org/en/latest/reference.html#hook-reference """ import contextlib -import subprocess import sys from certbot_integration_tests.utils import acme_server as acme_lib @@ -20,10 +19,6 @@ def pytest_addoption(parser): Standard pytest hook to add options to the pytest parser. :param parser: current pytest parser that will be used on the CLI """ - parser.addoption('--acme-server', default='pebble', - choices=['boulder-v2', 'pebble'], - help='select the ACME server to use (boulder-v2, pebble), ' - 'defaulting to pebble') parser.addoption('--dns-server', default='challtestsrv', choices=['bind', 'challtestsrv'], help='select the DNS server to use (bind, challtestsrv), ' @@ -80,23 +75,6 @@ def _setup_primary_node(config): :param config: Configuration of the pytest primary node. Is modified by this function. """ - # Check for runtime compatibility: some tools are required to be available in PATH - if 'boulder' in config.option.acme_server: - try: - subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError): - raise ValueError('Error: docker is required in PATH to launch the integration tests on' - 'boulder, but is not installed or not available for current user.') - - try: - subprocess.check_output(['docker', 'compose', 'ls'], stderr=subprocess.STDOUT) - except (subprocess.CalledProcessError, OSError): - raise ValueError( - 'Error: A version of Docker with the "compose" subcommand ' - 'is required in PATH to launch the integration tests, ' - 'but is not installed or not available for current user.' - ) - # Parameter numprocesses is added to option by pytest-xdist workers = ['primary'] if not config.option.numprocesses\ else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] @@ -116,8 +94,7 @@ def _setup_primary_node(config): # By calling setup_acme_server we ensure that all necessary acme server instances will be # fully started. This runtime is reflected by the acme_xdist returned. - acme_server = acme_lib.ACMEServer(config.option.acme_server, workers, - dns_server=acme_dns_server) + acme_server = acme_lib.ACMEServer(workers, dns_server=acme_dns_server) config.add_cleanup(acme_server.stop) print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist)) acme_server.start() diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 3802c6197..7413a471a 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -5,7 +5,6 @@ import argparse import errno import json import os -from os.path import join import shutil import subprocess import sys @@ -18,22 +17,15 @@ from typing import Dict from typing import List from typing import Mapping from typing import Optional -from typing import Tuple from typing import Type -import requests - # pylint: disable=wildcard-import,unused-wildcard-import from certbot_integration_tests.utils import misc from certbot_integration_tests.utils import pebble_artifacts +from certbot_integration_tests.utils import pebble_ocsp_server from certbot_integration_tests.utils import proxy from certbot_integration_tests.utils.constants import * -if sys.version_info >= (3, 9): # pragma: no cover - import importlib.resources as importlib_resources -else: # pragma: no cover - import importlib_resources - class ACMEServer: """ @@ -47,34 +39,30 @@ class ACMEServer: ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped upon context enter/exit. """ - def __init__(self, acme_server: str, nodes: List[str], http_proxy: bool = True, + def __init__(self, nodes: List[str], http_proxy: bool = True, stdout: bool = False, dns_server: Optional[str] = None, http_01_port: Optional[int] = None) -> None: """ Create an ACMEServer instance. - :param str acme_server: the type of acme server used (boulder-v2 or pebble) :param list nodes: list of node names that will be setup by pytest xdist :param bool http_proxy: if False do not start the HTTP proxy :param bool stdout: if True stream all subprocesses stdout to standard stdout - :param str dns_server: if set, Pebble/Boulder will use it to resolve domains + :param str dns_server: if set, Pebble will use it to resolve domains :param int http_01_port: port to use for http-01 validation; currently only supported for pebble without an HTTP proxy """ - self._construct_acme_xdist(acme_server, nodes) + self._construct_acme_xdist(nodes) - self._acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' self._proxy = http_proxy self._workspace = tempfile.mkdtemp() self._processes: List[subprocess.Popen] = [] self._stdout = sys.stdout if stdout else open(os.devnull, 'w') # pylint: disable=consider-using-with self._dns_server = dns_server - self._preterminate_cmds_args: List[Tuple[Tuple[Any, ...], Dict[str, Any]]] = [] - self._http_01_port = BOULDER_HTTP_01_PORT if self._acme_type == 'boulder' \ - else DEFAULT_HTTP_01_PORT + self._http_01_port = DEFAULT_HTTP_01_PORT if http_01_port: - if (self._acme_type == 'pebble' and self._proxy) or self._acme_type == 'boulder': + if self._proxy: raise ValueError('Setting http_01_port is not currently supported when ' - 'using Boulder or the HTTP proxy') + 'using the HTTP proxy') self._http_01_port = http_01_port def start(self) -> None: @@ -82,10 +70,7 @@ class ACMEServer: try: if self._proxy: self._prepare_http_proxy() - if self._acme_type == 'pebble': - self._prepare_pebble_server() - if self._acme_type == 'boulder': - self._prepare_boulder_server() + self._prepare_pebble_server() except BaseException as e: self.stop() raise e @@ -94,7 +79,6 @@ class ACMEServer: """Stop the test stack, and clean its resources""" print('=> Tear down the test infrastructure...') try: - self._run_preterminate_cmds() for process in self._processes: try: process.terminate() @@ -120,19 +104,14 @@ class ACMEServer: traceback: Optional[TracebackType]) -> None: self.stop() - def _construct_acme_xdist(self, acme_server: str, nodes: List[str]) -> None: + def _construct_acme_xdist(self, nodes: List[str]) -> None: """Generate and return the acme_xdist dict""" - acme_xdist: Dict[str, Any] = {'acme_server': acme_server} + acme_xdist: Dict[str, Any] = {} # Directory and ACME port are set implicitly in the docker-compose.yml - # files of Boulder/Pebble. - if acme_server == 'pebble': - acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL - acme_xdist['challtestsrv_url'] = PEBBLE_CHALLTESTSRV_URL - else: # boulder - acme_xdist['directory_url'] = BOULDER_V2_DIRECTORY_URL - acme_xdist['challtestsrv_url'] = BOULDER_V2_CHALLTESTSRV_URL - + # files of Pebble. + acme_xdist['directory_url'] = PEBBLE_DIRECTORY_URL + acme_xdist['challtestsrv_url'] = PEBBLE_CHALLTESTSRV_URL acme_xdist['http_port'] = dict(zip(nodes, range(5200, 5200 + len(nodes)))) acme_xdist['https_port'] = dict(zip(nodes, range(5100, 5100 + len(nodes)))) acme_xdist['other_port'] = dict(zip(nodes, range(5300, 5300 + len(nodes)))) @@ -166,11 +145,6 @@ class ACMEServer: [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'], env=environ) - # pebble_ocsp_server is imported here and not at the top of module in order to avoid a - # useless ImportError, in the case where cryptography dependency is too old to support - # ocsp, but Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is - # the typical situation of integration-certbot-oldest tox testenv. - from certbot_integration_tests.utils import pebble_ocsp_server self._launch_process([sys.executable, pebble_ocsp_server.__file__]) # Wait for the ACME CA server to be up. @@ -179,70 +153,6 @@ class ACMEServer: print('=> Finished pebble instance deployment.') - def _prepare_boulder_server(self) -> None: - """Configure and launch the Boulder server""" - print('=> Starting boulder instance deployment...') - instance_path = join(self._workspace, 'boulder') - - # Load Boulder from git, that includes a docker-compose.yml ready for production. - process = self._launch_process(['git', 'clone', 'https://github.com/letsencrypt/boulder', - '--single-branch', '--depth=1', instance_path]) - process.wait(MAX_SUBPROCESS_WAIT) - - # Allow Boulder to ignore usual limit rate policies, useful for tests. - ref = importlib_resources.files("certbot_integration_tests") - ref = ref / "assets" / "boulder-rate-limit-policies.yml" - with importlib_resources.as_file(ref) as path: - shutil.copyfile(path, join(instance_path, 'test/rate-limit-policies.yml')) - - if self._dns_server: - # Change Boulder config to use the provided DNS server - for suffix in ["", "-remote-a", "-remote-b"]: - with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'r') as f: - config = json.loads(f.read()) - config['va']['dnsResolvers'] = [self._dns_server] - with open(join(instance_path, 'test/config/va{}.json'.format(suffix)), 'w') as f: - f.write(json.dumps(config, indent=2, separators=(',', ': '))) - - # This command needs to be run before we try and terminate running processes because - # docker compose up doesn't always respond to SIGTERM. See - # https://github.com/certbot/certbot/pull/9435. - self._register_preterminate_cmd(['docker', 'compose', 'down'], cwd=instance_path) - # Boulder docker generates build artifacts owned by root with 0o744 permissions. - # If we started the acme server from a normal user that has access to the Docker - # daemon, this user will not be able to delete these artifacts from the host. - # We need to do it through a docker. - self._register_preterminate_cmd(['docker', 'run', '--rm', '-v', - '{0}:/workspace'.format(self._workspace), 'alpine', 'rm', - '-rf', '/workspace/boulder']) - try: - # Launch the Boulder server - self._launch_process(['docker', 'compose', 'up', '--force-recreate'], cwd=instance_path) - - # Wait for the ACME CA server to be up. - print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout( - self.acme_xdist['directory_url'], attempts=480) - - if not self._dns_server: - # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post( - f'{BOULDER_V2_CHALLTESTSRV_URL}/set-default-ipv4', - json={'ip': '10.77.77.1'}, - timeout=10 - ) - response.raise_for_status() - except BaseException: - # If we failed to set up boulder, print its logs. - print('=> Boulder setup failed. Boulder logs are:') - process = self._launch_process([ - 'docker', 'compose', 'logs'], cwd=instance_path, force_stderr=True - ) - process.wait(MAX_SUBPROCESS_WAIT) - raise - - print('=> Finished boulder instance deployment.') - def _prepare_http_proxy(self) -> None: """Configure and launch an HTTP proxy""" print(f'=> Configuring the HTTP proxy on port {self._http_01_port}...') @@ -267,26 +177,11 @@ class ACMEServer: self._processes.append(process) return process - def _register_preterminate_cmd(self, *args: Any, **kwargs: Any) -> None: - self._preterminate_cmds_args.append((args, kwargs)) - - def _run_preterminate_cmds(self) -> None: - for args, kwargs in self._preterminate_cmds_args: - process = self._launch_process(*args, **kwargs) - process.wait(MAX_SUBPROCESS_WAIT) - # It's unlikely to matter, but let's clear the list of cleanup commands - # once they've been run. - self._preterminate_cmds_args.clear() - def main() -> None: # pylint: disable=missing-function-docstring parser = argparse.ArgumentParser( - description='CLI tool to start a local instance of Pebble or Boulder CA server.') - parser.add_argument('--server-type', '-s', - choices=['pebble', 'boulder-v2'], default='pebble', - help='type of CA server to start: can be Pebble or Boulder. ' - 'Pebble is used if not set.') + description='CLI tool to start a local instance of Pebble CA server.') parser.add_argument('--dns-server', '-d', help='specify the DNS server as `IP:PORT` to use by ' 'Pebble; if not specified, a local mock DNS server will be used to ' @@ -297,8 +192,8 @@ def main() -> None: args = parser.parse_args() acme_server = ACMEServer( - args.server_type, [], http_proxy=False, stdout=True, - dns_server=args.dns_server, http_01_port=args.http_01_port, + [], http_proxy=False, stdout=True, dns_server=args.dns_server, + http_01_port=args.http_01_port, ) try: diff --git a/certbot-ci/certbot_integration_tests/utils/constants.py b/certbot-ci/certbot_integration_tests/utils/constants.py index 5aabe379a..2204ef7c1 100644 --- a/certbot-ci/certbot_integration_tests/utils/constants.py +++ b/certbot-ci/certbot_integration_tests/utils/constants.py @@ -1,10 +1,7 @@ """Some useful constants to use throughout certbot-ci integration tests""" DEFAULT_HTTP_01_PORT = 5002 -BOULDER_HTTP_01_PORT = 80 TLS_ALPN_01_PORT = 5001 CHALLTESTSRV_PORT = 8055 -BOULDER_V2_CHALLTESTSRV_URL = f'http://10.77.77.77:{CHALLTESTSRV_PORT}' -BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory' PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir' PEBBLE_MANAGEMENT_URL = 'https://localhost:15000' PEBBLE_CHALLTESTSRV_URL = f'http://localhost:{CHALLTESTSRV_PORT}' diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py index 89646f63b..cdce4f855 100644 --- a/certbot-ci/certbot_integration_tests/utils/misc.py +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -33,7 +33,6 @@ from cryptography.x509 import load_pem_x509_certificate from OpenSSL import crypto import requests -from certbot_integration_tests.certbot_tests.context import IntegrationTestsContext from certbot_integration_tests.utils.constants import PEBBLE_ALTERNATE_ROOTS from certbot_integration_tests.utils.constants import PEBBLE_MANAGEMENT_URL @@ -303,16 +302,12 @@ def echo(keyword: str, path: Optional[str] = None) -> str: os.path.basename(sys.executable), keyword, ' >> "{0}"'.format(path) if path else '') -def get_acme_issuers(context: IntegrationTestsContext) -> List[Certificate]: +def get_acme_issuers() -> List[Certificate]: """Gets the list of one or more issuer certificates from the ACME server used by the context. :param context: the testing context. :return: the `list of x509.Certificate` representing the list of issuers. """ - # TODO: in fact, Boulder has alternate chains in config-next/, just not yet in config/. - if context.acme_server != "pebble": - raise NotImplementedError() - _suppress_x509_verification_warnings() issuers = [] diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 8214eb761..ecefcb586 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -1,11 +1,13 @@ # pylint: disable=missing-module-docstring import atexit +import io import json import os import stat import sys +import zipfile from contextlib import ExitStack -from typing import Tuple +from typing import Optional, Tuple import requests @@ -17,39 +19,49 @@ if sys.version_info >= (3, 9): # pragma: no cover else: # pragma: no cover import importlib_resources -PEBBLE_VERSION = 'v2.3.1' +PEBBLE_VERSION = 'v2.5.1' def fetch(workspace: str, http_01_port: int = DEFAULT_HTTP_01_PORT) -> Tuple[str, str, str]: # pylint: disable=missing-function-docstring - suffix = 'linux-amd64' if os.name != 'nt' else 'windows-amd64.exe' - file_manager = ExitStack() atexit.register(file_manager.close) pebble_path_ref = importlib_resources.files('certbot_integration_tests') / 'assets' assets_path = str(file_manager.enter_context(importlib_resources.as_file(pebble_path_ref))) - pebble_path = _fetch_asset('pebble', suffix, assets_path) - challtestsrv_path = _fetch_asset('pebble-challtestsrv', suffix, assets_path) + pebble_path = _fetch_asset('pebble', assets_path) + challtestsrv_path = _fetch_asset('pebble-challtestsrv', assets_path) pebble_config_path = _build_pebble_config(workspace, http_01_port, assets_path) return pebble_path, challtestsrv_path, pebble_config_path -def _fetch_asset(asset: str, suffix: str, assets_path: str) -> str: - asset_path = os.path.join(assets_path, '{0}_{1}_{2}'.format(asset, PEBBLE_VERSION, suffix)) +def _fetch_asset(asset: str, assets_path: str) -> str: + platform = 'linux-amd64' + base_url = 'https://github.com/letsencrypt/pebble/releases/download' + asset_path = os.path.join(assets_path, f'{asset}_{PEBBLE_VERSION}_{platform}') if not os.path.exists(asset_path): - asset_url = ('https://github.com/letsencrypt/pebble/releases/download/{0}/{1}_{2}' - .format(PEBBLE_VERSION, asset, suffix)) + asset_url = f'{base_url}/{PEBBLE_VERSION}/{asset}-{platform}.zip' response = requests.get(asset_url, timeout=30) response.raise_for_status() + asset_data = _unzip_asset(response.content, asset) + if asset_data is None: + raise ValueError(f"zipfile {asset_url} didn't contain file {asset}") with open(asset_path, 'wb') as file_h: - file_h.write(response.content) + file_h.write(asset_data) os.chmod(asset_path, os.stat(asset_path).st_mode | stat.S_IEXEC) return asset_path +def _unzip_asset(zipped_data: bytes, asset_name: str) -> Optional[bytes]: + with zipfile.ZipFile(io.BytesIO(zipped_data)) as zip_file: + for entry in zip_file.filelist: + if not entry.is_dir() and entry.filename.endswith(asset_name): + return zip_file.read(entry) + return None + + def _build_pebble_config(workspace: str, http_01_port: int, assets_path: str) -> str: config_path = os.path.join(workspace, 'pebble-config.json') with open(config_path, 'w') as file_h: diff --git a/tox.ini b/tox.ini index 1ad33dc27..7fa72525a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,5 @@ [tox] -# mypy doesn't current pass for us on Windows. Fixing that is being tracked by -# https://github.com/certbot/certbot/issues/7803. -envlist = {cover,lint}-{win,posix},mypy +envlist = {cover,lint}-posix,mypy skipsdist = true @@ -10,11 +8,10 @@ pytest = python -m pytest {posargs} # Paths are listed on one line because tox seems to have inconsistent # behavior with substitutions that contain line continuations, see # https://github.com/tox-dev/tox/issues/2069 for more info. -source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-ci/windows_installer_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx +source_paths = acme/acme certbot/certbot certbot-apache/certbot_apache certbot-ci/certbot_integration_tests certbot-ci/snap_integration_tests certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx [testenv] platform = - win: win32 posix: ^(?!.*win32).*$ setenv = PYTEST_ADDOPTS = {env:PYTEST_ADDOPTS:--numprocesses auto} @@ -25,7 +22,7 @@ deps = -e acme[test] -e certbot -e certbot[test] - !win: -e certbot-apache[dev] + -e certbot-apache[dev] -e certbot-dns-cloudflare -e certbot-dns-digitalocean -e certbot-dns-dnsimple @@ -49,14 +46,9 @@ commands = echo "Unrecognized environment name {envname}" false -[testenv:py-win] -commands = - {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx - [testenv:py{,-posix}] -# We want to test everything we do on Windows plus the Apache plugin. commands = - {[testenv:py-win]commands} certbot-apache + {[base]pytest} acme certbot certbot-dns-cloudflare certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud certbot-nginx certbot-apache [testenv:py3{,8,9,10,11,12}] commands = {[testenv:py]commands} @@ -77,7 +69,6 @@ commands = {[testenv:py]commands} [testenv:cover] coverage_report = python -m coverage report -# These coverage report commands are used on both posix and windows common_coverage_report_commands = {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-cloudflare/* {[testenv:cover]coverage_report} --fail-under 99 --include certbot-dns-digitalocean/* @@ -104,14 +95,7 @@ commands = [testenv:cover-posix] commands = {[testenv:cover]commands} -[testenv:cover-win] -commands = - {[testenv:py-win]commands} --cov --cov-report= - {[testenv:cover]coverage_report} --fail-under 99 --include acme/* - {[testenv:cover]coverage_report} --fail-under 96 --include certbot/* - {[testenv:cover]common_coverage_report_commands} - -[testenv:lint{,-win,-posix}] +[testenv:lint{,-posix}] commands = python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] @@ -241,7 +225,6 @@ deps = -e certbot-ci commands = {[base]pytest} certbot-ci/certbot_integration_tests \ - --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=65 @@ -255,7 +238,6 @@ deps = -e certbot-ci commands = {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \ - --acme-server={env:ACME_SERVER:pebble} \ --cov=acme --cov=certbot --cov-report= \ --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=62 @@ -268,7 +250,7 @@ deps = -e certbot-ci commands = {[base]pytest} certbot-ci/certbot_integration_tests/rfc2136_tests \ - --acme-server=pebble --dns-server=bind \ + --dns-server=bind \ --numprocesses=1 \ --cov=acme --cov=certbot --cov=certbot_dns_rfc2136 --cov-report= \ --cov-config=certbot-ci/certbot_integration_tests/.coveragerc @@ -280,8 +262,7 @@ description = Run integration tests with Certbot outside of the tox virtual envi deps = -e certbot-ci commands = - {[base]pytest} certbot-ci/certbot_integration_tests \ - --acme-server={env:ACME_SERVER:pebble} + {[base]pytest} certbot-ci/certbot_integration_tests passenv = DOCKER_* [testenv:integration-certbot-oldest] @@ -292,8 +273,7 @@ deps = basepython = {[testenv:oldest]basepython} commands = - {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests \ - --acme-server={env:ACME_SERVER:pebble} + {[base]pytest} certbot-ci/certbot_integration_tests/certbot_tests passenv = DOCKER_* setenv = {[testenv:oldest]setenv} @@ -306,8 +286,7 @@ deps = basepython = {[testenv:oldest]basepython} commands = - {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests \ - --acme-server={env:ACME_SERVER:pebble} + {[base]pytest} certbot-ci/certbot_integration_tests/nginx_tests passenv = DOCKER_* setenv = {[testenv:oldest]setenv}