Add --dns-server option in run_acme_server (#7722)

Fixes #7717

This PR adds a `--dns-server` option to the `run_acme_server` test tool, in order to provide an arbitrary DNS server to Pebble or Boulder for the integration tests.

I also take this occasion to make `run_acme_server` a real CLI tool using argparse, and set the `--server-type` (default `pebble`) option as well.

* Set --dns-server flag in run_acme_server

* Default to pebble

* Add documentation

* Configure also Boulder

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -2,10 +2,12 @@
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
 from __future__ import print_function
 
+import argparse
 import errno
 import json
 import os
 from os.path import join
+import re
 import shutil
 import subprocess
 import sys
@@ -32,13 +34,14 @@ class ACMEServer(object):
     ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped
     upon context enter/exit.
     """
-    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
+    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False, dns_server=None):
         """
         Create an ACMEServer instance.
         :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
         :param list nodes: list of node names that will be setup by pytest xdist
         :param bool http_proxy: if False do not start the HTTP proxy
         :param bool stdout: if True stream all subprocesses stdout to standard stdout
+        :param str dns_server: if set, Pebble/Boulder will use it to resolve domains
         """
         self._construct_acme_xdist(acme_server, nodes)
 
@@ -47,6 +50,7 @@ class ACMEServer(object):
         self._workspace = tempfile.mkdtemp()
         self._processes = []
         self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
+        self._dns_server = dns_server
 
     def start(self):
         """Start the test stack"""
@@ -132,13 +136,18 @@ class ACMEServer(object):
         environ['PEBBLE_AUTHZREUSE'] = '100'
         environ['PEBBLE_ALTERNATE_ROOTS'] = str(PEBBLE_ALTERNATE_ROOTS)
 
-        self._launch_process(
-            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'],
-            env=environ)
+        if self._dns_server:
+            dns_server = self._dns_server
+        else:
+            dns_server = '127.0.0.1:8053'
+            self._launch_process(
+                [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT),
+                 '-defaultIPv6', '""', '-defaultIPv4', '127.0.0.1', '-http01', '""',
+                 '-tlsalpn01', '""', '-https01', '""'])
 
         self._launch_process(
-            [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
-             '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'],
+            env=environ)
 
         # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
         # ImportError, in the case where cryptography dependency is too old to support ocsp, but
@@ -167,6 +176,14 @@ class ACMEServer(object):
         os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
                   join(instance_path, 'test/rate-limit-policies.yml'))
 
+        if self._dns_server:
+            # Change Boulder config to use the provided DNS server
+            with open(join(instance_path, 'test/config/va.json'), 'r') as file_h:
+                config = json.loads(file_h.read())
+            config['va']['dnsResolvers'] = [self._dns_server]
+            with open(join(instance_path, 'test/config/va.json'), 'w') as file_h:
+                file_h.write(json.dumps(config, indent=2, separators=(',', ': ')))
+
         try:
             # Launch the Boulder server
             self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
@@ -175,10 +192,11 @@ class ACMEServer(object):
             print('=> Waiting for boulder instance to respond...')
             misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300)
 
-            # Configure challtestsrv to answer any A record request with ip of the docker host.
-            response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
-                                     json={'ip': '10.77.77.1'})
-            response.raise_for_status()
+            if not self._dns_server:
+                # Configure challtestsrv to answer any A record request with ip of the docker host.
+                response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                         json={'ip': '10.77.77.1'})
+                response.raise_for_status()
         except BaseException:
             # If we failed to set up boulder, print its logs.
             print('=> Boulder setup failed. Boulder logs are:')
@@ -208,14 +226,19 @@ class ACMEServer(object):
 
 
 def main():
-    args = sys.argv[1:]
-    server_type = args[0] if args else 'pebble'
-    possible_values = ('pebble', 'boulder-v1', 'boulder-v2')
-    if server_type not in possible_values:
-        raise ValueError('Invalid server value {0}, should be one of {1}'
-                         .format(server_type, possible_values))
+    parser = argparse.ArgumentParser(
+        description='CLI tool to start a local instance of Pebble or Boulder CA server.')
+    parser.add_argument('--server-type', '-s',
+                        choices=['pebble', 'boulder-v1', 'boulder-v2'], default='pebble',
+                        help='type of CA server to start: can be Pebble or Boulder '
+                             '(in ACMEv1 or ACMEv2 mode), Pebble is used if not set.')
+    parser.add_argument('--dns-server', '-d',
+                        help='specify the DNS server as `IP:PORT` to use by '
+                             'Pebble; if not specified, a local mock DNS server will be used to '
+                             'resolve domains to localhost.')
+    args = parser.parse_args()
 
-    acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True)
+    acme_server = ACMEServer(args.server_type, [], http_proxy=False, stdout=True, dns_server=args.dns_server)
 
     try:
         with acme_server as acme_xdist:

certbot/docs/contributing.rst

@@ -169,7 +169,7 @@ To do so you need:
 - Docker installed, and a user with access to the Docker client,
 - an available `local copy`_ of Certbot.
 
-The virtual environment set up with `python tools/venv3.py` contains two commands
+The virtual environment set up with `python tools/venv3.py` contains two CLI tools
 that can be used once the virtual environment is activated:
 
 .. code-block:: shell
@@ -180,6 +180,9 @@ that can be used once the virtual environment is activated:
 - Press CTRL+C to stop this instance.
 - This instance is configured to validate challenges against certbot executed locally.
 
+.. note:: Some options are available to tweak the local ACME server. You can execute
+    ``run_acme_server --help`` to see the inline help of the ``run_acme_server`` tool.
+
 .. code-block:: shell
 
     certbot_test [ARGS...]