From 81cda2903a11b660a88b4b346dd4d9870b9e8260 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Fri, 3 Jun 2016 15:30:20 -0700 Subject: [PATCH] Attempt at putting everything inside Docker --- .../configurators/apache/a2dismod.sh | 14 -- .../configurators/apache/a2enmod.sh | 18 --- .../configurators/apache/apache24.py | 63 -------- .../configurators/apache/common.py | 150 +----------------- .../configurators/common.py | 83 +--------- .../certbot_compatibility_test/test_driver.py | 4 +- .../certbot_compatibility_test/util.py | 10 -- 7 files changed, 5 insertions(+), 337 deletions(-) delete mode 100755 certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh delete mode 100755 certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh delete mode 100644 certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh deleted file mode 100755 index ca96e216f..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for disabling modules in the -# httpd docker image. First argument is the server_root and the second is the -# module to be disabled. - -apache_confdir=$1 -module=$2 - -sed -i "/.*"$module".*/d" "$apache_confdir/test.conf" -enabled_conf="$apache_confdir/mods-enabled/"$module".conf" -if [ -e "$enabled_conf" ] -then - rm $enabled_conf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh deleted file mode 100755 index 4da9288a2..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for enabling modules in the -# httpd docker image. First argument is the Apache ServerRoot which should be -# an absolute path. The second is the module to be enabled, such as `ssl`. - -confdir=$1 -module=$2 - -echo "LoadModule ${module}_module " \ - "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf" -availbase="/mods-available/${module}.conf" -availconf=$confdir$availbase -enabldir="$confdir/mods-enabled" -enablconf="$enabldir/${module}.conf" -if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ] -then - ln -s "..$availbase" $enablconf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py deleted file mode 100644 index 927c329ef..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Proxies ApacheConfigurator for Apache 2.4 tests""" - -import zope.interface - -from certbot_compatibility_test import errors -from certbot_compatibility_test import interfaces -from certbot_compatibility_test.configurators.apache import common as apache_common - - -# The docker image doesn't actually have the watchdog module, but unless the -# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and -# therefore the config won't be loaded), I believe this isn't a problem -# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html -STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog"]) - - -SHARED_MODULES = { - "log_config", "logio", "version", "unixd", "access_compat", "actions", - "alias", "allowmethods", "auth_basic", "auth_digest", "auth_form", - "authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file", - "authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm", - "authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex", - "buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs", - "dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter", - "file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness", - "lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap", - "log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp", - "proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi", - "proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit", - "remoteip", "reqtimeout", "request", "rewrite", "sed", "session", - "session_cookie", "session_crypto", "session_dbd", "setenvif", - "slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb", - "speling", "ssl", "status", "substitute", "unique_id", "userdir", - "vhost_alias"} - - -@zope.interface.implementer(interfaces.IConfiguratorProxy) -class Proxy(apache_common.Proxy): - """Wraps the ApacheConfigurator for Apache 2.4 tests""" - - def __init__(self, args): - """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) - # Running init isn't ideal, but the Docker container needs to survive - # Apache restarts - self.start_docker("bradmw/apache2.4", "init") - - def preprocess_config(self, server_root): - """Prepares the configuration for use in the Docker""" - super(Proxy, self).preprocess_config(server_root) - if self.version[1] != 4: - raise errors.Error("Apache version not 2.4") - - with open(self.test_conf, "a") as f: - for module in self.modules: - if module not in STATIC_MODULES: - if module in SHARED_MODULES: - f.write( - "LoadModule {0}_module /usr/local/apache2/modules/" - "mod_{0}.so\n".format(module)) - else: - raise errors.Error( - "Unsupported module {0}".format(module)) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index 9148666fc..7af0ee20e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -29,58 +29,9 @@ class Proxy(configurators_common.Proxy): super(Proxy, self).__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" - self._setup_mock() - self.modules = self.server_root = self.test_conf = self.version = None self._apache_configurator = self._all_names = self._test_names = None - def _setup_mock(self): - """Replaces specific modules with mock.MagicMock""" - mock_subprocess = mock.MagicMock() - mock_subprocess.check_call = self.check_call - mock_subprocess.Popen = self.popen - - mock.patch( - "certbot_apache.configurator.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.parser.subprocess", - mock_subprocess).start() - mock.patch( - "certbot.util.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.configurator.util.exe_exists", - _is_apache_command).start() - - patch = mock.patch( - "certbot_apache.configurator.display_ops.select_vhost") - mock_display = patch.start() - mock_display.side_effect = le_errors.PluginError( - "Unable to determine vhost") - - def check_call(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.check_call is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).check_call(command, *args, **kwargs) - else: - return subprocess.check_call(command, *args, **kwargs) - - def popen(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.Popen is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).popen(command, *args, **kwargs) - else: - return subprocess.Popen(command, *args, **kwargs) - def __getattr__(self, name): """Wraps the Apache Configurator methods""" method = getattr(self._apache_configurator, name, None) @@ -91,28 +42,19 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - if hasattr(self.le_config, "apache_init_script"): - try: - self.check_call([self.le_config.apache_init_script, "stop"]) - except errors.Error: - raise errors.Error( - "Failed to stop previous apache config from running") config = super(Proxy, self).load_config() - self.modules = _get_modules(config) - self.version = _get_version(config) self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) with open(os.path.join(config, "config_file")) as f: config_file = os.path.join(server_root, f.readline().rstrip()) - self.test_conf = _create_test_conf(server_root, config_file) self.preprocess_config(server_root) self._prepare_configurator(server_root, config_file) try: - self.check_call("apachectl -d {0} -f {1} -k start".format( + subprocess.check_call("apachectl -d {0} -f {1} -k start".format( server_root, config_file)) except errors.Error: raise errors.Error( @@ -121,34 +63,9 @@ class Proxy(configurators_common.Proxy): return config - def preprocess_config(self, server_root): - # pylint: disable=anomalous-backslash-in-string, no-self-use - """Prepares the configuration for use in the Docker""" - - find = subprocess.Popen( - ["find", server_root, "-type", "f"], - stdout=subprocess.PIPE) - subprocess.check_call([ - "xargs", "sed", "-e", "s/DocumentRoot.*/DocumentRoot " - "\/usr\/local\/apache2\/htdocs/I", - "-e", "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I", - "-e", "s/TypesConfig.*/TypesConfig " - "\/usr\/local\/apache2\/conf\/mime.types/I", - "-e", "s/LoadModule/#LoadModule/I", - "-e", "s/SSLCertificateFile.*/SSLCertificateFile " - "\/usr\/local\/apache2\/conf\/empty_cert.pem/I", - "-e", "s/SSLCertificateKeyFile.*/SSLCertificateKeyFile " - "\/usr\/local\/apache2\/conf\/rsa1024_key2.pem/I", - "-i"], stdin=find.stdout) - def _prepare_configurator(self, server_root, config_file): """Prepares the Apache plugin for testing""" self.le_config.apache_server_root = server_root - self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format( - server_root, config_file) - self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root) - self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root) - self.le_config.apache_init_script = self.le_config.apache_ctl + " -k" self._apache_configurator = configurator.ApacheConfigurator( config=configuration.NamespaceConfig(self.le_config), @@ -158,7 +75,6 @@ class Proxy(configurators_common.Proxy): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" super(Proxy, self).cleanup_from_tests() - mock.patch.stopall() def get_all_names_answer(self): """Returns the set of domain names that the plugin should find""" @@ -183,39 +99,6 @@ class Proxy(configurators_common.Proxy): domain, cert_path, key_path, chain_path, fullchain_path) -def _is_apache_command(command): - """Returns true if command is an Apache command""" - if isinstance(command, list): - command = command[0] - - for apache_command in APACHE_COMMANDS: - if command.startswith(apache_command): - return True - - return False - - -def _modify_command(command): - """Modifies command so configtest works inside the docker image""" - if isinstance(command, list): - for i in xrange(len(command)): - if command[i] == "configtest": - command[i] = "-t" - else: - command = command.replace("configtest", "-t") - - return command - - -def _create_test_conf(server_root, apache_config): - """Creates a test config file and adds it to the Apache config""" - test_conf = os.path.join(server_root, "test.conf") - open(test_conf, "w").close() - subprocess.check_call( - ["sed", "-i", "1iInclude test.conf", apache_config]) - return test_conf - - def _get_server_root(config): """Returns the server root directory in config""" subdirs = [ @@ -251,34 +134,3 @@ def _get_names(config): words[1].find(".") != -1): all_names.add(words[1]) return all_names, non_ip_names - - -def _get_modules(config): - """Returns the list of modules found in module_list""" - modules = [] - with open(os.path.join(config, "modules")) as f: - for line in f: - # Modules list is indented, everything else is headers/footers - if line[0].isspace(): - words = line.split() - # Modules redundantly end in "_module" which we can discard - modules.append(words[0][:-7]) - - return modules - - -def _get_version(config): - """Return version of Apache Server. - - Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from - the Apache plugin. - - """ - with open(os.path.join(config, "version")) as f: - # Should be on first line of input - matches = APACHE_VERSION_REGEX.findall(f.readline()) - - if len(matches) != 1: - raise errors.Error("Unable to find Apache version") - - return tuple([int(i) for i in matches[0].split(".")]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 4657883a3..4592eca39 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -4,8 +4,6 @@ import os import shutil import tempfile -import docker - from certbot import constants from certbot_compatibility_test import errors from certbot_compatibility_test import util @@ -18,20 +16,9 @@ class Proxy(object): # pylint: disable=too-many-instance-attributes """A common base for compatibility test configurators""" - _NOT_ADDED_ARGS = True - @classmethod def add_parser_arguments(cls, parser): """Adds command line arguments needed by the plugin""" - if Proxy._NOT_ADDED_ARGS: - group = parser.add_argument_group("docker") - group.add_argument( - "--docker-url", default="unix://var/run/docker.sock", - help="URL of the docker server") - group.add_argument( - "--no-remove", action="store_true", - help="do not delete container on program exit") - Proxy._NOT_ADDED_ARGS = False def __init__(self, args): """Initializes the plugin with the given command line args""" @@ -43,10 +30,8 @@ class Proxy(object): for config in os.listdir(config_dir)] self.args = args - self._docker_client = docker.Client( - base_url=self.args.docker_url, version="auto") - self.http_port, self.https_port = util.get_two_free_ports() - self._container_id = None + self.http_port = 80 + self.https_port = 443 def has_more_configs(self): """Returns true if there are more configs to test""" @@ -54,9 +39,6 @@ class Proxy(object): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - self._docker_client.stop(self._container_id, 0) - if not self.args.no_remove: - self._docker_client.remove_container(self._container_id) def load_config(self): """Returns the next config directory to be tested""" @@ -65,67 +47,6 @@ class Proxy(object): os.makedirs(backup) return self._configs.pop() - def start_docker(self, image_name, command): - """Creates and runs a Docker container with the specified image""" - logger.warning("Pulling Docker image. This may take a minute.") - for line in self._docker_client.pull(image_name, stream=True): - logger.debug(line) - - host_config = docker.utils.create_host_config( - binds={self._temp_dir: {"bind": self._temp_dir, "mode": "rw"}}, - port_bindings={ - 80: ("127.0.0.1", self.http_port), - 443: ("127.0.0.1", self.https_port)},) - container = self._docker_client.create_container( - image_name, command, ports=[80, 443], volumes=self._temp_dir, - host_config=host_config) - if container["Warnings"]: - logger.warning(container["Warnings"]) - self._container_id = container["Id"] - self._docker_client.start(self._container_id) - - def check_call(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to check_call but executes the command in the - running docker image - - """ - if self.popen(command).returncode: - raise errors.Error( - "{0} exited with a nonzero value".format(command)) - - def popen(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to Popen but executes the command in the - running docker image - - """ - class SimplePopen(object): - # pylint: disable=too-few-public-methods - """Simplified Popen object""" - def __init__(self, returncode, output): - self.returncode = returncode - self._stdout = output - self._stderr = output - - def communicate(self): - """Returns stdout and stderr""" - return self._stdout, self._stderr - - if isinstance(command, list): - command = " ".join(command) - - returncode, output = self.execute_in_docker(command) - return SimplePopen(returncode, output) - - def execute_in_docker(self, command): - """Executes command inside the running docker image""" - logger.debug("Executing '%s'", command) - exec_id = self._docker_client.exec_create(self._container_id, command) - output = self._docker_client.exec_start(exec_id) - returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"] - return returncode, output - def copy_certs_and_keys(self, cert_path, key_path, chain_path=None): """Copies certs and keys into the temporary directory""" cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys") diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 6823dfdab..ad5755d60 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -25,8 +25,8 @@ from certbot_compatibility_test.configurators.apache import apache24 DESCRIPTION = """ -Tests Certbot plugins against different server configuratons. It is -assumed that Docker is already installed. If no test types is specified, all +Tests Certbot plugins against different server configurations. It is +assumed that Docker is already installed. If no test type is specified, all tests that the plugin supports are performed. """ diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index cbce4fb56..570bf1a9e 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -52,13 +52,3 @@ def extract_configs(configs, parent_dir): raise errors.Error("Unknown configurations file type") return config_dir - - -def get_two_free_ports(): - """Returns two free ports to use for the tests""" - with contextlib.closing(socket.socket()) as sock1: - with contextlib.closing(socket.socket()) as sock2: - sock1.bind(("", 0)) - sock2.bind(("", 0)) - - return sock1.getsockname()[1], sock2.getsockname()[1]