Merge pull request #1594 from letsencrypt/webroot

Support multiple webroot paths

letsencrypt/cli.py

@@ -1,8 +1,11 @@
 """Let's Encrypt CLI."""
 # TODO: Sanity check all input.  Be sure to avoid shell code etc...
+# pylint: disable=too-many-lines
+# (TODO: split this file into main.py and cli.py)
 import argparse
 import atexit
 import functools
+import json
 import logging
 import logging.handlers
 import os
@@ -100,7 +103,7 @@ def usage_strings(plugins):
 
 
 def _find_domains(args, installer):
-    if args.domains is None:
+    if not args.domains:
         domains = display_ops.choose_names(installer)
     else:
         domains = args.domains
@@ -383,7 +386,6 @@ def diagnose_configurator_problem(cfg_type, requested, plugins):
 def choose_configurator_plugins(args, config, plugins, verb):  # pylint: disable=too-many-branches
     """
     Figure out which configurator we're going to use
-
     :raises error.PluginSelectionError if there was a problem
     """
 
@@ -475,7 +477,7 @@ def run(args, config, plugins):  # pylint: disable=too-many-branches,too-many-lo
 def obtain_cert(args, config, plugins):
     """Authenticate & obtain cert, but do not install it."""
 
-    if args.domains is not None and args.csr is not None:
+    if args.domains and args.csr is not None:
         # TODO: --csr could have a priority, when --domains is
         # supplied, check if CSR matches given domains?
         return "--domains and --csr are mutually exclusive"
@@ -671,7 +673,6 @@ class HelpfulArgumentParser(object):
             print usage
             sys.exit(0)
         self.visible_topics = self.determine_help_topics(self.help_arg)
-        #print self.visible_topics
         self.groups = {}  # elements are added by .add_group()
 
     def parse_args(self):
@@ -684,31 +685,8 @@ class HelpfulArgumentParser(object):
         parsed_args = self.parser.parse_args(self.args)
         parsed_args.func = self.VERBS[self.verb]
 
-        parsed_args.domains = self._parse_domains(parsed_args.domains)
         return parsed_args
 
-    def _parse_domains(self, domains):
-        """Helper function for parse_args() that parses domains from a
-        (possibly) comma separated list and returns list of unique domains.
-
-        :param domains: List of domain flags
-        :type domains: `list` of `string`
-
-        :returns: List of unique domains
-        :rtype: `list` of `string`
-
-        """
-
-        uniqd = None
-
-        if domains:
-            dlist = []
-            for domain in domains:
-                dlist.extend([d.strip() for d in domain.split(",")])
-            # Make sure we don't have duplicates
-            uniqd = [d for i, d in enumerate(dlist) if d not in dlist[:i]]
-
-        return uniqd
 
     def determine_verb(self):
         """Determines the verb/subcommand provided by the user.
@@ -861,8 +839,8 @@ def prepare_and_parse_args(plugins, args):
     # --domains is useful, because it can be stored in config
     #for subparser in parser_run, parser_auth, parser_install:
     #    subparser.add_argument("domains", nargs="*", metavar="domain")
-    helpful.add(None, "-d", "--domains", dest="domains",
-                metavar="DOMAIN", action="append",
+    helpful.add(None, "-d", "--domains", "--domain", dest="domains",
+                metavar="DOMAIN", action=DomainFlagProcessor, default=[],
                 help="Domain names to apply. For multiple domains you can use "
                 "multiple -d flags or enter a comma separated list of domains "
                 "as a parameter.")
@@ -968,8 +946,7 @@ def _create_subparsers(helpful):
         help="Set a custom user agent string for the client. User agent strings allow "
              "the CA to collect high level statistics about success rates by OS and "
              "plugin. If you wish to hide your server OS version from the Let's "
-             'Encrypt server, set this to "".'
-    )
+             'Encrypt server, set this to "".')
     helpful.add("certonly",
                 "--csr", type=read_file,
                 help="Path to a Certificate Signing Request (CSR) in DER"
@@ -1071,6 +1048,61 @@ def _plugins_parsing(helpful, plugins):
 
     helpful.add_plugin_args(plugins)
 
+    # These would normally be a flag within the webroot plugin, but because
+    # they are parsed in conjunction with --domains, they live here for
+    # legibiility. helpful.add_plugin_ags must be called first to add the
+    # "webroot" topic
+    helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor,
+                help="public_html / webroot path. This can be specified multiple times to "
+                     "handle different domains; each domain will have the webroot path that"
+                     " precededed it.  For instance: `-w /var/www/example -d example.com -d "
+                     "www.example.com -w /var/www/thing -d thing.net -d m.thing.net`")
+    parse_dict = lambda s: dict(json.loads(s))
+    # --webroot-map still has some awkward properties, so it is undocumented
+    helpful.add("webroot", "--webroot-map", default={}, type=parse_dict,
+                help=argparse.SUPPRESS)
+
+
+class WebrootPathProcessor(argparse.Action): # pylint: disable=missing-docstring
+    def __init__(self, *args, **kwargs):
+        self.domain_before_webroot = False
+        argparse.Action.__init__(self, *args, **kwargs)
+
+    def __call__(self, parser, config, webroot, option_string=None):
+        """
+        Keep a record of --webroot-path / -w flags during processing, so that
+        we know which apply to which -d flags
+        """
+        if config.webroot_path is None:      # first -w flag encountered
+            config.webroot_path = []
+            # if any --domain flags preceded the first --webroot-path flag,
+            # apply that webroot path to those; subsequent entries in
+            # config.webroot_map are filled in by cli.DomainFlagProcessor
+            if config.domains:
+                self.domain_before_webroot = True
+                for d in config.domains:
+                    config.webroot_map.setdefault(d, webroot)
+        elif self.domain_before_webroot:
+            # FIXME if you set domains in a config file, you should get a different error
+            # here, pointing you to --webroot-map
+            raise errors.Error("If you specify multiple webroot paths, one of "
+                               "them must precede all domain flags")
+        config.webroot_path.append(webroot)
+
+
+class DomainFlagProcessor(argparse.Action): # pylint: disable=missing-docstring
+    def __call__(self, parser, config, domain_arg, option_string=None):
+        """
+        Process a new -d flag, helping the webroot plugin construct a map of
+        {domain : webrootpath} if -w / --webroot-path is in use
+        """
+        for domain in (d.strip() for d in domain_arg.split(",")):
+            if domain not in config.domains:
+                config.domains.append(domain)
+                # Each domain has a webroot_path of the most recent -w flag
+                if config.webroot_path:
+                    config.webroot_map[domain] = config.webroot_path[-1]
+
 
 def setup_log_file_handler(args, logfile, fmt):
     """Setup file debug logging."""

letsencrypt/plugins/webroot.py

@@ -33,7 +33,9 @@ to serve all files under specified web root ({0})."""
 
     @classmethod
     def add_parser_arguments(cls, add):
-        add("path", help="public_html / webroot path")
+        # --webroot-path and --webroot-map are added in cli.py because they
+        # are parsed in conjunction with --domains
+        pass
 
     def get_chall_pref(self, domain):  # pragma: no cover
         # pylint: disable=missing-docstring,no-self-use,unused-argument
@@ -41,34 +43,43 @@ to serve all files under specified web root ({0})."""
 
     def __init__(self, *args, **kwargs):
         super(Authenticator, self).__init__(*args, **kwargs)
-        self.full_root = None
+        self.full_roots = {}
 
     def prepare(self):  # pylint: disable=missing-docstring
-        path = self.conf("path")
-        if path is None:
+        path_map = self.conf("map")
+
+        if not path_map:
             raise errors.PluginError("--{0} must be set".format(
                 self.option_name("path")))
-        if not os.path.isdir(path):
-            raise errors.PluginError(
-                path + " does not exist or is not a directory")
-        self.full_root = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH)
+        for name, path in path_map.items():
+            if not os.path.isdir(path):
+                raise errors.PluginError(path + " does not exist or is not a directory")
+            self.full_roots[name] = os.path.join(path, challenges.HTTP01.URI_ROOT_PATH)
 
-        logger.debug("Creating root challenges validation dir at %s",
-                     self.full_root)
-        try:
-            os.makedirs(self.full_root)
-        except OSError as exception:
-            if exception.errno != errno.EEXIST:
-                raise errors.PluginError(
-                    "Couldn't create root for http-01 "
-                    "challenge responses: {0}", exception)
+            logger.debug("Creating root challenges validation dir at %s",
+                         self.full_roots[name])
+            try:
+                os.makedirs(self.full_roots[name])
+            except OSError as exception:
+                if exception.errno != errno.EEXIST:
+                    raise errors.PluginError(
+                        "Couldn't create root for {0} http-01 "
+                        "challenge responses: {1}", name, exception)
 
     def perform(self, achalls):  # pylint: disable=missing-docstring
-        assert self.full_root is not None
+        assert self.full_roots, "Webroot plugin appears to be missing webroot map"
         return [self._perform_single(achall) for achall in achalls]
 
     def _path_for_achall(self, achall):
-        return os.path.join(self.full_root, achall.chall.encode("token"))
+        try:
+            path = self.full_roots[achall.domain]
+        except IndexError:
+            raise errors.PluginError("Missing --webroot-path for domain: {1}"
+                        .format(achall.domain))
+        if not os.path.exists(path):
+            raise errors.PluginError("Mysteriously missing path {0} for domain: {1}"
+                        .format(path, achall.domain))
+        return os.path.join(path, achall.chall.encode("token"))
 
     def _perform_single(self, achall):
         response, validation = achall.response_and_validation()

letsencrypt/plugins/webroot_test.py

@@ -23,7 +23,7 @@ class AuthenticatorTest(unittest.TestCase):
     """Tests for letsencrypt.plugins.webroot.Authenticator."""
 
     achall = achallenges.KeyAuthorizationAnnotatedChallenge(
-        challb=acme_util.HTTP01_P, domain=None, account_key=KEY)
+        challb=acme_util.HTTP01_P, domain="thing.com", account_key=KEY)
 
     def setUp(self):
         from letsencrypt.plugins.webroot import Authenticator
@@ -31,7 +31,8 @@ class AuthenticatorTest(unittest.TestCase):
         self.validation_path = os.path.join(
             self.path, ".well-known", "acme-challenge",
             "ZXZhR3hmQURzNnBTUmIyTEF2OUlaZjE3RHQzanV4R0orUEN0OTJ3citvQQ")
-        self.config = mock.MagicMock(webroot_path=self.path)
+        self.config = mock.MagicMock(webroot_path=self.path,
+                                     webroot_map={"thing.com": self.path})
         self.auth = Authenticator(self.config, "webroot")
         self.auth.prepare()
 
@@ -46,14 +47,16 @@ class AuthenticatorTest(unittest.TestCase):
     def test_add_parser_arguments(self):
         add = mock.MagicMock()
         self.auth.add_parser_arguments(add)
-        self.assertEqual(1, add.call_count)
+        self.assertEqual(0, add.call_count) # became 0 when we moved the args to cli.py!
 
     def test_prepare_bad_root(self):
         self.config.webroot_path = os.path.join(self.path, "null")
+        self.config.webroot_map["thing.com"] = self.config.webroot_path
         self.assertRaises(errors.PluginError, self.auth.prepare)
 
     def test_prepare_missing_root(self):
         self.config.webroot_path = None
+        self.config.webroot_map = {}
         self.assertRaises(errors.PluginError, self.auth.prepare)
 
     def test_prepare_full_root_exists(self):

letsencrypt/tests/cli_test.py

@@ -236,7 +236,8 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
             self._call(['plugins'] + list(args))
 
     @mock.patch('letsencrypt.cli.plugins_disco')
-    def test_plugins_no_args(self, mock_disco):
+    @mock.patch('letsencrypt.cli.HelpfulArgumentParser.determine_help_topics')
+    def test_plugins_no_args(self, _det, mock_disco):
         ifaces = []
         plugins = mock_disco.PluginsRegistry.find_all()
 
@@ -247,7 +248,8 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         stdout.write.called_once_with(str(filtered))
 
     @mock.patch('letsencrypt.cli.plugins_disco')
-    def test_plugins_init(self, mock_disco):
+    @mock.patch('letsencrypt.cli.HelpfulArgumentParser.determine_help_topics')
+    def test_plugins_init(self, _det, mock_disco):
         ifaces = []
         plugins = mock_disco.PluginsRegistry.find_all()
 
@@ -261,10 +263,10 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         stdout.write.called_once_with(str(verified))
 
     @mock.patch('letsencrypt.cli.plugins_disco')
-    def test_plugins_prepare(self, mock_disco):
+    @mock.patch('letsencrypt.cli.HelpfulArgumentParser.determine_help_topics')
+    def test_plugins_prepare(self, _det, mock_disco):
         ifaces = []
         plugins = mock_disco.PluginsRegistry.find_all()
-
         _, stdout, _, _ = self._call(['plugins', '--init', '--prepare'])
         plugins.visible.assert_called_once_with()
         plugins.visible().ifaces.assert_called_once_with(ifaces)
@@ -339,6 +341,25 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         namespace = cli.prepare_and_parse_args(plugins, long_args)
         self.assertEqual(namespace.domains, ['example.com', 'another.net'])
 
+    def test_parse_webroot(self):
+        plugins = disco.PluginsRegistry.find_all()
+        webroot_args = ['--webroot', '-w', '/var/www/example',
+            '-d', 'example.com,www.example.com', '-w', '/var/www/superfluous',
+            '-d', 'superfluo.us', '-d', 'www.superfluo.us']
+        namespace = cli.prepare_and_parse_args(plugins, webroot_args)
+        self.assertEqual(namespace.webroot_map, {
+            'example.com': '/var/www/example',
+            'www.example.com': '/var/www/example',
+            'www.superfluo.us': '/var/www/superfluous',
+            'superfluo.us': '/var/www/superfluous'})
+
+        webroot_args = ['-d', 'stray.example.com'] + webroot_args
+        self.assertRaises(errors.Error, cli.prepare_and_parse_args, plugins, webroot_args)
+
+        webroot_map_args = ['--webroot-map', '{"eg.com" : "/tmp"}']
+        namespace = cli.prepare_and_parse_args(plugins, webroot_map_args)
+        self.assertEqual(namespace.webroot_map, {u"eg.com": u"/tmp"})
+
     @mock.patch('letsencrypt.crypto_util.notAfter')
     @mock.patch('letsencrypt.cli.zope.component.getUtility')
     def test_certonly_new_request_success(self, mock_get_utility, mock_notAfter):