Merge branch 'master' into warnings-are-errors

.gitignore

@@ -6,7 +6,8 @@ dist*/
 /venv*/
 /kgs/
 /.tox/
-/releases/
+/releases*/
+/log*
 letsencrypt.log
 certbot.log
 letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64
@@ -39,6 +40,7 @@ tests/letstest/venv/
 # pytest cache
 .cache
 .mypy_cache/
+.pytest_cache/
 
 # docker files
 .docker

.travis.yml

@@ -13,11 +13,11 @@ before_script:
 matrix:
   include:
     - python: "2.7"
-      env: TOXENV=py27_install BOULDER_INTEGRATION=v1
+      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
       sudo: required
       services: docker
     - python: "2.7"
-      env: TOXENV=py27_install BOULDER_INTEGRATION=v2
+      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
       sudo: required
       services: docker
     - python: "2.7"
@@ -34,7 +34,7 @@ matrix:
     - python: "3.5"
       env: TOXENV=mypy
     - python: "2.7"
-      env: TOXENV='py27-{acme,apache,certbot,dns,nginx}-oldest'
+      env: TOXENV='py27-{acme,apache,certbot,dns,nginx,postfix}-oldest'
       sudo: required
       services: docker
     - python: "3.4"
@@ -90,12 +90,12 @@ addons:
     - nginx-light
     - openssl
 
-install: "travis_retry $(command -v pip || command -v pip3) install tox coveralls"
+install: "travis_retry $(command -v pip || command -v pip3) install codecov tox"
 script:
     - travis_retry tox
     - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
-after_success: '[ "$TOXENV" == "cover" ] && coveralls'
+after_success: '[ "$TOXENV" == "cover" ] && codecov'
 
 notifications:
   email: false

CHANGELOG.md

@@ -2,6 +2,147 @@
 
 Certbot adheres to [Semantic Versioning](http://semver.org/).
 
+## 0.28.0 - master
+
+### Added
+
+* `revoke` accepts `--cert-name`, and doesn't accept both `--cert-name` and `--cert-path`.
+
+### Changed
+
+* Removed documentation mentions of `#letsencrypt` IRC on Freenode.
+* Write README to the base of (config-dir)/live directory
+* `--manual` will explicitly warn users that earlier challenges should remain in place when setting up subsequent challenges.
+
+### Fixed
+
+* Match Nginx parser update in allowing variable names to start with `${`.
+* Fix ranking of vhosts in Nginx so that all port-matching vhosts come first
+* Correct OVH integration tests on machines without internet access.
+* Stop caching the results of ipv6_info in http01.py
+
+## 0.27.1 - 2018-09-06
+
+### Fixed
+
+* Fixed parameter name in OpenSUSE overrides for default parameters in the
+  Apache plugin. Certbot on OpenSUSE works again.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* certbot-apache
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/60?closed=1
+
+## 0.27.0 - 2018-09-05
+
+### Added
+
+* The Apache plugin now accepts the parameter --apache-ctl which can be
+  used to configure the path to the Apache control script.
+
+### Changed
+
+* When using `acme.client.ClientV2` (or
+ `acme.client.BackwardsCompatibleClientV2` with an ACME server that supports a
+ newer version of the ACME protocol), an `acme.errors.ConflictError` will be
+ raised if you try to create an ACME account with a key that has already been
+ used. Previously, a JSON parsing error was raised in this scenario when using
+ the library with Let's Encrypt's ACMEv2 endpoint.
+
+### Fixed
+
+* When Apache is not installed, Certbot's Apache plugin no longer prints
+  messages about being unable to find apachectl to the terminal when the plugin
+  is not selected.
+* If you're using the Apache plugin with the --apache-vhost-root flag set to a
+  directory containing a disabled virtual host for the domain you're requesting
+  a certificate for, the virtual host will now be temporarily enabled if
+  necessary to pass the HTTP challenge.
+* The documentation for the Certbot package can now be built using Sphinx 1.6+.
+* You can now call `query_registration` without having to first call
+  `new_account` on `acme.client.ClientV2` objects.
+* The requirement of `setuptools>=1.0` has been removed from `certbot-dns-ovh`.
+* Names in certbot-dns-sakuracloud's tests have been updated to refer to Sakura
+  Cloud rather than NS1 whose plugin certbot-dns-sakuracloud was based on.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+package with changes other than its version number was:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/57?closed=1
+
+## 0.26.1 - 2018-07-17
+
+### Fixed
+
+* Fix a bug that was triggered when users who had previously manually set `--server` to get ACMEv2 certs tried to renew ACMEv1 certs.
+
+Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was:
+
+* certbot
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/58?closed=1
+
+## 0.26.0 - 2018-07-11
+
+### Added
+
+* A new security enhancement which we're calling AutoHSTS has been added to
+  Certbot's Apache plugin. This enhancement configures your webserver to send a
+  HTTP Strict Transport Security header with a low max-age value that is slowly
+  increased over time. The max-age value is not increased to a large value
+  until you've successfully managed to renew your certificate. This enhancement
+  can be requested with the --auto-hsts flag.
+* New official DNS plugins have been created for Gehirn Infrastracture Service,
+  Linode, OVH, and Sakura Cloud. These plugins can be found on our Docker Hub
+  page at https://hub.docker.com/u/certbot and on PyPI.
+* The ability to reuse ACME accounts from Let's Encrypt's ACMEv1 endpoint on
+  Let's Encrypt's ACMEv2 endpoint has been added.
+* Certbot and its components now support Python 3.7.
+* Certbot's install subcommand now allows you to interactively choose which
+  certificate to install from the list of certificates managed by Certbot.
+* Certbot now accepts the flag `--no-autorenew` which causes any obtained
+  certificates to not be automatically renewed when it approaches expiration.
+* Support for parsing the TLS-ALPN-01 challenge has been added back to the acme
+  library.
+
+### Changed
+
+* Certbot's default ACME server has been changed to Let's Encrypt's ACMEv2
+  endpoint. By default, this server will now be used for both new certificate
+  lineages and renewals.
+* The Nginx plugin is no longer marked labeled as an "Alpha" version.
+* The `prepare` method of Certbot's plugins is no longer called before running
+  "Updater" enhancements that are run on every invocation of `certbot renew`.
+
+Despite us having broken lockstep, we are continuing to release new versions of
+all Certbot components during releases for the time being, however, the only
+packages with functional changes were:
+
+* acme
+* certbot
+* certbot-apache
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
+* certbot-nginx
+
+More details about these changes can be found on our GitHub repo:
+https://github.com/certbot/certbot/milestone/55?closed=1
+
 ## 0.25.1 - 2018-06-13
 
 ### Fixed
@@ -735,7 +876,7 @@ https://github.com/certbot/certbot/pulls?q=is%3Apr%20milestone%3A0.11.1%20is%3Ac
 
 ### Added
 
-* When using the standalone plugin while running Certbot interactively 
+* When using the standalone plugin while running Certbot interactively
 and a required port is bound by another process, Certbot will give you
 the option to retry to grab the port rather than immediately exiting.
 * You are now able to deactivate your account with the Let's Encrypt

CHANGES.rst

@@ -1,8 +0,0 @@
-ChangeLog
-=========
-
-To see the changes in a given release, view the issues closed in a given
-release's GitHub milestone:
-
- - `Past releases <https://github.com/certbot/certbot/milestones?state=closed>`_
- - `Upcoming releases <https://github.com/certbot/certbot/milestones>`_

Dockerfile

@@ -5,7 +5,7 @@ EXPOSE 80 443
 VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
-COPY CHANGES.rst README.rst setup.py src/
+COPY CHANGELOG.md README.rst setup.py src/
 COPY acme src/acme
 COPY certbot src/certbot
 

Dockerfile-old

@@ -34,7 +34,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # Dockerfile we make sure we cache as much as possible
 
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in letsencrypt-auto-source/pieces/pipstrap.py /opt/certbot/src/
 
 # all above files are necessary for setup.py and venv setup, however,
 # package source code directory has to be copied separately to a

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.rst
-include CHANGES.rst
+include CHANGELOG.md
 include CONTRIBUTING.md
 include LICENSE.txt
 include linter_plugin.py

README.rst

@@ -6,7 +6,7 @@ Anyone who has gone through the trouble of setting up a secure website knows wha
 
 How you use Certbot depends on the configuration of your web server. The best way to get started is to use our `interactive guide <https://certbot.eff.org>`_. It generates instructions based on your configuration settings. In most cases, you’ll need `root or administrator access <https://certbot.eff.org/faq/#does-certbot-require-root-administrator-privileges>`_ to your web server to run Certbot.
 
-If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
+Certbot is meant to be run directly on your web server, not on your personal computer. If you’re using a hosted service and don’t have direct access to your web server, you might not be able to use Certbot. Check with your hosting provider for documentation about uploading certificates or using certificates issued by Let’s Encrypt.
 
 Certbot is a fully-featured, extensible client for the Let's
 Encrypt CA (or any other CA that speaks the `ACME
@@ -91,8 +91,6 @@ Main Website: https://certbot.eff.org
 
 Let's Encrypt Website: https://letsencrypt.org
 
-IRC Channel: #letsencrypt on `Freenode`_
-
 Community: https://community.letsencrypt.org
 
 ACME spec: http://ietf-wg-acme.github.io/acme/
@@ -101,14 +99,12 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme
 
 |build-status| |coverage| |docs| |container|
 
-.. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt
-
 .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master
    :target: https://travis-ci.org/certbot/certbot
    :alt: Travis CI status
 
-.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master
-   :target: https://coveralls.io/r/certbot/certbot
+.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/certbot/certbot
    :alt: Coverage status
 
 .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/

acme/acme/client.py

@@ -577,16 +577,33 @@ class ClientV2(ClientBase):
 
         :param .NewRegistration new_account:
 
+        :raises .ConflictError: in case the account already exists
+
         :returns: Registration Resource.
         :rtype: `.RegistrationResource`
         """
         response = self._post(self.directory['newAccount'], new_account)
+        # if account already exists
+        if response.status_code == 200 and 'Location' in response.headers:
+            raise errors.ConflictError(response.headers.get('Location'))
         # "Instance of 'Field' has no key/contact member" bug:
         # pylint: disable=no-member
         regr = self._regr_from_response(response)
         self.net.account = regr
         return regr
 
+    def query_registration(self, regr):
+        """Query server about registration.
+
+        :param messages.RegistrationResource: Existing Registration
+            Resource.
+
+        """
+        self.net.account = regr
+        updated_regr = super(ClientV2, self).query_registration(regr)
+        self.net.account = updated_regr
+        return updated_regr
+
     def update_registration(self, regr, update=None):
         """Update registration.
 

acme/acme/client_test.py

@@ -134,6 +134,12 @@ class BackwardsCompatibleClientV2Test(ClientTestBase):
         client = self._init()
         self.assertEqual(client.acme_version, 2)
 
+    def test_query_registration_client_v2(self):
+        self.response.json.return_value = DIRECTORY_V2.to_json()
+        client = self._init()
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, client.query_registration(self.regr))
+
     def test_forwarding(self):
         self.response.json.return_value = DIRECTORY_V1.to_json()
         client = self._init()
@@ -706,6 +712,11 @@ class ClientV2Test(ClientTestBase):
 
         self.assertEqual(self.regr, self.client.new_account(self.new_reg))
 
+    def test_new_account_conflict(self):
+        self.response.status_code = http_client.OK
+        self.response.headers['Location'] = self.regr.uri
+        self.assertRaises(errors.ConflictError, self.client.new_account, self.new_reg)
+
     def test_new_order(self):
         order_response = copy.deepcopy(self.response)
         order_response.status_code = http_client.CREATED
@@ -1027,8 +1038,8 @@ class ClientNetworkTest(unittest.TestCase):
 
         # Requests Library Exceptions
         except requests.exceptions.ConnectionError as z: #pragma: no cover
-            self.assertEqual("('Connection aborted.', "
-                             "error(111, 'Connection refused'))", str(z))
+            self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))"
+                              == str(z) or "[WinError 10061]" in str(z))
 
 class ClientNetworkWithMockedResponseTest(unittest.TestCase):
     """Tests for acme.client.ClientNetwork which mock out response."""

acme/acme/crypto_util.py

@@ -136,22 +136,16 @@ def probe_sni(name, host, port=443, timeout=300,
 
     socket_kwargs = {'source_address': source_address}
 
-    host_protocol_agnostic = host
-    if host == '::' or host == '0':
-        # https://github.com/python/typeshed/pull/2136
-        # while PR is not merged, we need to ignore
-        host_protocol_agnostic = None
-
     try:
         # pylint: disable=star-args
         logger.debug(
-            "Attempting to connect to %s:%d%s.", host_protocol_agnostic, port,
+            "Attempting to connect to %s:%d%s.", host, port,
             " from {0}:{1}".format(
                 source_address[0],
                 source_address[1]
             ) if socket_kwargs else ""
         )
-        socket_tuple = (host_protocol_agnostic, port)  # type: Tuple[Optional[str], int]
+        socket_tuple = (host, port)  # type: Tuple[str, int]
         sock = socket.create_connection(socket_tuple, **socket_kwargs)  # type: ignore
     except socket.error as error:
         raise errors.Error(error)

acme/acme/errors.py

@@ -110,6 +110,8 @@ class ConflictError(ClientError):
 
     In the version of ACME implemented by Boulder, this is used to find an
     account if you only have the private key, but don't know the account URL.
+
+    Also used in V2 of the ACME client for the same purpose.
     """
     def __init__(self, location):
         self.location = location

acme/acme/messages.py

@@ -523,7 +523,7 @@ class Order(ResourceBody):
     """
     identifiers = jose.Field('identifiers', omitempty=True)
     status = jose.Field('status', decoder=Status.from_json,
-                        omitempty=True, default=STATUS_PENDING)
+                        omitempty=True)
     authorizations = jose.Field('authorizations', omitempty=True)
     certificate = jose.Field('certificate', omitempty=True)
     finalize = jose.Field('finalize', omitempty=True)
@@ -553,4 +553,3 @@ class OrderResource(ResourceWithURI):
 class NewOrder(Order):
     """New order."""
     resource_type = 'new-order'
-    resource = fields.Resource(resource_type)

acme/acme/messages_test.py

@@ -424,6 +424,19 @@ class OrderResourceTest(unittest.TestCase):
             'authorizations': None,
         })
 
+class NewOrderTest(unittest.TestCase):
+    """Tests for acme.messages.NewOrder."""
+
+    def setUp(self):
+        from acme.messages import NewOrder
+        self.reg = NewOrder(
+            identifiers=mock.sentinel.identifiers)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.reg.to_json(), {
+            'identifiers': mock.sentinel.identifiers,
+        })
+
 
 if __name__ == '__main__':
     unittest.main()  # pragma: no cover

acme/acme/standalone_test.py

@@ -48,7 +48,7 @@ class TLSSNI01ServerTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01Server
-        self.server = TLSSNI01Server(("", 0), certs=self.certs)
+        self.server = TLSSNI01Server(('localhost', 0), certs=self.certs)
         # pylint: disable=no-member
         self.thread = threading.Thread(target=self.server.serve_forever)
         self.thread.start()
@@ -133,8 +133,11 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
                 self.address_family = socket.AF_INET
             socketserver.TCPServer.__init__(self, *args, **kwargs)
             if ipv6:
+                # NB: On Windows, socket.IPPROTO_IPV6 constant may be missing.
+                # We use the corresponding value (41) instead.
+                level = getattr(socket, "IPPROTO_IPV6", 41)
                 # pylint: disable=no-member
-                self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+                self.socket.setsockopt(level, socket.IPV6_V6ONLY, 1)
                 try:
                     self.server_bind()
                     self.server_activate()
@@ -147,15 +150,15 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
         mock_bind.side_effect = socket.error
         from acme.standalone import BaseDualNetworkedServers
         self.assertRaises(socket.error, BaseDualNetworkedServers,
-            BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
-            socketserver.BaseRequestHandler)
+                          BaseDualNetworkedServersTest.SingleProtocolServer,
+                          ('', 0),
+                          socketserver.BaseRequestHandler)
 
     def test_ports_equal(self):
         from acme.standalone import BaseDualNetworkedServers
         servers = BaseDualNetworkedServers(
             BaseDualNetworkedServersTest.SingleProtocolServer,
-            ("", 0),
+            ('', 0),
             socketserver.BaseRequestHandler)
         socknames = servers.getsocknames()
         prev_port = None
@@ -177,7 +180,7 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
             test_util.load_cert('rsa2048_cert.pem'),
         )}
         from acme.standalone import TLSSNI01DualNetworkedServers
-        self.servers = TLSSNI01DualNetworkedServers(("", 0), certs=self.certs)
+        self.servers = TLSSNI01DualNetworkedServers(('localhost', 0), certs=self.certs)
         self.servers.serve_forever()
 
     def tearDown(self):
@@ -245,6 +248,7 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
         self.assertFalse(self._test_http01(add=False))
 
 
+@test_util.broken_on_windows
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 

acme/acme/test_util.py

@@ -4,6 +4,7 @@
 
 """
 import os
+import sys
 import pkg_resources
 import unittest
 
@@ -94,3 +95,11 @@ def skip_unless(condition, reason):  # pragma: no cover
         return lambda cls: cls
     else:
         return lambda cls: None
+
+def broken_on_windows(function):
+    """Decorator to skip temporarily a broken test on Windows."""
+    reason = 'Test is broken and ignored on windows but should be fixed.'
+    return unittest.skipIf(
+        sys.platform == 'win32'
+        and os.environ.get('SKIP_BROKEN_TESTS_ON_WINDOWS', 'true') == 'true',
+        reason)(function)

acme/setup.py

@@ -3,7 +3,7 @@ from setuptools import find_packages
 from setuptools.command.test import test as TestCommand
 import sys
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

appveyor.yml

@@ -0,0 +1,29 @@
+image:
+  # => Windows Server 2012 R2
+  - Visual Studio 2015
+  # => Windows Server 2016
+  - Visual Studio 2017
+
+branches:
+  only:
+    - master
+    - /^\d+\.\d+\.x$/ # Version branches like X.X.X
+    - /^test-.*$/
+
+install:
+  # Use Python 3.7 by default
+  - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%"
+  # Check env
+  - "python --version"
+  # Upgrade pip to avoid warnings
+  - "python -m pip install --upgrade pip"
+  # Ready to install tox and coverage
+  - "pip install tox codecov"
+
+build: off
+
+test_script:
+  - tox -c tox-win.ini -e py34,py35,py36,py37-cover
+
+on_success:
+  - codecov

certbot-apache/certbot_apache/configurator.py

@@ -1,5 +1,6 @@
 """Apache Configuration based off of Augeas Configurator."""
 # pylint: disable=too-many-lines
+import copy
 import fnmatch
 import logging
 import os
@@ -97,48 +98,72 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         vhost_root="/etc/apache2/sites-available",
         vhost_files="*",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "options-ssl-apache.conf")
     )
 
-    def constant(self, key):
-        """Get constant for OS_DEFAULTS"""
-        return self.OS_DEFAULTS.get(key)
+    def option(self, key):
+        """Get a value from options"""
+        return self.options.get(key)
+
+    def _prepare_options(self):
+        """
+        Set the values possibly changed by command line parameters to
+        OS_DEFAULTS constant dictionary
+        """
+        opts = ["enmod", "dismod", "le_vhost_ext", "server_root", "vhost_root",
+                "logs_root", "challenge_location", "handle_modules", "handle_sites",
+                "ctl"]
+        for o in opts:
+            # Config options use dashes instead of underscores
+            if self.conf(o.replace("_", "-")) is not None:
+                self.options[o] = self.conf(o.replace("_", "-"))
+            else:
+                self.options[o] = self.OS_DEFAULTS[o]
+
+        # Special cases
+        self.options["version_cmd"][0] = self.option("ctl")
+        self.options["restart_cmd"][0] = self.option("ctl")
+        self.options["conftest_cmd"][0] = self.option("ctl")
 
     @classmethod
     def add_parser_arguments(cls, add):
+        # When adding, modifying or deleting command line arguments, be sure to
+        # include the changes in the list used in method _prepare_options() to
+        # ensure consistent behavior.
         add("enmod", default=cls.OS_DEFAULTS["enmod"],
-            help="Path to the Apache 'a2enmod' binary.")
+            help="Path to the Apache 'a2enmod' binary")
         add("dismod", default=cls.OS_DEFAULTS["dismod"],
-            help="Path to the Apache 'a2dismod' binary.")
+            help="Path to the Apache 'a2dismod' binary")
         add("le-vhost-ext", default=cls.OS_DEFAULTS["le_vhost_ext"],
-            help="SSL vhost configuration extension.")
+            help="SSL vhost configuration extension")
         add("server-root", default=cls.OS_DEFAULTS["server_root"],
-            help="Apache server root directory.")
+            help="Apache server root directory")
         add("vhost-root", default=None,
             help="Apache server VirtualHost configuration root")
         add("logs-root", default=cls.OS_DEFAULTS["logs_root"],
             help="Apache server logs directory")
         add("challenge-location",
             default=cls.OS_DEFAULTS["challenge_location"],
-            help="Directory path for challenge configuration.")
-        add("handle-modules", default=cls.OS_DEFAULTS["handle_mods"],
-            help="Let installer handle enabling required modules for you. " +
+            help="Directory path for challenge configuration")
+        add("handle-modules", default=cls.OS_DEFAULTS["handle_modules"],
+            help="Let installer handle enabling required modules for you " +
                  "(Only Ubuntu/Debian currently)")
         add("handle-sites", default=cls.OS_DEFAULTS["handle_sites"],
-            help="Let installer handle enabling sites for you. " +
+            help="Let installer handle enabling sites for you " +
                  "(Only Ubuntu/Debian currently)")
-        util.add_deprecated_argument(add, argument_name="ctl", nargs=1)
+        add("ctl", default=cls.OS_DEFAULTS["ctl"],
+            help="Full path to Apache control script")
         util.add_deprecated_argument(
             add, argument_name="init-script", nargs=1)
 
@@ -169,7 +194,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         self.parser = None
         self.version = version
         self.vhosts = None
-        self.vhostroot = None
+        self.options = copy.deepcopy(self.OS_DEFAULTS)
         self._enhance_func = {"redirect": self._enable_redirect,
                               "ensure-http-header": self._set_http_header,
                               "staple-ocsp": self._enable_ocsp_stapling}
@@ -201,12 +226,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         except ImportError:
             raise errors.NoInstallationError("Problem in Augeas installation")
 
+        self._prepare_options()
+
         # Verify Apache is installed
-        restart_cmd = self.constant("restart_cmd")[0]
-        if not util.exe_exists(restart_cmd):
-            if not path_surgery(restart_cmd):
-                raise errors.NoInstallationError(
-                    'Cannot find Apache control command {0}'.format(restart_cmd))
+        self._verify_exe_availability(self.option("ctl"))
 
         # Make sure configuration is valid
         self.config_test()
@@ -226,12 +249,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        # Parse vhost-root if defined on cli
-        if not self.conf("vhost-root"):
-            self.vhostroot = self.constant("vhost_root")
-        else:
-            self.vhostroot = os.path.abspath(self.conf("vhost-root"))
-
         self.parser = self.get_parser()
 
         # Check for errors in parsing files with Augeas
@@ -245,13 +262,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         # Prevent two Apache plugins from modifying a config at once
         try:
-            util.lock_dir_until_exit(self.conf("server-root"))
+            util.lock_dir_until_exit(self.option("server_root"))
         except (OSError, errors.LockError):
             logger.debug("Encountered error:", exc_info=True)
             raise errors.PluginError(
-                "Unable to lock %s", self.conf("server-root"))
+                "Unable to lock %s", self.option("server_root"))
         self._prepared = True
 
+    def _verify_exe_availability(self, exe):
+        """Checks availability of Apache executable"""
+        if not util.exe_exists(exe):
+            if not path_surgery(exe):
+                raise errors.NoInstallationError(
+                    'Cannot find Apache executable {0}'.format(exe))
+
     def _check_aug_version(self):
         """ Checks that we have recent enough version of libaugeas.
         If augeas version is recent enough, it will support case insensitive
@@ -269,8 +293,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
     def get_parser(self):
         """Initializes the ApacheParser"""
+        # If user provided vhost_root value in command line, use it
         return parser.ApacheParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.conf("vhost-root"),
             self.version, configurator=self)
 
     def _wildcard_domain(self, domain):
@@ -1037,7 +1062,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :param boolean temp: If the change is temporary
         """
 
-        if self.conf("handle-modules"):
+        if self.option("handle_modules"):
             if self.version >= (2, 4) and ("socache_shmcb_module" not in
                                            self.parser.modules):
                 self.enable_mod("socache_shmcb", temp=temp)
@@ -1066,7 +1091,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         Duplicates vhost and adds default ssl options
         New vhost will reside as (nonssl_vhost.path) +
-        ``self.constant("le_vhost_ext")``
+        ``self.option("le_vhost_ext")``
 
         .. note:: This function saves the configuration
 
@@ -1165,18 +1190,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
 
         if self.conf("vhost-root") and os.path.exists(self.conf("vhost-root")):
-            # Defined by user on CLI
-
-            fp = os.path.join(os.path.realpath(self.vhostroot),
+            fp = os.path.join(os.path.realpath(self.option("vhost_root")),
                               os.path.basename(non_ssl_vh_fp))
         else:
             # Use non-ssl filepath
             fp = os.path.realpath(non_ssl_vh_fp)
 
         if fp.endswith(".conf"):
-            return fp[:-(len(".conf"))] + self.conf("le_vhost_ext")
+            return fp[:-(len(".conf"))] + self.option("le_vhost_ext")
         else:
-            return fp + self.conf("le_vhost_ext")
+            return fp + self.option("le_vhost_ext")
 
     def _sift_rewrite_rule(self, line):
         """Decides whether a line should be copied to a SSL vhost.
@@ -2025,7 +2048,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
                             addr in self._get_proposed_addrs(ssl_vhost)),
                    servername, serveralias,
                    " ".join(rewrite_rule_args),
-                   self.conf("logs-root")))
+                   self.option("logs_root")))
 
     def _write_out_redirect(self, ssl_vhost, text):
         # This is the default name
@@ -2037,7 +2060,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
             if len(ssl_vhost.name) < (255 - (len(redirect_filename) + 1)):
                 redirect_filename = "le-redirect-%s.conf" % ssl_vhost.name
 
-        redirect_filepath = os.path.join(self.vhostroot,
+        redirect_filepath = os.path.join(self.option("vhost_root"),
                                          redirect_filename)
 
         # Register the new file that will be created
@@ -2158,18 +2181,18 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         """
         error = ""
         try:
-            util.run_script(self.constant("restart_cmd"))
+            util.run_script(self.option("restart_cmd"))
         except errors.SubprocessError as err:
             logger.info("Unable to restart apache using %s",
-                        self.constant("restart_cmd"))
-            alt_restart = self.constant("restart_cmd_alt")
+                        self.option("restart_cmd"))
+            alt_restart = self.option("restart_cmd_alt")
             if alt_restart:
                 logger.debug("Trying alternative restart command: %s",
                              alt_restart)
                 # There is an alternative restart command available
                 # This usually is "restart" verb while original is "graceful"
                 try:
-                    util.run_script(self.constant(
+                    util.run_script(self.option(
                         "restart_cmd_alt"))
                     return
                 except errors.SubprocessError as secerr:
@@ -2185,7 +2208,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
         try:
-            util.run_script(self.constant("conftest_cmd"))
+            util.run_script(self.option("conftest_cmd"))
         except errors.SubprocessError as err:
             raise errors.MisconfigurationError(str(err))
 
@@ -2201,11 +2224,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
 
         """
         try:
-            stdout, _ = util.run_script(self.constant("version_cmd"))
+            stdout, _ = util.run_script(self.option("version_cmd"))
         except errors.SubprocessError:
             raise errors.PluginError(
                 "Unable to run %s -v" %
-                self.constant("version_cmd"))
+                self.option("version_cmd"))
 
         regex = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
         matches = regex.findall(stdout)
@@ -2295,7 +2318,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         # certbot for unprivileged users via setuid), this function will need
         # to be modified.
         return common.install_version_controlled_file(options_ssl, options_ssl_digest,
-            self.constant("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
+            self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES)
 
     def enable_autohsts(self, _unused_lineage, domains):
         """

certbot-apache/certbot_apache/display_ops.py

@@ -113,8 +113,7 @@ def _vhost_menu(domain, vhosts):
         code, tag = zope.component.getUtility(interfaces.IDisplay).menu(
             "We were unable to find a vhost with a ServerName "
             "or Address of {0}.{1}Which virtual host would you "
-            "like to choose?\n(note: conf files with multiple "
-            "vhosts are not yet supported)".format(domain, os.linesep),
+            "like to choose?".format(domain, os.linesep),
             choices, force_interactive=True)
     except errors.MissingCommandlineFlag:
         msg = (

certbot-apache/certbot_apache/http_01.py

@@ -6,6 +6,7 @@ from acme.magic_typing import Set  # pylint: disable=unused-import, no-name-in-m
 from certbot import errors
 from certbot.plugins import common
 from certbot_apache.obj import VirtualHost  # pylint: disable=unused-import
+from certbot_apache.parser import get_aug_path
 
 logger = logging.getLogger(__name__)
 
@@ -172,4 +173,9 @@ class ApacheHttp01(common.TLSSNI01):
             self.configurator.parser.add_dir(
                 vhost.path, "Include", self.challenge_conf_post)
 
+            if not vhost.enabled:
+                self.configurator.parser.add_dir(
+                    get_aug_path(self.configurator.parser.loc["default"]),
+                    "Include", vhost.filep)
+
             self.moded_vhosts.add(vhost)

certbot-apache/certbot_apache/override_arch.py

@@ -16,14 +16,14 @@ class ArchConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_centos.py

@@ -18,25 +18,33 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/httpd/conf.d",
         vhost_files="*.conf",
         logs_root="/var/log/httpd",
+        ctl="apachectl",
         version_cmd=['apachectl', '-v'],
-        apache_cmd="apachectl",
         restart_cmd=['apachectl', 'graceful'],
         restart_cmd_alt=['apachectl', 'restart'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/httpd/conf.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "centos-options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in CentOS.
+        """
+        super(CentOSConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return CentOSParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 

certbot-apache/certbot_apache/override_darwin.py

@@ -16,14 +16,14 @@ class DarwinConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/other",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/httpd', '-v'],
-        apache_cmd="/usr/sbin/httpd",
+        ctl="apachectl",
+        version_cmd=['apachectl', '-v'],
         restart_cmd=['apachectl', 'graceful'],
         conftest_cmd=['apachectl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/other",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/override_debian.py

@@ -23,14 +23,14 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/sites-available",
         vhost_files="*",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=True,
+        handle_modules=True,
         handle_sites=True,
         challenge_location="/etc/apache2",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
@@ -134,11 +134,11 @@ class DebianConfigurator(configurator.ApacheConfigurator):
         # Generate reversal command.
         # Try to be safe here... check that we can probably reverse before
         # applying enmod command
-        if not util.exe_exists(self.conf("dismod")):
+        if not util.exe_exists(self.option("dismod")):
             raise errors.MisconfigurationError(
                 "Unable to find a2dismod, please make sure a2enmod and "
                 "a2dismod are configured correctly for certbot.")
 
         self.reverter.register_undo_command(
-            temp, [self.conf("dismod"), "-f", mod_name])
-        util.run_script([self.conf("enmod"), mod_name])
+            temp, [self.option("dismod"), "-f", mod_name])
+        util.run_script([self.option("enmod"), mod_name])

certbot-apache/certbot_apache/override_gentoo.py

@@ -18,25 +18,33 @@ class GentooConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
-        version_cmd=['/usr/sbin/apache2', '-v'],
-        apache_cmd="apache2ctl",
+        ctl="apache2ctl",
+        version_cmd=['apache2ctl', '-v'],
         restart_cmd=['apache2ctl', 'graceful'],
         restart_cmd_alt=['apache2ctl', 'restart'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod=None,
         dismod=None,
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(
             "certbot_apache", "options-ssl-apache.conf")
     )
 
+    def _prepare_options(self):
+        """
+        Override the options dictionary initialization in order to support
+        alternative restart cmd used in Gentoo.
+        """
+        super(GentooConfigurator, self)._prepare_options()
+        self.options["restart_cmd_alt"][0] = self.option("ctl")
+
     def get_parser(self):
         """Initializes the ApacheParser"""
         return GentooParser(
-            self.aug, self.conf("server-root"), self.conf("vhost-root"),
+            self.aug, self.option("server_root"), self.option("vhost_root"),
             self.version, configurator=self)
 
 
@@ -61,7 +69,7 @@ class GentooParser(parser.ApacheParser):
 
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
-        mod_cmd = [self.configurator.constant("apache_cmd"), "modules"]
+        mod_cmd = [self.configurator.option("ctl"), "modules"]
         matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:
             self.add_mod(mod.strip())

certbot-apache/certbot_apache/override_suse.py

@@ -16,14 +16,14 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator):
         vhost_root="/etc/apache2/vhosts.d",
         vhost_files="*.conf",
         logs_root="/var/log/apache2",
+        ctl="apache2ctl",
         version_cmd=['apache2ctl', '-v'],
-        apache_cmd="apache2ctl",
         restart_cmd=['apache2ctl', 'graceful'],
         conftest_cmd=['apache2ctl', 'configtest'],
         enmod="a2enmod",
         dismod="a2dismod",
         le_vhost_ext="-le-ssl.conf",
-        handle_mods=False,
+        handle_modules=False,
         handle_sites=False,
         challenge_location="/etc/apache2/vhosts.d",
         MOD_SSL_CONF_SRC=pkg_resources.resource_filename(

certbot-apache/certbot_apache/parser.py

@@ -69,7 +69,7 @@ class ApacheParser(object):
         # Must also attempt to parse additional virtual host root
         if vhostroot:
             self.parse_file(os.path.abspath(vhostroot) + "/" +
-                            self.configurator.constant("vhost_files"))
+                            self.configurator.option("vhost_files"))
 
         # check to see if there were unparsed define statements
         if version < (2, 4):
@@ -152,7 +152,7 @@ class ApacheParser(object):
         """Get Defines from httpd process"""
 
         variables = dict()
-        define_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        define_cmd = [self.configurator.option("ctl"), "-t", "-D",
                       "DUMP_RUN_CFG"]
         matches = self.parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)")
         try:
@@ -179,7 +179,7 @@ class ApacheParser(object):
         # configuration files
         _ = self.find_dir("Include")
 
-        inc_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        inc_cmd = [self.configurator.option("ctl"), "-t", "-D",
                    "DUMP_INCLUDES"]
         matches = self.parse_from_subprocess(inc_cmd, r"\(.*\) (.*)")
         if matches:
@@ -190,7 +190,7 @@ class ApacheParser(object):
     def update_modules(self):
         """Get loaded modules from httpd process, and add them to DOM"""
 
-        mod_cmd = [self.configurator.constant("apache_cmd"), "-t", "-D",
+        mod_cmd = [self.configurator.option("ctl"), "-t", "-D",
                        "DUMP_MODULES"]
         matches = self.parse_from_subprocess(mod_cmd, r"(.*)_module")
         for mod in matches:

certbot-apache/certbot_apache/tests/autohsts_test.py

@@ -119,6 +119,9 @@ class AutoHSTSTest(util.ApacheTest):
             cur_val = maxage.format(constants.AUTOHSTS_STEPS[i+1])
             self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path),
                               cur_val)
+        # Ensure that the value is raised to max
+        self.assertEquals(self.get_autohsts_value(self.vh_truth[7].path),
+                          maxage.format(constants.AUTOHSTS_STEPS[-1]))
         # Make permanent
         self.config.deploy_autohsts(mock_lineage)
         self.assertEqual(self.get_autohsts_value(self.vh_truth[7].path),

certbot-apache/certbot_apache/tests/centos_test.py

@@ -135,5 +135,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest):
                                        errors.SubprocessError,
                                        errors.SubprocessError]
         self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -115,9 +115,22 @@ class MultipleVhostsTest(util.ApacheTest):
         # Weak test..
         ApacheConfigurator.add_parser_arguments(mock.MagicMock())
 
+    def test_add_parser_arguments_all_configurators(self):  # pylint: disable=no-self-use
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        for cls in OVERRIDE_CLASSES.values():
+            cls.add_parser_arguments(mock.MagicMock())
+
+    def test_all_configurators_defaults_defined(self):
+        from certbot_apache.entrypoint import OVERRIDE_CLASSES
+        from certbot_apache.configurator import ApacheConfigurator
+        parameters = set(ApacheConfigurator.OS_DEFAULTS.keys())
+        for cls in OVERRIDE_CLASSES.values():
+            self.assertTrue(parameters.issubset(set(cls.OS_DEFAULTS.keys())))
+
     def test_constant(self):
-        self.assertEqual(self.config.constant("server_root"), "/etc/apache2")
-        self.assertEqual(self.config.constant("nonexistent"), None)
+        self.assertTrue("debian_apache_2_4/multiple_vhosts/apache" in
+                        self.config.option("server_root"))
+        self.assertEqual(self.config.option("nonexistent"), None)
 
     @certbot_util.patch_get_utility()
     def test_get_all_names(self, mock_getutility):
@@ -651,22 +664,10 @@ class MultipleVhostsTest(util.ApacheTest):
         self.assertEqual(ssl_vhost_slink.name, "nonsym.link")
 
     def test_make_vhost_ssl_nonexistent_vhost_path(self):
-        def conf_side_effect(arg):
-            """ Mock function for ApacheConfigurator.conf """
-            confvars = {
-                "vhost-root": "/tmp/nonexistent",
-                "le_vhost_ext": "-le-ssl.conf",
-                "handle-sites": True}
-            return confvars[arg]
-
-        with mock.patch(
-                "certbot_apache.configurator.ApacheConfigurator.conf"
-        ) as mock_conf:
-            mock_conf.side_effect = conf_side_effect
-            ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
-            self.assertEqual(os.path.dirname(ssl_vhost.filep),
-                             os.path.dirname(os.path.realpath(
-                                 self.vh_truth[1].filep)))
+        ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1])
+        self.assertEqual(os.path.dirname(ssl_vhost.filep),
+                            os.path.dirname(os.path.realpath(
+                                self.vh_truth[1].filep)))
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -1583,7 +1584,7 @@ class AugeasVhostsTest(util.ApacheTest):
                           broken_vhost)
 
 class MultiVhostsTest(util.ApacheTest):
-    """Test vhosts with illegal names dependent on augeas version."""
+    """Test configuration with multiple virtualhosts in a single file."""
     # pylint: disable=protected-access
 
     def setUp(self):  # pylint: disable=arguments-differ
@@ -1703,7 +1704,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                                              self.config.updated_mod_ssl_conf_digest)
 
     def _current_ssl_options_hash(self):
-        return crypto_util.sha256sum(self.config.constant("MOD_SSL_CONF_SRC"))
+        return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC"))
 
     def _assert_current_file(self):
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
@@ -1739,7 +1740,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
             self.assertFalse(mock_logger.warning.called)
         self.assertTrue(os.path.isfile(self.config.mod_ssl_conf))
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf),
             self._current_ssl_options_hash())
@@ -1755,7 +1756,7 @@ class InstallSslOptionsConfTest(util.ApacheTest):
                 "%s has been manually modified; updated file "
                 "saved to %s. We recommend updating %s for security purposes.")
         self.assertEqual(crypto_util.sha256sum(
-            self.config.constant("MOD_SSL_CONF_SRC")),
+            self.config.option("MOD_SSL_CONF_SRC")),
             self._current_ssl_options_hash())
         # only print warning once
         with mock.patch("certbot.plugins.common.logger") as mock_logger:

certbot-apache/certbot_apache/tests/debian_test.py

@@ -20,7 +20,7 @@ class MultipleVhostsTestDebian(util.ApacheTest):
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTestDebian, self).setUp()
         self.config = util.get_apache_configurator(
-            self.config_path, None, self.config_dir, self.work_dir,
+            self.config_path, self.vhost_path, self.config_dir, self.work_dir,
             os_info="debian")
         self.config = self.mock_deploy_cert(self.config)
         self.vh_truth = util.get_vh_truth(self.temp_dir,

certbot-apache/certbot_apache/tests/gentoo_test.py

@@ -117,7 +117,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest):
         self.config.parser.modules = set()
 
         with mock.patch("certbot.util.get_os_info") as mock_osi:
-            # Make sure we have the have the CentOS httpd constants
+            # Make sure we have the have the Gentoo httpd constants
             mock_osi.return_value = ("gentoo", "123")
             self.config.parser.update_runtime_variables()
 

certbot-apache/certbot_apache/tests/http_01_test.py

@@ -10,6 +10,7 @@ from certbot import achallenges
 from certbot import errors
 
 from certbot.tests import acme_util
+from certbot_apache.parser import get_aug_path
 from certbot_apache.tests import util
 
 
@@ -134,6 +135,21 @@ class ApacheHttp01Test(util.ApacheTest):
     def test_perform_3_achall_apache_2_4(self):
         self.combinations_perform_test(num_achalls=3, minor_version=4)
 
+    def test_activate_disabled_vhost(self):
+        vhosts = [v for v in self.config.vhosts if v.name == "certbot.demo"]
+        achalls = [
+            achallenges.KeyAuthorizationAnnotatedChallenge(
+                challb=acme_util.chall_to_challb(
+                    challenges.HTTP01(token=((b'a' * 16))),
+                    "pending"),
+                domain="certbot.demo", account_key=self.account_key)]
+        vhosts[0].enabled = False
+        self.common_perform_test(achalls, vhosts)
+        matches = self.config.parser.find_dir(
+            "Include", vhosts[0].filep,
+            get_aug_path(self.config.parser.loc["default"]))
+        self.assertEqual(len(matches), 1)
+
     def combinations_perform_test(self, num_achalls, minor_version):
         """Test perform with the given achall count and Apache version."""
         achalls = self.achalls[:num_achalls]

certbot-apache/certbot_apache/tests/parser_test.py

@@ -282,11 +282,11 @@ class BasicParserTest(util.ParserTest):
         self.assertRaises(
             errors.PluginError, self.parser.update_runtime_variables)
 
-    @mock.patch("certbot_apache.configurator.ApacheConfigurator.constant")
+    @mock.patch("certbot_apache.configurator.ApacheConfigurator.option")
     @mock.patch("certbot_apache.parser.subprocess.Popen")
-    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_const):
+    def test_update_runtime_vars_bad_ctl(self, mock_popen, mock_opt):
         mock_popen.side_effect = OSError
-        mock_const.return_value = "nonexistent"
+        mock_opt.return_value = "nonexistent"
         self.assertRaises(
             errors.MisconfigurationError,
             self.parser.update_runtime_variables)

certbot-apache/certbot_apache/tests/util.py

@@ -97,9 +97,10 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
     backups = os.path.join(work_dir, "backups")
     mock_le_config = mock.MagicMock(
         apache_server_root=config_path,
-        apache_vhost_root=conf_vhost_path,
+        apache_vhost_root=None,
         apache_le_vhost_ext="-le-ssl.conf",
         apache_challenge_location=config_path,
+        apache_enmod=None,
         backup_dir=backups,
         config_dir=config_dir,
         http01_port=80,
@@ -107,33 +108,25 @@ def get_apache_configurator(  # pylint: disable=too-many-arguments, too-many-loc
         in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
         work_dir=work_dir)
 
-    orig_os_constant = configurator.ApacheConfigurator(mock_le_config,
-                                                       name="apache",
-                                                       version=version).constant
-
-    def mock_os_constant(key, vhost_path=vhost_path):
-        """Mock default vhost path"""
-        if key == "vhost_root":
-            return vhost_path
-        else:
-            return orig_os_constant(key)
-
-    with mock.patch("certbot_apache.configurator.ApacheConfigurator.constant") as mock_cons:
-        mock_cons.side_effect = mock_os_constant
-        with mock.patch("certbot_apache.configurator.util.run_script"):
-            with mock.patch("certbot_apache.configurator.util."
-                            "exe_exists") as mock_exe_exists:
-                mock_exe_exists.return_value = True
-                with mock.patch("certbot_apache.parser.ApacheParser."
-                                "update_runtime_variables"):
-                    try:
-                        config_class = entrypoint.OVERRIDE_CLASSES[os_info]
-                    except KeyError:
-                        config_class = configurator.ApacheConfigurator
-                    config = config_class(config=mock_le_config, name="apache",
-                        version=version)
-
-                    config.prepare()
+    with mock.patch("certbot_apache.configurator.util.run_script"):
+        with mock.patch("certbot_apache.configurator.util."
+                        "exe_exists") as mock_exe_exists:
+            mock_exe_exists.return_value = True
+            with mock.patch("certbot_apache.parser.ApacheParser."
+                            "update_runtime_variables"):
+                try:
+                    config_class = entrypoint.OVERRIDE_CLASSES[os_info]
+                except KeyError:
+                    config_class = configurator.ApacheConfigurator
+                config = config_class(config=mock_le_config, name="apache",
+                    version=version)
+                if not conf_vhost_path:
+                    config_class.OS_DEFAULTS["vhost_root"] = vhost_path
+                else:
+                    # Custom virtualhost path was requested
+                    config.config.apache_vhost_root = conf_vhost_path
+                config.config.apache_ctl = config_class.OS_DEFAULTS["ctl"]
+                config.prepare()
     return config
 
 

certbot-apache/setup.py

@@ -2,13 +2,13 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.25.0',
-    'certbot>=0.26.0.dev0',
+    'certbot>=0.26.0',
     'mock',
     'python-augeas',
     'setuptools',

certbot-auto

@@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then
 fi
 VENV_BIN="$VENV_PATH/bin"
 BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt"
-LE_AUTO_VERSION="0.25.1"
+LE_AUTO_VERSION="0.27.1"
 BASENAME=$(basename $0)
 USAGE="Usage: $BASENAME [OPTIONS]
 A self-updating wrapper script for the Certbot ACME client. When run, updates
@@ -1060,37 +1060,26 @@ ConfigArgParse==0.12.0 \
 configobj==5.0.6 \
     --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 \
     --no-binary configobj
-cryptography==2.0.2 \
-    --hash=sha256:187ae17358436d2c760f28c2aeb02fefa3f37647a9c5b6f7f7c3e83cd1c5a972 \
-    --hash=sha256:19e43a13bbf52028dd1e810c803f2ad8880d0692d772f98d42e1eaf34bdee3d6 \
-    --hash=sha256:da9291502cbc87dc0284a20c56876e4d2e68deac61cc43df4aec934e44ca97b1 \
-    --hash=sha256:0954f8813095f581669330e0a2d5e726c33ac7f450c1458fac58bab54595e516 \
-    --hash=sha256:d68b0cc40a8432ed3fc84876c519de704d6001800ec22b136e75ae841910c45b \
-    --hash=sha256:2f8ad9580ab4da645cfea52a91d2da99a49a1e76616d8be68441a986fad652b0 \
-    --hash=sha256:cc00b4511294f5f6b65c4e77a1a9c62f52490a63d2c120f3872176b40a82351e \
-    --hash=sha256:cf896020f6a9f095a547b3d672c8db1ef2ed71fca11250731fa1d4a4cb8b1590 \
-    --hash=sha256:e0fdb8322206fa02aa38f71519ff75dce2eb481b7e1110e2936795cb376bb6ee \
-    --hash=sha256:277538466657ca5d6637f80be100242f9831d75138b788d718edd3aab34621f8 \
-    --hash=sha256:2c77eb0560f54ce654ab82d6b2a64327a71ee969b29022bf9746ca311c9f5069 \
-    --hash=sha256:755a7853b679e79d0a799351c092a9b0271f95ff54c8dd8823d8b527a2926a86 \
-    --hash=sha256:77197a2d525e761cdd4c771180b4bd0d80703654c6385e4311cbbbe2beb56fa1 \
-    --hash=sha256:eb8bb79d0ab00c931c8333b745f06fec481a51c52d70acd4ee95d6093ba5c386 \
-    --hash=sha256:131f61de82ef28f3e20beb4bfc24f9692d28cecfd704e20e6c7f070f7793013a \
-    --hash=sha256:ac35435974b2e27cd4520f29c191d7da36f4189aa3264e52c4c6c6d089ab6142 \
-    --hash=sha256:04b6ea99daa2a8460728794213d76d45ad58ea247dc7e7ff148d7dd726e87863 \
-    --hash=sha256:2b9442f8b4c3d575f6cc3db0e856034e0f5a9d55ecd636f52d8c496795b26952 \
-    --hash=sha256:b3d3b3ecba1fe1bdb6f180770a137f877c8f07571f7b2934bb269475bcf0e5e8 \
-    --hash=sha256:670a58c0d75cb0e78e73dd003bd96d4440bbb1f2bc041dcf7b81767ca4fb0ce9 \
-    --hash=sha256:5af84d23bdb86b5e90aca263df1424b43f1748480bfcde3ac2a3cbe622612468 \
-    --hash=sha256:ba22e8eefabdd7aca37d0c0c00d2274000d2cebb5cce9e5a710cb55bf8797b31 \
-    --hash=sha256:b798b22fa7e92b439547323b8b719d217f1e1b7677585cfeeedf3b55c70bb7fb \
-    --hash=sha256:59cff28af8cce96cb7e94a459726e1d88f6f5fa75097f9dcbebd99118d64ea4c \
-    --hash=sha256:fe859e445abc9ba9e97950ddafb904e23234c4ecb76b0fae6c86e80592ce464a \
-    --hash=sha256:655f3c474067f1e277430f23cc0549f0b1dc99b82aec6e53f80b9b2db7f76f11 \
-    --hash=sha256:0ebc2be053c9a03a2f3e20a466e87bf12a51586b3c79bd2a22171b073a805346 \
-    --hash=sha256:01e6e60654df64cca53733cda39446d67100c819c181d403afb120e0d2a71e1b \
-    --hash=sha256:d46f4e5d455cb5563685c52ef212696f0a6cc1ea627603218eabbd8a095291d8 \
-    --hash=sha256:3780b2663ee7ebb37cb83263326e3cd7f8b2ea439c448539d4b87de12c8d06ab
+cryptography==2.2.2 \
+    --hash=sha256:3f3b65d5a16e6b52fba63dc860b62ca9832f51f1a2ae5083c78b6840275f12dd \
+    --hash=sha256:5251e7de0de66810833606439ca65c9b9e45da62196b0c88bfadf27740aac09f \
+    --hash=sha256:551a3abfe0c8c6833df4192a63371aa2ff43afd8f570ed345d31f251d78e7e04 \
+    --hash=sha256:5cb990056b7cadcca26813311187ad751ea644712022a3976443691168781b6f \
+    --hash=sha256:60bda7f12ecb828358be53095fc9c6edda7de8f1ef571f96c00b2363643fa3cd \
+    --hash=sha256:64b5c67acc9a7c83fbb4b69166f3105a0ab722d27934fac2cb26456718eec2ba \
+    --hash=sha256:6fef51ec447fe9f8351894024e94736862900d3a9aa2961528e602eb65c92bdb \
+    --hash=sha256:77d0ad229d47a6e0272d00f6bf8ac06ce14715a9fd02c9a97f5a2869aab3ccb2 \
+    --hash=sha256:808fe471b1a6b777f026f7dc7bd9a4959da4bfab64972f2bbe91e22527c1c037 \
+    --hash=sha256:9b62fb4d18529c84b961efd9187fecbb48e89aa1a0f9f4161c61b7fc42a101bd \
+    --hash=sha256:9e5bed45ec6b4f828866ac6a6bedf08388ffcfa68abe9e94b34bb40977aba531 \
+    --hash=sha256:9fc295bf69130a342e7a19a39d7bbeb15c0bcaabc7382ec33ef3b2b7d18d2f63 \
+    --hash=sha256:abd070b5849ed64e6d349199bef955ee0ad99aefbad792f0c587f8effa681a5e \
+    --hash=sha256:ba6a774749b6e510cffc2fb98535f717e0e5fd91c7c99a61d223293df79ab351 \
+    --hash=sha256:c332118647f084c983c6a3e1dba0f3bcb051f69d12baccac68db8d62d177eb8a \
+    --hash=sha256:d6f46e862ee36df81e6342c2177ba84e70f722d9dc9c6c394f9f1f434c4a5563 \
+    --hash=sha256:db6013746f73bf8edd9c3d1d3f94db635b9422f503db3fc5ef105233d4c011ab \
+    --hash=sha256:f57008eaff597c69cf692c3518f6d4800f0309253bb138b526a37fe9ef0c7471 \
+    --hash=sha256:f6c821ac253c19f2ad4c8691633ae1d1a17f120d5b01ea1d256d7b602bc59887
 enum34==1.1.2 ; python_version < '3.4' \
     --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \
     --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501
@@ -1103,9 +1092,9 @@ idna==2.5 \
 ipaddress==1.0.16 \
     --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \
     --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0
-josepy==1.0.1 \
-    --hash=sha256:354a3513038a38bbcd27c97b7c68a8f3dfaff0a135b20a92c6db4cc4ea72915e \
-    --hash=sha256:9f48b88ca37f0244238b1cc77723989f7c54f7b90b2eee6294390bacfe870acc
+josepy==1.1.0 \
+    --hash=sha256:1309a25aac3caeff5239729c58ff9b583f7d022ffdb1553406ddfc8e5b52b76e \
+    --hash=sha256:fb5c62c77d26e04df29cb5ecd01b9ce69b6fcc9e521eb1ca193b7faa2afa7086
 linecache2==1.0.0 \
     --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \
     --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c
@@ -1208,18 +1197,18 @@ letsencrypt==0.7.0 \
     --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \
     --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9
 
-certbot==0.25.1 \
-    --hash=sha256:01689015364685fef3f1e1fb7832ba84eb3b0aa85bc5a71c96661f6d4c59981f \
-    --hash=sha256:5c23e5186133bb1afd805be5e0cd2fb7b95862a8b0459c9ecad4ae60f933e54e
-acme==0.25.1 \
-    --hash=sha256:26e641a01536705fe5f12d856703b8ef06e5a07981a7b6379d2771dcdb69a742 \
-    --hash=sha256:47b5f3f73d69b7b1d13f918aa2cd75a8093069a68becf4af38e428e4613b2734
-certbot-apache==0.25.1 \
-    --hash=sha256:a28b7c152cc11474bef5b5e7967aaea42b2c0aaf86fd82ee4082713d33cee5a9 \
-    --hash=sha256:ed012465617073a0f1057fe854dc8d1eb6d2dd7ede1fb2eee765129fed2a095a
-certbot-nginx==0.25.1 \
-    --hash=sha256:83f82c3ba08c0b1d4bf449ac24018e8e7dd34a6248d35466f2de7da1cd312e15 \
-    --hash=sha256:68f98b41c54e0bf4218ef293079597176617bee3837ae3aa6528ce2ff0bf4f9c
+certbot==0.27.1 \
+    --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \
+    --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a
+acme==0.27.1 \
+    --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \
+    --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90
+certbot-apache==0.27.1 \
+    --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \
+    --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854
+certbot-nginx==0.27.1 \
+    --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \
+    --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f
 
 UNLIKELY_EOF
     # -------------------------------------------------------------------------

certbot-compatibility-test/Dockerfile

@@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
+COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/
 
 # all above files are necessary for setup.py, however, package source
 # code directory has to be copied separately to a subdirectory...

certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py

@@ -59,9 +59,6 @@ class Proxy(configurators_common.Proxy):
             setattr(self.le_config, "apache_" + k,
                     entrypoint.ENTRYPOINT.OS_DEFAULTS[k])
 
-        # An alias
-        self.le_config.apache_handle_modules = self.le_config.apache_handle_mods
-
         self._configurator = entrypoint.ENTRYPOINT(
             config=configuration.NamespaceConfig(self.le_config),
             name="apache")

certbot-compatibility-test/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 install_requires = [
     'certbot',

certbot-dns-cloudflare/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-cloudxns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-digitalocean/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsimple/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-dnsmadeeasy/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-gehirn/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.25.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-google/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-linode/certbot_dns_linode/__init__.py

@@ -14,7 +14,11 @@ Named Arguments
                                             DNS to propagate before asking the
                                             ACME server to verify the DNS
                                             record.
-                                            (Default: 960)
+                                            (Default: 1200 because Linode
+                                            updates its first DNS every 15
+                                            minutes and we allow 5 more minutes
+                                            for the update to reach the other 5
+                                            servers)
 ==========================================  ===================================
 
 
@@ -23,7 +27,7 @@ Credentials
 
 Use of this plugin requires a configuration file containing Linode API
 credentials, obtained from your Linode account's `Applications & API
-Tokens page <https://cloud.linode.com/settings/api/tokens>`_.
+Tokens page <https://manager.linode.com/profile/api>`_.
 
 .. code-block:: ini
    :name: credentials.ini
@@ -74,13 +78,15 @@ Examples
      -d www.example.com
 
 .. code-block:: bash
-   :caption: To acquire a certificate for ``example.com``, waiting 60 seconds
-             for DNS propagation
+   :caption: To acquire a certificate for ``example.com``, waiting 1000 seconds
+             for DNS propagation (Linode updates its first DNS every 15 minutes
+             and we allow some extra time for the update to reach the other 5
+             servers)
 
    certbot certonly \\
      --dns-linode \\
      --dns-linode-credentials ~/.secrets/certbot/linode.ini \\
-     --dns-linode-propagation-seconds 60 \\
+     --dns-linode-propagation-seconds 1000 \\
      -d example.com
 
 """

certbot-dns-linode/certbot_dns_linode/dns_linode.py

@@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator):
 
     @classmethod
     def add_parser_arguments(cls, add):  # pylint: disable=arguments-differ
-        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=960)
+        super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=1200)
         add('credentials', help='Linode credentials INI file.')
 
     def more_info(self):  # pylint: disable=missing-docstring,no-self-use

certbot-dns-linode/setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-dns-luadns/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-nsone/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-ovh/setup.py

@@ -4,18 +4,16 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.25.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
     'acme>=0.21.1',
     'certbot>=0.21.1',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.7.3', # Correct OVH integration tests
     'mock',
-    # For pkg_resources. >=1.0 so pip resolves it to a version cryptography
-    # will tolerate; see #2599:
-    'setuptools>=1.0',
+    'setuptools',
     'zope.interface',
 ]
 

certbot-dns-rfc2136/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-route53/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.

certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py

@@ -38,7 +38,8 @@ class AuthenticatorTest(test_util.TempDirTestCase,
         self.auth._get_sakuracloud_client = mock.MagicMock(return_value=self.mock_client)
 
 
-class NS1LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest):
+class SakuraCloudLexiconClientTest(unittest.TestCase,
+                                   dns_test_common_lexicon.BaseLexiconClientTest):
     DOMAIN_NOT_FOUND = HTTPError('404 Client Error: Not Found for url: {0}.'.format(DOMAIN))
     LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: {0}.'.format(DOMAIN))
 

certbot-dns-sakuracloud/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.25.0.dev0'
+version = '0.28.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [

certbot-nginx/certbot_nginx/configurator.py

@@ -8,7 +8,6 @@ import tempfile
 import time
 
 import OpenSSL
-import six
 import zope.interface
 
 from acme import challenges
@@ -32,6 +31,12 @@ from certbot_nginx import obj # pylint: disable=unused-import
 from acme.magic_typing import List, Dict, Set # pylint: disable=unused-import, no-name-in-module
 
 
+NAME_RANK = 0
+START_WILDCARD_RANK = 1
+END_WILDCARD_RANK = 2
+REGEX_RANK = 3
+NO_SSL_MODIFIER = 4
+
 
 logger = logging.getLogger(__name__)
 
@@ -136,7 +141,9 @@ class NginxConfigurator(common.Installer):
         """
         # Verify Nginx is installed
         if not util.exe_exists(self.conf('ctl')):
-            raise errors.NoInstallationError
+            raise errors.NoInstallationError(
+                "Could not find a usable 'nginx' binary. Ensure nginx exists, "
+                "the binary is executable, and your PATH is set correctly.")
 
         # Make sure configuration is valid
         self.config_test()
@@ -403,7 +410,8 @@ class NginxConfigurator(common.Installer):
         """
         if not matches:
             return None
-        elif matches[0]['rank'] in six.moves.range(2, 6):
+        elif matches[0]['rank'] in [START_WILDCARD_RANK, END_WILDCARD_RANK,
+            START_WILDCARD_RANK + NO_SSL_MODIFIER, END_WILDCARD_RANK + NO_SSL_MODIFIER]:
             # Wildcard match - need to find the longest one
             rank = matches[0]['rank']
             wildcards = [x for x in matches if x['rank'] == rank]
@@ -412,10 +420,9 @@ class NginxConfigurator(common.Installer):
             # Exact or regex match
             return matches[0]['vhost']
 
-
-    def _rank_matches_by_name_and_ssl(self, vhost_list, target_name):
+    def _rank_matches_by_name(self, vhost_list, target_name):
         """Returns a ranked list of vhosts from vhost_list that match target_name.
-        The ranking gives preference to SSL vhosts.
+        This method should always be followed by a call to _select_best_name_match.
 
         :param list vhost_list: list of vhosts to filter and rank
         :param str target_name: The name to match
@@ -435,21 +442,37 @@ class NginxConfigurator(common.Installer):
             if name_type == 'exact':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 0 if vhost.ssl else 1})
+                                'rank': NAME_RANK})
             elif name_type == 'wildcard_start':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 2 if vhost.ssl else 3})
+                                'rank': START_WILDCARD_RANK})
             elif name_type == 'wildcard_end':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 4 if vhost.ssl else 5})
+                                'rank': END_WILDCARD_RANK})
             elif name_type == 'regex':
                 matches.append({'vhost': vhost,
                                 'name': name,
-                                'rank': 6 if vhost.ssl else 7})
+                                'rank': REGEX_RANK})
         return sorted(matches, key=lambda x: x['rank'])
 
+    def _rank_matches_by_name_and_ssl(self, vhost_list, target_name):
+        """Returns a ranked list of vhosts from vhost_list that match target_name.
+        The ranking gives preference to SSLishness before name match level.
+
+        :param list vhost_list: list of vhosts to filter and rank
+        :param str target_name: The name to match
+        :returns: list of dicts containing the vhost, the matching name, and
+            the numerical rank
+        :rtype: list
+
+        """
+        matches = self._rank_matches_by_name(vhost_list, target_name)
+        for match in matches:
+            if not match['vhost'].ssl:
+                match['rank'] += NO_SSL_MODIFIER
+        return sorted(matches, key=lambda x: x['rank'])
 
     def choose_redirect_vhosts(self, target_name, port, create_if_no_match=False):
         """Chooses a single virtual host for redirect enhancement.
@@ -529,9 +552,7 @@ class NginxConfigurator(common.Installer):
 
         matching_vhosts = [vhost for vhost in all_vhosts if _vhost_matches(vhost, port)]
 
-        # We can use this ranking function because sslishness doesn't matter to us, and
-        # there shouldn't be conflicting plaintextish servers listening on 80.
-        return self._rank_matches_by_name_and_ssl(matching_vhosts, target_name)
+        return self._rank_matches_by_name(matching_vhosts, target_name)
 
     def get_all_names(self):
         """Returns all names found in the Nginx Configuration.
@@ -566,6 +587,7 @@ class NginxConfigurator(common.Installer):
         return util.get_filtered_names(all_names)
 
     def _get_snakeoil_paths(self):
+        """Generate invalid certs that let us create ssl directives for Nginx"""
         # TODO: generate only once
         tmp_dir = os.path.join(self.config.work_dir, "snakeoil")
         le_key = crypto_util.init_save_key(

certbot-nginx/certbot_nginx/http_01.py

@@ -40,8 +40,6 @@ class NginxHttp01(common.ChallengePerformer):
         super(NginxHttp01, self).__init__(configurator)
         self.challenge_conf = os.path.join(
             configurator.config.config_dir, "le_http_01_cert_challenge.conf")
-        self._ipv6 = None
-        self._ipv6only = None
 
     def perform(self):
         """Perform a challenge on Nginx.
@@ -102,6 +100,7 @@ class NginxHttp01(common.ChallengePerformer):
         config = [self._make_or_mod_server_block(achall) for achall in self.achalls]
         config = [x for x in config if x is not None]
         config = nginxparser.UnspacedList(config)
+        logger.debug("Generated server block:\n%s", str(config))
 
         self.configurator.reverter.register_file_creation(
             True, self.challenge_conf)
@@ -120,9 +119,7 @@ class NginxHttp01(common.ChallengePerformer):
             self.configurator.config.http01_port)
         port = self.configurator.config.http01_port
 
-        if self._ipv6 is None or self._ipv6only is None:
-            self._ipv6, self._ipv6only = self.configurator.ipv6_info(port)
-        ipv6, ipv6only = self._ipv6, self._ipv6only
+        ipv6, ipv6only = self.configurator.ipv6_info(port)
 
         if ipv6:
             # If IPv6 is active in Nginx configuration

certbot-nginx/certbot_nginx/nginxparser.py

@@ -26,7 +26,7 @@ class RawNginxParser(object):
     dquoted = QuotedString('"', multiline=True, unquoteResults=False, escChar='\\')
     squoted = QuotedString("'", multiline=True, unquoteResults=False, escChar='\\')
     quoted = dquoted | squoted
-    head_tokenchars = Regex(r"[^{};\s'\"]") # if (last_space)
+    head_tokenchars = Regex(r"(\$\{)|[^{};\s'\"]") # if (last_space)
     tail_tokenchars = Regex(r"(\$\{)|[^{;\s]") # else
     tokenchars = Combine(head_tokenchars + ZeroOrMore(tail_tokenchars))
     paren_quote_extend = Combine(quoted + Literal(')') + ZeroOrMore(tail_tokenchars))

certbot-nginx/certbot_nginx/parser.py

@@ -222,7 +222,7 @@ class NginxParser(object):
                 return os.path.join(self.root, name)
 
         raise errors.NoInstallationError(
-            "Could not find configuration root")
+            "Could not find Nginx root configuration file (nginx.conf)")
 
     def filedump(self, ext='tmp', lazy=True):
         """Dumps parsed configurations into files.
@@ -395,12 +395,17 @@ class NginxParser(object):
                 addr.ipv6only = False
             for directive in enclosing_block[new_vhost.path[-1]][1]:
                 if len(directive) > 0 and directive[0] == 'listen':
-                    if 'default_server' in directive:
-                        del directive[directive.index('default_server')]
-                    if 'default' in directive:
-                        del directive[directive.index('default')]
-                    if 'ipv6only=on' in directive:
-                        del directive[directive.index('ipv6only=on')]
+                    # Exclude one-time use parameters which will cause an error if repeated.
+                    # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen
+                    exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog',
+                                   'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind',
+                                   'ipv6only', 'reuseport', 'so_keepalive'))
+
+                    for param in exclude:
+                        # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225
+                        keys = [x.split('=')[0] for x in directive]
+                        if param in keys:
+                            del directive[keys.index(param)]
         return new_vhost
 
 

certbot-nginx/certbot_nginx/parser_obj.py

@@ -0,0 +1,392 @@
+""" This file contains parsing routines and object classes to help derive meaning from
+raw lists of tokens from pyparsing. """
+
+import abc
+import logging
+import six
+
+from certbot import errors
+
+from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module
+
+logger = logging.getLogger(__name__)
+COMMENT = " managed by Certbot"
+COMMENT_BLOCK = ["#", COMMENT]
+
+class Parsable(object):
+    """ Abstract base class for "Parsable" objects whose underlying representation
+    is a tree of lists.
+
+    :param .Parsable parent: This object's parsed parent in the tree
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, parent=None):
+        self._data = [] # type: List[object]
+        self._tabs = None
+        self.parent = parent
+
+    @classmethod
+    def parsing_hooks(cls):
+        """Returns object types that this class should be able to `parse` recusrively.
+        The order of the objects indicates the order in which the parser should
+        try to parse each subitem.
+        :returns: A list of Parsable classes.
+        :rtype list:
+        """
+        return (Block, Sentence, Statements)
+
+    @staticmethod
+    @abc.abstractmethod
+    def should_parse(lists):
+        """ Returns whether the contents of `lists` can be parsed into this object.
+
+        :returns: Whether `lists` can be parsed as this object.
+        :rtype bool:
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def parse(self, raw_list, add_spaces=False):
+        """ Loads information into this object from underlying raw_list structure.
+        Each Parsable object might make different assumptions about the structure of
+        raw_list.
+
+        :param list raw_list: A list or sublist of tokens from pyparsing, containing whitespace
+            as separate tokens.
+        :param bool add_spaces: If set, the method can and should manipulate and insert spacing
+            between non-whitespace tokens and lists to delimit them.
+        :raises .errors.MisconfigurationError: when the assumptions about the structure of
+            raw_list are not met.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def iterate(self, expanded=False, match=None):
+        """ Iterates across this object. If this object is a leaf object, only yields
+        itself. If it contains references other parsing objects, and `expanded` is set,
+        this function should first yield itself, then recursively iterate across all of them.
+        :param bool expanded: Whether to recursively iterate on possible children.
+        :param callable match: If provided, an object is only iterated if this callable
+            returns True when called on that object.
+
+        :returns: Iterator over desired objects.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_tabs(self):
+        """ Guess at the tabbing style of this parsed object, based on whitespace.
+
+        If this object is a leaf, it deducts the tabbing based on its own contents.
+        Other objects may guess by calling `get_tabs` recursively on child objects.
+
+        :returns: Guess at tabbing for this object. Should only return whitespace strings
+            that does not contain newlines.
+        :rtype str:
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def set_tabs(self, tabs="    "):
+        """This tries to set and alter the tabbing of the current object to a desired
+        whitespace string. Primarily meant for objects that were constructed, so they
+        can conform to surrounding whitespace.
+
+        :param str tabs: A whitespace string (not containing newlines).
+        """
+        raise NotImplementedError()
+
+    def dump(self, include_spaces=False):
+        """ Dumps back to pyparsing-like list tree. The opposite of `parse`.
+
+        Note: if this object has not been modified, `dump` with `include_spaces=True`
+        should always return the original input of `parse`.
+
+        :param bool include_spaces: If set to False, magically hides whitespace tokens from
+            dumped output.
+
+        :returns: Pyparsing-like list tree.
+        :rtype list:
+        """
+        return [elem.dump(include_spaces) for elem in self._data]
+
+class Statements(Parsable):
+    """ A group or list of "Statements". A Statement is either a Block or a Sentence.
+
+    The underlying representation is simply a list of these Statement objects, with
+    an extra `_trailing_whitespace` string to keep track of the whitespace that does not
+    precede any more statements.
+    """
+    def __init__(self, parent=None):
+        super(Statements, self).__init__(parent)
+        self._trailing_whitespace = None
+
+    # ======== Begin overridden functions
+
+    @staticmethod
+    def should_parse(lists):
+        return isinstance(lists, list)
+
+    def set_tabs(self, tabs="    "):
+        """ Sets the tabbing for this set of statements. Does this by calling `set_tabs`
+        on each of the child statements.
+
+        Then, if a parent is present, sets trailing whitespace to parent tabbing. This
+        is so that the trailing } of any Block that contains Statements lines up
+        with parent tabbing.
+        """
+        for statement in self._data:
+            statement.set_tabs(tabs)
+        if self.parent is not None:
+            self._trailing_whitespace = "\n" + self.parent.get_tabs()
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list of statements.
+        Expects all elements in `parse_this` to be parseable by `type(self).parsing_hooks`,
+        with an optional whitespace string at the last index of `parse_this`.
+        """
+        if not isinstance(parse_this, list):
+            raise errors.MisconfigurationError("Statements parsing expects a list!")
+        # If there's a trailing whitespace in the list of statements, keep track of it.
+        if len(parse_this) > 0 and isinstance(parse_this[-1], six.string_types) \
+                               and parse_this[-1].isspace():
+            self._trailing_whitespace = parse_this[-1]
+            parse_this = parse_this[:-1]
+        self._data = [parse_raw(elem, self, add_spaces) for elem in parse_this]
+
+    def get_tabs(self):
+        """ Takes a guess at the tabbing of all contained Statements by retrieving the
+        tabbing of the first Statement."""
+        if len(self._data) > 0:
+            return self._data[0].get_tabs()
+        return ""
+
+    def dump(self, include_spaces=False):
+        """ Dumps this object by first dumping each statement, then appending its
+        trailing whitespace (if `include_spaces` is set) """
+        data = super(Statements, self).dump(include_spaces)
+        if include_spaces and self._trailing_whitespace is not None:
+            return data + [self._trailing_whitespace]
+        return data
+
+    def iterate(self, expanded=False, match=None):
+        """ Combines each statement's iterator.  """
+        for elem in self._data:
+            for sub_elem in elem.iterate(expanded, match):
+                yield sub_elem
+
+    # ======== End overridden functions
+
+def _space_list(list_):
+    """ Inserts whitespace between adjacent non-whitespace tokens. """
+    spaced_statement = [] # type: List[str]
+    for i in reversed(six.moves.xrange(len(list_))):
+        spaced_statement.insert(0, list_[i])
+        if i > 0 and not list_[i].isspace() and not list_[i-1].isspace():
+            spaced_statement.insert(0, " ")
+    return spaced_statement
+
+class Sentence(Parsable):
+    """ A list of words. Non-whitespace words are typically separated with whitespace tokens. """
+
+    # ======== Begin overridden functions
+
+    @staticmethod
+    def should_parse(lists):
+        """ Returns True if `lists` can be parseable as a `Sentence`-- that is,
+        every element is a string type.
+
+        :param list lists: The raw unparsed list to check.
+
+        :returns: whether this lists is parseable by `Sentence`.
+        """
+        return isinstance(lists, list) and len(lists) > 0 and \
+            all([isinstance(elem, six.string_types) for elem in lists])
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list of string types into this object.
+        If add_spaces is set, adds whitespace tokens between adjacent non-whitespace tokens."""
+        if add_spaces:
+            parse_this = _space_list(parse_this)
+        if not isinstance(parse_this, list) or \
+                any([not isinstance(elem, six.string_types) for elem in parse_this]):
+            raise errors.MisconfigurationError("Sentence parsing expects a list of string types.")
+        self._data = parse_this
+
+    def iterate(self, expanded=False, match=None):
+        """ Simply yields itself. """
+        if match is None or match(self):
+            yield self
+
+    def set_tabs(self, tabs="    "):
+        """ Sets the tabbing on this sentence. Inserts a newline and `tabs` at the
+        beginning of `self._data`. """
+        if self._data[0].isspace():
+            return
+        self._data.insert(0, "\n" + tabs)
+
+    def dump(self, include_spaces=False):
+        """ Dumps this sentence. If include_spaces is set, includes whitespace tokens."""
+        if not include_spaces:
+            return self.words
+        return self._data
+
+    def get_tabs(self):
+        """ Guesses at the tabbing of this sentence. If the first element is whitespace,
+        returns the whitespace after the rightmost newline in the string. """
+        first = self._data[0]
+        if not first.isspace():
+            return ""
+        rindex = first.rfind("\n")
+        return first[rindex+1:]
+
+    # ======== End overridden functions
+
+    @property
+    def words(self):
+        """ Iterates over words, but without spaces. Like Unspaced List. """
+        return [word.strip("\"\'") for word in self._data if not word.isspace()]
+
+    def __getitem__(self, index):
+        return self.words[index]
+
+    def __contains__(self, word):
+        return word in self.words
+
+class Block(Parsable):
+    """ Any sort of bloc, denoted by a block name and curly braces, like so:
+    The parsed block:
+        block name {
+            content 1;
+            content 2;
+        }
+    might be represented with the list [names, contents], where
+        names = ["block", " ", "name", " "]
+        contents = [["\n    ", "content", " ", "1"], ["\n    ", "content", " ", "2"], "\n"]
+    """
+    def __init__(self, parent=None):
+        super(Block, self).__init__(parent)
+        self.names = None # type: Sentence
+        self.contents = None # type: Block
+
+    @staticmethod
+    def should_parse(lists):
+        """ Returns True if `lists` can be parseable as a `Block`-- that is,
+        it's got a length of 2, the first element is a `Sentence` and the second can be
+        a `Statements`.
+
+        :param list lists: The raw unparsed list to check.
+
+        :returns: whether this lists is parseable by `Block`. """
+        return isinstance(lists, list) and len(lists) == 2 and \
+            Sentence.should_parse(lists[0]) and isinstance(lists[1], list)
+
+    def set_tabs(self, tabs="    "):
+        """ Sets tabs by setting equivalent tabbing on names, then adding tabbing
+        to contents."""
+        self.names.set_tabs(tabs)
+        self.contents.set_tabs(tabs + "    ")
+
+    def iterate(self, expanded=False, match=None):
+        """ Iterator over self, and if expanded is set, over its contents. """
+        if match is None or match(self):
+            yield self
+        if expanded:
+            for elem in self.contents.iterate(expanded, match):
+                yield elem
+
+    def parse(self, parse_this, add_spaces=False):
+        """ Parses a list that resembles a block.
+
+        The assumptions that this routine makes are:
+            1. the first element of `parse_this` is a valid Sentence.
+            2. the second element of `parse_this` is a valid Statement.
+        If add_spaces is set, we call it recursively on `names` and `contents`, and
+        add an extra trailing space to `names` (to separate the block's opening bracket
+        and the block name).
+        """
+        if not Block.should_parse(parse_this):
+            raise errors.MisconfigurationError("Block parsing expects a list of length 2. "
+                "First element should be a list of string types (the bloc names), "
+                "and second should be another list of statements (the bloc content).")
+        self.names = Sentence(self)
+        if add_spaces:
+            parse_this[0].append(" ")
+        self.names.parse(parse_this[0], add_spaces)
+        self.contents = Statements(self)
+        self.contents.parse(parse_this[1], add_spaces)
+        self._data = [self.names, self.contents]
+
+    def get_tabs(self):
+        """ Guesses tabbing by retrieving tabbing guess of self.names. """
+        return self.names.get_tabs()
+
+def _is_comment(parsed_obj):
+    """ Checks whether parsed_obj is a comment.
+
+    :param .Parsable parsed_obj:
+
+    :returns: whether parsed_obj represents a comment sentence.
+    :rtype bool:
+    """
+    if not isinstance(parsed_obj, Sentence):
+        return False
+    return parsed_obj.words[0] == "#"
+
+def _is_certbot_comment(parsed_obj):
+    """ Checks whether parsed_obj is a "managed by Certbot" comment.
+
+    :param .Parsable parsed_obj:
+
+    :returns: whether parsed_obj is a "managed by Certbot" comment.
+    :rtype bool:
+    """
+    if not _is_comment(parsed_obj):
+        return False
+    if len(parsed_obj.words) != len(COMMENT_BLOCK):
+        return False
+    for i, word in enumerate(parsed_obj.words):
+        if word != COMMENT_BLOCK[i]:
+            return False
+    return True
+
+def _certbot_comment(parent, preceding_spaces=4):
+    """ A "Managed by Certbot" comment.
+    :param int preceding_spaces: Number of spaces between the end of the previous
+        statement and the comment.
+    :returns: Sentence containing the comment.
+    :rtype: .Sentence
+    """
+    result = Sentence(parent)
+    result.parse([" " * preceding_spaces] + COMMENT_BLOCK)
+    return result
+
+def _choose_parser(parent, list_):
+    """ Choose a parser from type(parent).parsing_hooks, depending on whichever hook
+    returns True first. """
+    hooks = Parsable.parsing_hooks()
+    if parent:
+        hooks = type(parent).parsing_hooks()
+    for type_ in hooks:
+        if type_.should_parse(list_):
+            return type_(parent)
+    raise errors.MisconfigurationError(
+        "None of the parsing hooks succeeded, so we don't know how to parse this set of lists.")
+
+def parse_raw(lists_, parent=None, add_spaces=False):
+    """ Primary parsing factory function.
+
+    :param list lists_: raw lists from pyparsing to parse.
+    :param .Parent parent: The parent containing this object.
+    :param bool add_spaces: Whether to pass add_spaces to the parser.
+
+    :returns .Parsable: The parsed object.
+
+    :raises errors.MisconfigurationError: If no parsing hook passes, and we can't
+        determine which type to parse the raw lists into.
+    """
+    parser = _choose_parser(parent, lists_)
+    parser.parse(lists_, add_spaces)
+    return parser

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -128,22 +128,39 @@ class NginxConfiguratorTest(util.NginxTest):
                             ['#', parser.COMMENT]]]],
                          parsed[0])
 
-    def test_choose_vhosts(self):
-        localhost_conf = set(['localhost', r'~^(www\.)?(example|bar)\.'])
-        server_conf = set(['somename', 'another.alias', 'alias'])
-        example_conf = set(['.example.com', 'example.*'])
-        foo_conf = set(['*.www.foo.com', '*.www.example.com'])
-        ipv6_conf = set(['ipv6.com'])
+    def test_choose_vhosts_alias(self):
+        self._test_choose_vhosts_common('alias', 'server_conf')
 
-        results = {'localhost': localhost_conf,
-                   'alias': server_conf,
-                   'example.com': example_conf,
-                   'example.com.uk.test': example_conf,
-                   'www.example.com': example_conf,
-                   'test.www.example.com': foo_conf,
-                   'abc.www.foo.com': foo_conf,
-                   'www.bar.co.uk': localhost_conf,
-                   'ipv6.com': ipv6_conf}
+    def test_choose_vhosts_example_com(self):
+        self._test_choose_vhosts_common('example.com', 'example_conf')
+
+    def test_choose_vhosts_localhost(self):
+        self._test_choose_vhosts_common('localhost', 'localhost_conf')
+
+    def test_choose_vhosts_example_com_uk_test(self):
+        self._test_choose_vhosts_common('example.com.uk.test', 'example_conf')
+
+    def test_choose_vhosts_www_example_com(self):
+        self._test_choose_vhosts_common('www.example.com', 'example_conf')
+
+    def test_choose_vhosts_test_www_example_com(self):
+        self._test_choose_vhosts_common('test.www.example.com', 'foo_conf')
+
+    def test_choose_vhosts_abc_www_foo_com(self):
+        self._test_choose_vhosts_common('abc.www.foo.com', 'foo_conf')
+
+    def test_choose_vhosts_www_bar_co_uk(self):
+        self._test_choose_vhosts_common('www.bar.co.uk', 'localhost_conf')
+
+    def test_choose_vhosts_ipv6_com(self):
+        self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf')
+
+    def _test_choose_vhosts_common(self, name, conf):
+        conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']),
+                 'server_conf': set(['somename', 'another.alias', 'alias']),
+                 'example_conf': set(['.example.com', 'example.*']),
+                 'foo_conf': set(['*.www.foo.com', '*.www.example.com']),
+                 'ipv6_conf': set(['ipv6.com'])}
 
         conf_path = {'localhost': "etc_nginx/nginx.conf",
                    'alias': "etc_nginx/nginx.conf",
@@ -155,22 +172,22 @@ class NginxConfiguratorTest(util.NginxTest):
                    'www.bar.co.uk': "etc_nginx/nginx.conf",
                    'ipv6.com': "etc_nginx/sites-enabled/ipv6.com"}
 
+        vhost = self.config.choose_vhosts(name)[0]
+        path = os.path.relpath(vhost.filep, self.temp_dir)
+
+        self.assertEqual(conf_names[conf], vhost.names)
+        self.assertEqual(conf_path[name], path)
+        # IPv6 specific checks
+        if name == "ipv6.com":
+            self.assertTrue(vhost.ipv6_enabled())
+            # Make sure that we have SSL enabled also for IPv6 addr
+            self.assertTrue(
+                any([True for x in vhost.addrs if x.ssl and x.ipv6]))
+
+    def test_choose_vhosts_bad(self):
         bad_results = ['www.foo.com', 'example', 't.www.bar.co',
                        '69.255.225.155']
 
-        for name in results:
-            vhost = self.config.choose_vhosts(name)[0]
-            path = os.path.relpath(vhost.filep, self.temp_dir)
-
-            self.assertEqual(results[name], vhost.names)
-            self.assertEqual(conf_path[name], path)
-            # IPv6 specific checks
-            if name == "ipv6.com":
-                self.assertTrue(vhost.ipv6_enabled())
-                # Make sure that we have SSL enabled also for IPv6 addr
-                self.assertTrue(
-                    any([True for x in vhost.addrs if x.ssl and x.ipv6]))
-
         for name in bad_results:
             self.assertRaises(errors.MisconfigurationError,
                               self.config.choose_vhosts, name)

certbot-nginx/certbot_nginx/tests/http_01_test.py

@@ -12,6 +12,7 @@ from certbot import achallenges
 from certbot.plugins import common_test
 from certbot.tests import acme_util
 
+from certbot_nginx.obj import Addr
 from certbot_nginx.tests import util
 
 
@@ -108,6 +109,41 @@ class HttpPerformTest(util.NginxTest):
             #     self.assertEqual(vhost.addrs, set(v_addr2_print))
             # self.assertEqual(vhost.names, set([response.z_domain.decode('ascii')]))
 
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_no_memoization(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, True)
+        self.http01._default_listen_addresses()
+        self.assertEqual(ipv6_info.call_count, 1)
+        ipv6_info.return_value = (False, False)
+        self.http01._default_listen_addresses()
+        self.assertEqual(ipv6_info.call_count, 2)
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_t_t(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, True)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        http_ipv6_addr = Addr.fromstring("[::]:80")
+        self.assertEqual(addrs, [http_addr, http_ipv6_addr])
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_t_f(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (True, False)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        http_ipv6_addr = Addr.fromstring("[::]:80 ipv6only=on")
+        self.assertEqual(addrs, [http_addr, http_ipv6_addr])
+
+    @mock.patch("certbot_nginx.configurator.NginxConfigurator.ipv6_info")
+    def test_default_listen_addresses_f_f(self, ipv6_info):
+        # pylint: disable=protected-access
+        ipv6_info.return_value = (False, False)
+        addrs = self.http01._default_listen_addresses()
+        http_addr = Addr.fromstring("80")
+        self.assertEqual(addrs, [http_addr])
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tests/nginxparser_test.py

@@ -271,6 +271,8 @@ class TestRawNginxParser(unittest.TestCase):
             location ~ ^/users/(.+\.(?:gif|jpe?g|png))$ {
               alias /data/w3/images/$1;
             }
+
+            proxy_set_header X-Origin-URI ${scheme}://${http_host}/$request_uri;
         """
         parsed = loads(test)
         self.assertEqual(parsed, [[['if', '($http_user_agent', '~', 'MSIE)'],
@@ -281,7 +283,8 @@ class TestRawNginxParser(unittest.TestCase):
             [['return', '403']]], [['if', '($args', '~', 'post=140)'],
             [['rewrite', '^', 'http://example.com/']]],
             [['location', '~', '^/users/(.+\\.(?:gif|jpe?g|png))$'],
-            [['alias', '/data/w3/images/$1']]]]
+            [['alias', '/data/w3/images/$1']]],
+            ['proxy_set_header', 'X-Origin-URI', '${scheme}://${http_host}/$request_uri']]
         )
 
     def test_edge_cases(self):
@@ -289,10 +292,6 @@ class TestRawNginxParser(unittest.TestCase):
         parsed = loads(r'"hello\""; # blah "heh heh"')
         self.assertEqual(parsed, [['"hello\\""'], ['#', ' blah "heh heh"']])
 
-        # empty var as block
-        parsed = loads(r"${}")
-        self.assertEqual(parsed, [[['$'], []]])
-
         # if with comment
         parsed = loads("""if ($http_cookie ~* "id=([^;]+)(?:;|$)") { # blah )
             }""")
@@ -342,10 +341,9 @@ class TestRawNginxParser(unittest.TestCase):
         ])
 
         # variable weirdness
-        parsed = loads("directive $var;")
-        self.assertEqual(parsed, [['directive', '$var']])
+        parsed = loads("directive $var ${var} $ ${};")
+        self.assertEqual(parsed, [['directive', '$var', '${var}', '$', '${}']])
         self.assertRaises(ParseException, loads, "server {server_name test.com};")
-        self.assertRaises(ParseException, loads, "directive ${var};")
         self.assertEqual(loads("blag${dfgdfg};"), [['blag${dfgdfg}']])
         self.assertRaises(ParseException, loads, "blag${dfgdf{g};")
 

certbot-nginx/certbot_nginx/tests/parser_obj_test.py

@@ -0,0 +1,253 @@
+""" Tests for functions and classes in parser_obj.py """
+
+import unittest
+import mock
+
+from certbot_nginx.parser_obj import parse_raw
+from certbot_nginx.parser_obj import COMMENT_BLOCK
+
+class CommentHelpersTest(unittest.TestCase):
+    def test_is_comment(self):
+        from certbot_nginx.parser_obj import _is_comment
+        self.assertTrue(_is_comment(parse_raw(['#'])))
+        self.assertTrue(_is_comment(parse_raw(['#', ' literally anything else'])))
+        self.assertFalse(_is_comment(parse_raw(['not', 'even', 'a', 'comment'])))
+
+    def test_is_certbot_comment(self):
+        from certbot_nginx.parser_obj import _is_certbot_comment
+        self.assertTrue(_is_certbot_comment(
+            parse_raw(COMMENT_BLOCK)))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['#', ' not a certbot comment'])))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['#', ' managed by Certbot', ' also not a certbot comment'])))
+        self.assertFalse(_is_certbot_comment(
+            parse_raw(['not', 'even', 'a', 'comment'])))
+
+    def test_certbot_comment(self):
+        from certbot_nginx.parser_obj import _certbot_comment, _is_certbot_comment
+        comment = _certbot_comment(None)
+        self.assertTrue(_is_certbot_comment(comment))
+        self.assertEqual(comment.dump(), COMMENT_BLOCK)
+        self.assertEqual(comment.dump(True), ['    '] + COMMENT_BLOCK)
+        self.assertEqual(_certbot_comment(None, 2).dump(True),
+            ['  '] + COMMENT_BLOCK)
+
+class ParsingHooksTest(unittest.TestCase):
+    def test_is_sentence(self):
+        from certbot_nginx.parser_obj import Sentence
+        self.assertFalse(Sentence.should_parse([]))
+        self.assertTrue(Sentence.should_parse(['']))
+        self.assertTrue(Sentence.should_parse(['word']))
+        self.assertTrue(Sentence.should_parse(['two', 'words']))
+        self.assertFalse(Sentence.should_parse([[]]))
+        self.assertFalse(Sentence.should_parse(['word', []]))
+
+    def test_is_block(self):
+        from certbot_nginx.parser_obj import Block
+        self.assertFalse(Block.should_parse([]))
+        self.assertFalse(Block.should_parse(['']))
+        self.assertFalse(Block.should_parse(['two', 'words']))
+        self.assertFalse(Block.should_parse([[[]], []]))
+        self.assertFalse(Block.should_parse([['block_name'], ['hi', []], []]))
+        self.assertFalse(Block.should_parse([['block_name'], 'lol']))
+        self.assertTrue(Block.should_parse([['block_name'], ['hi', []]]))
+        self.assertTrue(Block.should_parse([['hello'], []]))
+        self.assertTrue(Block.should_parse([['block_name'], [['many'], ['statements'], 'here']]))
+        self.assertTrue(Block.should_parse([['if', ' ', '(whatever)'], ['hi']]))
+
+    def test_parse_raw(self):
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: True
+        fake_parser2 = mock.Mock()
+        fake_parser2.should_parse = lambda x: False
+        # First encountered "match" should parse.
+        parse_raw([])
+        fake_parser1.called_once()
+        fake_parser2.not_called()
+        fake_parser1.reset_mock()
+        # "match" that returns False shouldn't parse.
+        parse_raw([])
+        fake_parser1.not_called()
+        fake_parser2.called_once()
+
+    @mock.patch("certbot_nginx.parser_obj.Parsable.parsing_hooks")
+    def test_parse_raw_no_match(self, parsing_hooks):
+        from certbot import errors
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: False
+        parsing_hooks.return_value = (fake_parser1,)
+        self.assertRaises(errors.MisconfigurationError, parse_raw, [])
+        parsing_hooks.return_value = tuple()
+        self.assertRaises(errors.MisconfigurationError, parse_raw, [])
+
+    def test_parse_raw_passes_add_spaces(self):
+        fake_parser1 = mock.Mock()
+        fake_parser1.should_parse = lambda x: True
+        parse_raw([])
+        fake_parser1.parse.called_with([None])
+        parse_raw([], add_spaces=True)
+        fake_parser1.parse.called_with([None, True])
+
+class SentenceTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Sentence
+        self.sentence = Sentence(None)
+
+    def test_parse_bad_sentence_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, 'lol')
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [[]])
+        self.assertRaises(errors.MisconfigurationError, self.sentence.parse, [5])
+
+    def test_parse_sentence_words_hides_spaces(self):
+        og_sentence = ['\r\n', 'hello', ' ', ' ', '\t\n  ', 'lol', ' ', 'spaces']
+        self.sentence.parse(og_sentence)
+        self.assertEquals(self.sentence.words, ['hello', 'lol', 'spaces'])
+        self.assertEquals(self.sentence.dump(), ['hello', 'lol', 'spaces'])
+        self.assertEquals(self.sentence.dump(True), og_sentence)
+
+    def test_parse_sentence_with_add_spaces(self):
+        self.sentence.parse(['hi', 'there'], add_spaces=True)
+        self.assertEquals(self.sentence.dump(True), ['hi', ' ', 'there'])
+        self.sentence.parse(['one', ' ', 'space', 'none'], add_spaces=True)
+        self.assertEquals(self.sentence.dump(True), ['one', ' ', 'space', ' ', 'none'])
+
+    def test_iterate(self):
+        expected = [['1', '2', '3']]
+        self.sentence.parse(['1', ' ', '2', ' ', '3'])
+        for i, sentence in enumerate(self.sentence.iterate()):
+            self.assertEquals(sentence.dump(), expected[i])
+
+    def test_set_tabs(self):
+        self.sentence.parse(['tabs', 'pls'], add_spaces=True)
+        self.sentence.set_tabs()
+        self.assertEquals(self.sentence.dump(True)[0], '\n    ')
+        self.sentence.parse(['tabs', 'pls'], add_spaces=True)
+
+    def test_get_tabs(self):
+        self.sentence.parse(['no', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '')
+        self.sentence.parse(['\n \n  ', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '  ')
+        self.sentence.parse(['\n\t  ', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '\t  ')
+        self.sentence.parse(['\n\t \n', 'tabs'])
+        self.assertEquals(self.sentence.get_tabs(), '')
+
+class BlockTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Block
+        self.bloc = Block(None)
+        self.name = ['server', 'name']
+        self.contents = [['thing', '1'], ['thing', '2'], ['another', 'one']]
+        self.bloc.parse([self.name, self.contents])
+
+    def test_iterate(self):
+        # Iterates itself normally
+        self.assertEquals(self.bloc, next(self.bloc.iterate()))
+        # Iterates contents while expanded
+        expected = [self.bloc.dump()] + self.contents
+        for i, elem in enumerate(self.bloc.iterate(expanded=True)):
+            self.assertEquals(expected[i], elem.dump())
+
+    def test_iterate_match(self):
+        # can match on contents while expanded
+        from certbot_nginx.parser_obj import Block, Sentence
+        expected = [['thing', '1'], ['thing', '2']]
+        for i, elem in enumerate(self.bloc.iterate(expanded=True,
+            match=lambda x: isinstance(x, Sentence) and 'thing' in x.words)):
+            self.assertEquals(expected[i], elem.dump())
+        # can match on self
+        self.assertEquals(self.bloc, next(self.bloc.iterate(
+            expanded=True,
+            match=lambda x: isinstance(x, Block) and 'server' in x.names)))
+
+    def test_parse_with_added_spaces(self):
+        import copy
+        self.bloc.parse([copy.copy(self.name), self.contents], add_spaces=True)
+        self.assertEquals(self.bloc.dump(), [self.name, self.contents])
+        self.assertEquals(self.bloc.dump(True), [
+            ['server', ' ', 'name', ' '],
+            [['thing', ' ', '1'],
+             ['thing', ' ', '2'],
+             ['another', ' ', 'one']]])
+
+    def test_bad_parse_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, [[[]], [[]]])
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['lol'])
+        self.assertRaises(errors.MisconfigurationError, self.bloc.parse, ['fake', 'news'])
+
+    def test_set_tabs(self):
+        self.bloc.set_tabs()
+        self.assertEquals(self.bloc.names.dump(True)[0], '\n    ')
+        for elem in self.bloc.contents.dump(True)[:-1]:
+            self.assertEquals(elem[0], '\n        ')
+        self.assertEquals(self.bloc.contents.dump(True)[-1][0], '\n')
+
+    def test_get_tabs(self):
+        self.bloc.parse([[' \n  \t', 'lol'], []])
+        self.assertEquals(self.bloc.get_tabs(), '  \t')
+
+class StatementsTest(unittest.TestCase):
+    def setUp(self):
+        from certbot_nginx.parser_obj import Statements
+        self.statements = Statements(None)
+        self.raw = [
+            ['sentence', 'one'],
+            ['sentence', 'two'],
+            ['and', 'another']
+        ]
+        self.raw_spaced = [
+            ['\n  ', 'sentence', ' ', 'one'],
+            ['\n  ', 'sentence', ' ', 'two'],
+            ['\n  ', 'and', ' ', 'another'],
+            '\n\n'
+        ]
+
+    def test_set_tabs(self):
+        self.statements.parse(self.raw)
+        self.statements.set_tabs()
+        for statement in self.statements.iterate():
+            self.assertEquals(statement.dump(True)[0], '\n    ')
+
+    def test_set_tabs_with_parent(self):
+        # Trailing whitespace should inherit from parent tabbing.
+        self.statements.parse(self.raw)
+        self.statements.parent = mock.Mock()
+        self.statements.parent.get_tabs.return_value = '\t\t'
+        self.statements.set_tabs()
+        for statement in self.statements.iterate():
+            self.assertEquals(statement.dump(True)[0], '\n    ')
+        self.assertEquals(self.statements.dump(True)[-1], '\n\t\t')
+
+    def test_get_tabs(self):
+        self.raw[0].insert(0, '\n \n  \t')
+        self.statements.parse(self.raw)
+        self.assertEquals(self.statements.get_tabs(), '  \t')
+        self.statements.parse([])
+        self.assertEquals(self.statements.get_tabs(), '')
+
+    def test_parse_with_added_spaces(self):
+        self.statements.parse(self.raw, add_spaces=True)
+        self.assertEquals(self.statements.dump(True)[0], ['sentence', ' ', 'one'])
+
+    def test_parse_bad_list_raises_error(self):
+        from certbot import errors
+        self.assertRaises(errors.MisconfigurationError, self.statements.parse, 'lol not a list')
+
+    def test_parse_hides_trailing_whitespace(self):
+        self.statements.parse(self.raw + ['\n\n  '])
+        self.assertTrue(isinstance(self.statements.dump()[-1], list))
+        self.assertTrue(self.statements.dump(True)[-1].isspace())
+        self.assertEquals(self.statements.dump(True)[-1], '\n\n  ')
+
+    def test_iterate(self):
+        self.statements.parse(self.raw)
+        expected = [['sentence', 'one'], ['sentence', 'two']]
+        for i, elem in enumerate(self.statements.iterate(match=lambda x: 'sentence' in x)):
+            self.assertEquals(expected[i], elem.dump())
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot-nginx/certbot_nginx/tls_sni_01.py

@@ -51,9 +51,6 @@ class NginxTlsSni01(common.TLSSNI01):
         default_addr = "{0} ssl".format(
             self.configurator.config.tls_sni_01_port)
 
-        ipv6, ipv6only = self.configurator.ipv6_info(
-            self.configurator.config.tls_sni_01_port)
-
         for achall in self.achalls:
             vhosts = self.configurator.choose_vhosts(achall.domain, create_if_no_match=True)
 
@@ -61,6 +58,9 @@ class NginxTlsSni01(common.TLSSNI01):
             if vhosts and vhosts[0].addrs:
                 addresses.append(list(vhosts[0].addrs))
             else:
+                # choose_vhosts might have modified vhosts, so put this after
+                ipv6, ipv6only = self.configurator.ipv6_info(
+                    self.configurator.config.tls_sni_01_port)
                 if ipv6:
                     # If IPv6 is active in Nginx configuration
                     ipv6_addr = "[::]:{0} ssl".format(
@@ -141,6 +141,8 @@ class NginxTlsSni01(common.TLSSNI01):
         with open(self.challenge_conf, "w") as new_conf:
             nginxparser.dump(config, new_conf)
 
+        logger.debug("Generated server block:\n%s", str(config))
+
     def _make_server_block(self, achall, addrs):
         """Creates a server block for a challenge.
 

certbot-nginx/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.25.0
--e .[dev]
+acme[dev]==0.26.0
+certbot[dev]==0.22.0

certbot-nginx/setup.py

@@ -2,12 +2,12 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.26.0.dev0'
+version = '0.28.0.dev0'
 
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.25.0',
+    'acme>=0.26.0',
     'certbot>=0.22.0',
     'mock',
     'PyOpenSSL',

certbot-nginx/tests/boulder-integration.sh

@@ -35,6 +35,7 @@ test_deployment_and_rollback() {
 }
 
 export default_server="default_server"
+nginx -v
 reload_nginx
 certbot_test_nginx --domains nginx.wtf run
 test_deployment_and_rollback nginx.wtf

certbot-postfix/certbot_postfix/tests/installer_test.py

@@ -253,7 +253,7 @@ class InstallerTest(certbot_test_util.ConfigTestCase):
                 fake_set.reset_mock()
                 installer.deploy_cert("example.com", "cert_path", "key_path",
                                       "chain_path", "fullchain_path")
-                fake_set.assert_not_called()
+                self.assertFalse(fake_set.called)
 
     @certbot_test_util.patch_get_utility()
     def test_deploy_already_secure(self, mock_util):

certbot/__init__.py

@@ -1,4 +1,4 @@
 """Certbot client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.26.0.dev0'
+__version__ = '0.28.0.dev0'

certbot/account.py

@@ -1,5 +1,6 @@
 """Creates ACME accounts for server."""
 import datetime
+import functools
 import hashlib
 import logging
 import os
@@ -16,6 +17,7 @@ import zope.component
 from acme import fields as acme_fields
 from acme import messages
 
+from certbot import compat
 from certbot import constants
 from certbot import errors
 from certbot import interfaces
@@ -139,7 +141,7 @@ class AccountFileStorage(interfaces.AccountStorage):
     """
     def __init__(self, config):
         self.config = config
-        util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(),
+        util.make_or_verify_dir(config.accounts_dir, 0o700, compat.os_geteuid(),
                                    self.config.strict_permissions)
 
     def _account_dir_path(self, account_id):
@@ -191,6 +193,11 @@ class AccountFileStorage(interfaces.AccountStorage):
     def find_all(self):
         return self._find_all_for_server_path(self.config.server_path)
 
+    def _symlink_to_account_dir(self, prev_server_path, server_path, account_id):
+        prev_account_dir = self._account_dir_path_for_server_path(account_id, prev_server_path)
+        new_account_dir = self._account_dir_path_for_server_path(account_id, server_path)
+        os.symlink(prev_account_dir, new_account_dir)
+
     def _symlink_to_accounts_dir(self, prev_server_path, server_path):
         accounts_dir = self.config.accounts_dir_for_server_path(server_path)
         if os.path.islink(accounts_dir):
@@ -207,7 +214,12 @@ class AccountFileStorage(interfaces.AccountStorage):
                 prev_server_path = constants.LE_REUSE_SERVERS[server_path]
                 prev_loaded_account = self._load_for_server_path(account_id, prev_server_path)
                 # we didn't error so we found something, so create a symlink to that
-                self._symlink_to_accounts_dir(prev_server_path, server_path)
+                accounts_dir = self.config.accounts_dir_for_server_path(server_path)
+                # If accounts_dir isn't empty, make an account specific symlink
+                if os.listdir(accounts_dir):
+                    self._symlink_to_account_dir(prev_server_path, server_path, account_id)
+                else:
+                    self._symlink_to_accounts_dir(prev_server_path, server_path)
                 return prev_loaded_account
             else:
                 raise errors.AccountNotFound(
@@ -250,53 +262,69 @@ class AccountFileStorage(interfaces.AccountStorage):
         :param account_id: id of account which should be deleted
 
         """
-        # Step 1: remove the account itself
         account_dir_path = self._account_dir_path(account_id)
         if not os.path.isdir(account_dir_path):
             raise errors.AccountNotFound(
                 "Account at %s does not exist" % account_dir_path)
-        shutil.rmtree(account_dir_path)
+        # Step 1: Delete account specific links and the directory
+        self._delete_account_dir_for_server_path(account_id, self.config.server_path)
 
-        # Step 2: remove the directory if it's empty, and linked directories
+        # Step 2: Remove any accounts links and directories that are now empty
         if not os.listdir(self.config.accounts_dir):
             self._delete_accounts_dir_for_server_path(self.config.server_path)
 
+    def _delete_account_dir_for_server_path(self, account_id, server_path):
+        link_func = functools.partial(self._account_dir_path_for_server_path, account_id)
+        nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func)
+        shutil.rmtree(nonsymlinked_dir)
+
     def _delete_accounts_dir_for_server_path(self, server_path):
-        accounts_dir_path = self.config.accounts_dir_for_server_path(server_path)
+        link_func = self.config.accounts_dir_for_server_path
+        nonsymlinked_dir = self._delete_links_and_find_target_dir(server_path, link_func)
+        os.rmdir(nonsymlinked_dir)
+
+    def _delete_links_and_find_target_dir(self, server_path, link_func):
+        """Delete symlinks and return the nonsymlinked directory path.
+
+        :param str server_path: file path based on server
+        :param callable link_func: callable that returns possible links
+            given a server_path
+
+        :returns: the final, non-symlinked target
+        :rtype: str
+
+        """
+        dir_path = link_func(server_path)
 
         # does an appropriate directory link to me? if so, make sure that's gone
         reused_servers = {}
         for k in constants.LE_REUSE_SERVERS:
             reused_servers[constants.LE_REUSE_SERVERS[k]] = k
 
-        # is there a next one up? call that and be done
-        if server_path in reused_servers:
-            next_server_path = reused_servers[server_path]
-            next_accounts_dir_path = self.config.accounts_dir_for_server_path(next_server_path)
-            if os.path.islink(next_accounts_dir_path) \
-                and os.readlink(next_accounts_dir_path) == accounts_dir_path:
-                self._delete_accounts_dir_for_server_path(next_server_path)
-                return
+        # is there a next one up?
+        possible_next_link = True
+        while possible_next_link:
+            possible_next_link = False
+            if server_path in reused_servers:
+                next_server_path = reused_servers[server_path]
+                next_dir_path = link_func(next_server_path)
+                if os.path.islink(next_dir_path) and os.readlink(next_dir_path) == dir_path:
+                    possible_next_link = True
+                    server_path = next_server_path
+                    dir_path = next_dir_path
 
         # if there's not a next one up to delete, then delete me
-        # and whatever I link to if applicable
-        if os.path.islink(accounts_dir_path):
-            # save my info then delete me
-            target = os.readlink(accounts_dir_path)
-            os.unlink(accounts_dir_path)
-            # then delete whatever I linked to, if appropriate
-            if server_path in constants.LE_REUSE_SERVERS:
-                prev_server_path = constants.LE_REUSE_SERVERS[server_path]
-                prev_accounts_dir_path = self.config.accounts_dir_for_server_path(prev_server_path)
-                if target == prev_accounts_dir_path:
-                    self._delete_accounts_dir_for_server_path(prev_server_path)
-        else:
-            # just delete me
-            os.rmdir(accounts_dir_path)
+        # and whatever I link to
+        while os.path.islink(dir_path):
+            target = os.readlink(dir_path)
+            os.unlink(dir_path)
+            dir_path = target
+
+        return dir_path
 
     def _save(self, account, acme, regr_only):
         account_dir_path = self._account_dir_path(account.id)
-        util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(),
+        util.make_or_verify_dir(account_dir_path, 0o700, compat.os_geteuid(),
                                 self.config.strict_permissions)
         try:
             with open(self._regr_path(account_dir_path), "w") as regr_file:

certbot/cert_manager.py

@@ -8,6 +8,7 @@ import traceback
 import zope.component
 
 from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
+from certbot import compat
 from certbot import crypto_util
 from certbot import errors
 from certbot import interfaces
@@ -104,7 +105,7 @@ def lineage_for_certname(cli_config, certname):
     """Find a lineage object with name certname."""
     configs_dir = cli_config.renewal_configs_dir
     # Verify the directory is there
-    util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid())
+    util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid())
     try:
         renewal_file = storage.renewal_file_for_certname(cli_config, certname)
     except errors.CertStorageError:
@@ -353,7 +354,7 @@ def _describe_certs(config, parsed_certs, parse_failures):
             notify("Found the following {0}certs:".format(match))
             notify(_report_human_readable(config, parsed_certs))
         if parse_failures:
-            notify("\nThe following renewal configuration files "
+            notify("\nThe following renewal configurations "
                "were invalid:")
             notify(_report_lines(parse_failures))
 
@@ -374,7 +375,7 @@ def _search_lineages(cli_config, func, initial_rv, *args):
     """
     configs_dir = cli_config.renewal_configs_dir
     # Verify the directory is there
-    util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid())
+    util.make_or_verify_dir(configs_dir, mode=0o755, uid=compat.os_geteuid())
 
     rv = initial_rv
     for renewal_file in storage.renewal_conf_files(cli_config):

certbot/cli.py

@@ -96,7 +96,7 @@ obtain, install, and renew certificates:
 
 manage certificates:
     certificates    Display information about certificates you have from Certbot
-    revoke          Revoke a certificate (supply --cert-path)
+    revoke          Revoke a certificate (supply --cert-path or --cert-name)
     delete          Delete a certificate
 
 manage your account with Let's Encrypt:
@@ -387,9 +387,10 @@ VERB_HELP = [
         "usage": "\n\n  certbot delete --cert-name CERTNAME\n\n"
     }),
     ("revoke", {
-        "short": "Revoke a certificate specified with --cert-path",
+        "short": "Revoke a certificate specified with --cert-path or --cert-name",
         "opts": "Options for revocation of certificates",
-        "usage": "\n\n  certbot revoke --cert-path /path/to/fullchain.pem [options]\n\n"
+        "usage": "\n\n  certbot revoke [--cert-path /path/to/fullchain.pem | "
+        "--cert-name example.com] [options]\n\n"
     }),
     ("register", {
         "short": "Register for account with Let's Encrypt / other ACME server",
@@ -1333,7 +1334,7 @@ def _paths_parser(helpful):
         add(sections, "--cert-path", type=os.path.abspath,
             default=flag_default("auth_cert_path"), help=cph)
     elif verb == "revoke":
-        add(sections, "--cert-path", type=read_file, required=True, help=cph)
+        add(sections, "--cert-path", type=read_file, required=False, help=cph)
     else:
         add(sections, "--cert-path", type=os.path.abspath, help=cph)
 

certbot/client.py

@@ -24,6 +24,7 @@ import certbot
 from certbot import account
 from certbot import auth_handler
 from certbot import cli
+from certbot import compat
 from certbot import constants
 from certbot import crypto_util
 from certbot import eff
@@ -447,7 +448,7 @@ class Client(object):
         """
         for path in cert_path, chain_path, fullchain_path:
             util.make_or_verify_dir(
-                os.path.dirname(path), 0o755, os.geteuid(),
+                os.path.dirname(path), 0o755, compat.os_geteuid(),
                 self.config.strict_permissions)
 
 

certbot/compat.py

@@ -0,0 +1,150 @@
+"""
+Compatibility layer to run certbot both on Linux and Windows.
+
+The approach used here is similar to Modernizr for Web browsers.
+We do not check the platform type to determine if a particular logic is supported.
+Instead, we apply a logic, and then fallback to another logic if first logic
+is not supported at runtime.
+
+Then logic chains are abstracted into single functions to be exposed to certbot.
+"""
+import os
+import select
+import sys
+import errno
+import ctypes
+import stat
+
+from certbot import errors
+
+try:
+    # Linux specific
+    import fcntl # pylint: disable=import-error
+except ImportError:
+    # Windows specific
+    import msvcrt # pylint: disable=import-error
+
+UNPRIVILEGED_SUBCOMMANDS_ALLOWED = [
+    'certificates', 'enhance', 'revoke', 'delete',
+    'register', 'unregister', 'config_changes', 'plugins']
+def raise_for_non_administrative_windows_rights(subcommand):
+    """
+    On Windows, raise if current shell does not have the administrative rights.
+    Do nothing on Linux.
+
+    :param str subcommand: The subcommand (like 'certonly') passed to the certbot client.
+
+    :raises .errors.Error: If the provided subcommand must be run on a shell with
+        administrative rights, and current shell does not have these rights.
+
+    """
+    # Why not simply try ctypes.windll.shell32.IsUserAnAdmin() and catch AttributeError ?
+    # Because windll exists only on a Windows runtime, and static code analysis engines
+    # do not like at all non existent objects when run from Linux (even if we handle properly
+    # all the cases in the code).
+    # So we access windll only by reflection to trick theses engines.
+    if hasattr(ctypes, 'windll') and subcommand not in UNPRIVILEGED_SUBCOMMANDS_ALLOWED:
+        windll = getattr(ctypes, 'windll')
+        if windll.shell32.IsUserAnAdmin() == 0:
+            raise errors.Error(
+                'Error, "{0}" subcommand must be run on a shell with administrative rights.'
+                .format(subcommand))
+
+def os_geteuid():
+    """
+    Get current user uid
+
+    :returns: The current user uid.
+    :rtype: int
+
+    """
+    try:
+        # Linux specific
+        return os.geteuid()
+    except AttributeError:
+        # Windows specific
+        return 0
+
+def readline_with_timeout(timeout, prompt):
+    """
+    Read user input to return the first line entered, or raise after specified timeout.
+
+    :param float timeout: The timeout in seconds given to the user.
+    :param str prompt: The prompt message to display to the user.
+
+    :returns: The first line entered by the user.
+    :rtype: str
+
+    """
+    try:
+        # Linux specific
+        #
+        # Call to select can only be done like this on UNIX
+        rlist, _, _ = select.select([sys.stdin], [], [], timeout)
+        if not rlist:
+            raise errors.Error(
+                "Timed out waiting for answer to prompt '{0}'".format(prompt))
+        return rlist[0].readline()
+    except OSError:
+        # Windows specific
+        #
+        # No way with select to make a timeout to the user input on Windows,
+        # as select only supports socket in this case.
+        # So no timeout on Windows for now.
+        return sys.stdin.readline()
+
+def lock_file(fd):
+    """
+    Lock the file linked to the specified file descriptor.
+
+    :param int fd: The file descriptor of the file to lock.
+
+    """
+    if 'fcntl' in sys.modules:
+        # Linux specific
+        fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    else:
+        # Windows specific
+        msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
+
+def release_locked_file(fd, path):
+    """
+    Remove, close, and release a lock file specified by its file descriptor and its path.
+
+    :param int fd: The file descriptor of the lock file.
+    :param str path: The path of the lock file.
+
+    """
+    # Linux specific
+    #
+    # It is important the lock file is removed before it's released,
+    # otherwise:
+    #
+    # process A: open lock file
+    # process B: release lock file
+    # process A: lock file
+    # process A: check device and inode
+    # process B: delete file
+    # process C: open and lock a different file at the same path
+    try:
+        os.remove(path)
+    except OSError as err:
+        if err.errno == errno.EACCES:
+            # Windows specific
+            # We will not be able to remove a file before closing it.
+            # To avoid race conditions described for Linux, we will not delete the lockfile,
+            # just close it to be reused on the next Certbot call.
+            pass
+        else:
+            raise
+    finally:
+        os.close(fd)
+
+def compare_file_modes(mode1, mode2):
+    """Return true if the two modes can be considered as equals for this platform"""
+    if 'fcntl' in sys.modules:
+        # Linux specific: standard compare
+        return oct(stat.S_IMODE(mode1)) == oct(stat.S_IMODE(mode2))
+    # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights.
+    return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD
+            and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE)

certbot/crypto_util.py

@@ -25,6 +25,7 @@ from OpenSSL import SSL  # type: ignore
 
 from acme import crypto_util as acme_crypto_util
 from acme.magic_typing import IO  # pylint: disable=unused-import, no-name-in-module
+from certbot import compat
 from certbot import errors
 from certbot import interfaces
 from certbot import util
@@ -60,7 +61,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"):
 
     config = zope.component.getUtility(interfaces.IConfig)
     # Save file
-    util.make_or_verify_dir(key_dir, 0o700, os.geteuid(),
+    util.make_or_verify_dir(key_dir, 0o700, compat.os_geteuid(),
                             config.strict_permissions)
     key_f, key_path = util.unique_file(
         os.path.join(key_dir, keyname), 0o600, "wb")
@@ -91,7 +92,7 @@ def init_save_csr(privkey, names, path):
         privkey.pem, names, must_staple=config.must_staple)
 
     # Save CSR
-    util.make_or_verify_dir(path, 0o755, os.geteuid(),
+    util.make_or_verify_dir(path, 0o755, compat.os_geteuid(),
                                config.strict_permissions)
     csr_f, csr_filename = util.unique_file(
         os.path.join(path, "csr-certbot.pem"), 0o644, "wb")
@@ -448,14 +449,17 @@ def _notAfterBefore(cert_path, method):
 def sha256sum(filename):
     """Compute a sha256sum of a file.
 
+    NB: In given file, platform specific newlines characters will be converted
+    into their equivalent unicode counterparts before calculating the hash.
+
     :param str filename: path to the file whose hash will be computed
 
     :returns: sha256 digest of the file in hexadecimal
     :rtype: str
     """
     sha256 = hashlib.sha256()
-    with open(filename, 'rb') as f:
-        sha256.update(f.read())
+    with open(filename, 'rU') as file_d:
+        sha256.update(file_d.read().encode('UTF-8'))
     return sha256.hexdigest()
 
 def cert_and_chain_from_fullchain(fullchain_pem):

certbot/display/completer.py

@@ -49,9 +49,9 @@ class Completer(object):
         readline.set_completer(self.complete)
         readline.set_completer_delims(' \t\n;')
 
-        # readline can be implemented using GNU readline or libedit
+        # readline can be implemented using GNU readline, pyreadline or libedit
         # which have different configuration syntax
-        if 'libedit' in readline.__doc__:
+        if readline.__doc__ is not None and 'libedit' in readline.__doc__:
             readline.parse_and_bind('bind ^I rl_complete')
         else:
             readline.parse_and_bind('tab: complete')

certbot/display/util.py

@@ -1,12 +1,12 @@
 """Certbot display."""
 import logging
 import os
-import select
 import sys
 import textwrap
 
 import zope.interface
 
+from certbot import compat
 from certbot import constants
 from certbot import interfaces
 from certbot import errors
@@ -53,7 +53,7 @@ def _wrap_lines(msg):
             break_long_words=False,
             break_on_hyphens=False))
 
-    return os.linesep.join(fixed_l)
+    return '\n'.join(fixed_l)
 
 
 def input_with_timeout(prompt=None, timeout=36000.0):
@@ -79,13 +79,8 @@ def input_with_timeout(prompt=None, timeout=36000.0):
         sys.stdout.write(prompt)
         sys.stdout.flush()
 
-    # select can only be used like this on UNIX
-    rlist, _, _ = select.select([sys.stdin], [], [], timeout)
-    if not rlist:
-        raise errors.Error(
-            "Timed out waiting for answer to prompt '{0}'".format(prompt))
+    line = compat.readline_with_timeout(timeout, prompt)
 
-    line = rlist[0].readline()
     if not line:
         raise EOFError
     return line.rstrip('\n')

certbot/lock.py

@@ -1,9 +1,9 @@
 """Implements file locks for locking files and directories in UNIX."""
 import errno
-import fcntl
 import logging
 import os
 
+from certbot import compat
 from certbot import errors
 
 logger = logging.getLogger(__name__)
@@ -74,7 +74,7 @@ class LockFile(object):
 
         """
         try:
-            fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            compat.lock_file(fd)
         except IOError as err:
             if err.errno in (errno.EACCES, errno.EAGAIN):
                 logger.debug(
@@ -118,22 +118,7 @@ class LockFile(object):
 
     def release(self):
         """Remove, close, and release the lock file."""
-        # It is important the lock file is removed before it's released,
-        # otherwise:
-        #
-        # process A: open lock file
-        # process B: release lock file
-        # process A: lock file
-        # process A: check device and inode
-        # process B: delete file
-        # process C: open and lock a different file at the same path
-        #
-        # Calling os.remove on a file that's in use doesn't work on
-        # Windows, but neither does locking with fcntl.
         try:
-            os.remove(self._path)
+            compat.release_locked_file(self._fd, self._path)
         finally:
-            try:
-                os.close(self._fd)
-            finally:
-                self._fd = None
+            self._fd = None

certbot/log.py

@@ -23,6 +23,7 @@ import traceback
 
 from acme import messages
 
+from certbot import compat
 from certbot import constants
 from certbot import errors
 from certbot import util
@@ -133,7 +134,7 @@ def setup_log_file_handler(config, logfile, fmt):
     # TODO: logs might contain sensitive data such as contents of the
     # private key! #525
     util.set_up_core_dir(
-        config.logs_dir, 0o700, os.geteuid(), config.strict_permissions)
+        config.logs_dir, 0o700, compat.os_geteuid(), config.strict_permissions)
     log_file_path = os.path.join(config.logs_dir, logfile)
     try:
         handler = logging.handlers.RotatingFileHandler(

certbot/main.py

@@ -19,6 +19,7 @@ from certbot import account
 from certbot import cert_manager
 from certbot import cli
 from certbot import client
+from certbot import compat
 from certbot import configuration
 from certbot import constants
 from certbot import crypto_util
@@ -531,8 +532,7 @@ def _determine_account(config):
 
 def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-branches
     """Does the user want to delete their now-revoked certs? If run in non-interactive mode,
-    deleting happens automatically, unless if both `--cert-name` and `--cert-path` were
-    specified with conflicting values.
+    deleting happens automatically.
 
     :param config: parsed command line arguments
     :type config: interfaces.IConfig
@@ -556,50 +556,13 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b
         reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY)
         return
 
-    if not (config.certname or config.cert_path):
-        raise errors.Error('At least one of --cert-path or --cert-name must be specified.')
+    # config.cert_path must have been set
+    # config.certname may have been set
+    assert config.cert_path
 
-    if config.certname and config.cert_path:
-        # first, check if certname and cert_path imply the same certs
-        implied_cert_name = cert_manager.cert_path_to_lineage(config)
-
-        if implied_cert_name != config.certname:
-            cert_path_implied_cert_name = cert_manager.cert_path_to_lineage(config)
-            cert_path_implied_conf = storage.renewal_file_for_certname(config,
-                    cert_path_implied_cert_name)
-            cert_path_cert = storage.RenewableCert(cert_path_implied_conf, config)
-            cert_path_info = cert_manager.human_readable_cert_info(config, cert_path_cert,
-                    skip_filter_checks=True)
-
-            cert_name_implied_conf = storage.renewal_file_for_certname(config, config.certname)
-            cert_name_cert = storage.RenewableCert(cert_name_implied_conf, config)
-            cert_name_info = cert_manager.human_readable_cert_info(config, cert_name_cert)
-
-            msg = ("You specified conflicting values for --cert-path and --cert-name. "
-                    "Which did you mean to select?")
-            choices = [cert_path_info, cert_name_info]
-            try:
-                code, index = display.menu(msg,
-                        choices, ok_label="Select", force_interactive=True)
-            except errors.MissingCommandlineFlag:
-                error_msg = ('To run in non-interactive mode, you must either specify only one of '
-                '--cert-path or --cert-name, or both must point to the same certificate lineages.')
-                raise errors.Error(error_msg)
-
-            if code != display_util.OK or not index in range(0, len(choices)):
-                raise errors.Error("User ended interaction.")
-
-            if index == 0:
-                config.certname = cert_path_implied_cert_name
-            else:
-                config.cert_path = storage.cert_path_for_cert_name(config, config.certname)
-
-    elif config.cert_path:
+    if not config.certname:
         config.certname = cert_manager.cert_path_to_lineage(config)
 
-    else: # if only config.certname was specified
-        config.cert_path = storage.cert_path_for_cert_name(config, config.certname)
-
     # don't delete if the archive_dir is used by some other lineage
     archive_dir = storage.full_archive_path(
             configobj.ConfigObj(storage.renewal_file_for_certname(config, config.certname)),
@@ -1065,6 +1028,14 @@ def revoke(config, unused_plugins):  # TODO: coop with renewal config
     """
     # For user-agent construction
     config.installer = config.authenticator = None
+
+    if config.cert_path is None and config.certname:
+        config.cert_path = storage.cert_path_for_cert_name(config, config.certname)
+    elif not config.cert_path or (config.cert_path and config.certname):
+        # intentionally not supporting --cert-path & --cert-name together,
+        # to avoid dealing with mismatched values
+        raise errors.Error("Error! Exactly one of --cert-path or --cert-name must be specified!")
+
     if config.key_path is not None:  # revocation by cert key
         logger.debug("Revoking %s using cert key %s",
                      config.cert_path[0], config.key_path[0])
@@ -1077,7 +1048,6 @@ def revoke(config, unused_plugins):  # TODO: coop with renewal config
         acme = client.acme_from_config_key(config, acc.key, acc.regr)
     cert = crypto_util.pyopenssl_load_certificate(config.cert_path[1])[0]
     logger.debug("Reason code for revocation: %s", config.reason)
-
     try:
         acme.revoke(jose.ComparableX509(cert), config.reason)
         _delete_if_appropriate(config)
@@ -1165,7 +1135,8 @@ def _csr_get_and_save_cert(config, le_client):
             "Dry run: skipping saving certificate to %s", config.cert_path)
         return None, None
     cert_path, _, fullchain_path = le_client.save_certificate(
-            cert, chain, config.cert_path, config.chain_path, config.fullchain_path)
+        cert, chain, os.path.normpath(config.cert_path),
+        os.path.normpath(config.chain_path), os.path.normpath(config.fullchain_path))
     return cert_path, fullchain_path
 
 def renew_cert(config, plugins, lineage):
@@ -1289,16 +1260,16 @@ def make_or_verify_needed_dirs(config):
 
     """
     util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE,
-                         os.geteuid(), config.strict_permissions)
+                         compat.os_geteuid(), config.strict_permissions)
     util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE,
-                         os.geteuid(), config.strict_permissions)
+                         compat.os_geteuid(), config.strict_permissions)
 
     hook_dirs = (config.renewal_pre_hooks_dir,
                  config.renewal_deploy_hooks_dir,
                  config.renewal_post_hooks_dir,)
     for hook_dir in hook_dirs:
         util.make_or_verify_dir(hook_dir,
-                                uid=os.geteuid(),
+                                uid=compat.os_geteuid(),
                                 strict=config.strict_permissions)
 
 
@@ -1333,6 +1304,7 @@ def main(cli_args=sys.argv[1:]):
     :raises errors.Error: error if plugin command is not supported
 
     """
+
     log.pre_arg_parse_setup()
 
     plugins = plugins_disco.PluginsRegistry.find_all()
@@ -1346,6 +1318,10 @@ def main(cli_args=sys.argv[1:]):
     config = configuration.NamespaceConfig(args)
     zope.component.provideUtility(config)
 
+    # On windows, shell without administrative right cannot create symlinks required by certbot.
+    # So we check the rights before continuing.
+    compat.raise_for_non_administrative_windows_rights(config.verb)
+
     try:
         log.post_arg_parse_setup(config)
         make_or_verify_needed_dirs(config)

certbot/plugins/manual.py

@@ -94,6 +94,16 @@ using the secret key
 {key}
 when it receives a TLS ClientHello with the SNI extension set to
 {sni_domain}
+"""
+    _SUBSEQUENT_CHALLENGE_INSTRUCTIONS = """
+(This must be set up in addition to the previous challenges; do not remove,
+replace, or undo the previous challenge tasks yet.)
+"""
+    _SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS = """
+(This must be set up in addition to the previous challenges; do not remove,
+replace, or undo the previous challenge tasks yet. Note that you might be
+asked to create multiple distinct TXT records with the same name. This is
+permitted by DNS standards.)
 """
 
     def __init__(self, *args, **kwargs):
@@ -103,6 +113,8 @@ when it receives a TLS ClientHello with the SNI extension set to
         self.env = dict() \
         # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]]
         self.tls_sni_01 = None
+        self.subsequent_dns_challenge = False
+        self.subsequent_any_challenge = False
 
     @classmethod
     def add_parser_arguments(cls, add):
@@ -212,8 +224,17 @@ when it receives a TLS ClientHello with the SNI extension set to
                 key=self.tls_sni_01.get_key_path(achall),
                 port=self.config.tls_sni_01_port,
                 sni_domain=self.tls_sni_01.get_z_domain(achall))
+        if isinstance(achall.chall, challenges.DNS01):
+            if self.subsequent_dns_challenge:
+                # 2nd or later dns-01 challenge
+                msg += self._SUBSEQUENT_DNS_CHALLENGE_INSTRUCTIONS
+            self.subsequent_dns_challenge = True
+        elif self.subsequent_any_challenge:
+            # 2nd or later challenge of another type
+            msg += self._SUBSEQUENT_CHALLENGE_INSTRUCTIONS
         display = zope.component.getUtility(interfaces.IDisplay)
         display.notification(msg, wrap=False, force_interactive=True)
+        self.subsequent_any_challenge = True
 
     def cleanup(self, achalls):  # pylint: disable=missing-docstring
         if self.conf('cleanup-hook'):

certbot/plugins/manual_test.py

@@ -4,6 +4,7 @@ import unittest
 
 import six
 import mock
+import sys
 
 from acme import challenges
 
@@ -20,8 +21,9 @@ class AuthenticatorTest(test_util.TempDirTestCase):
         super(AuthenticatorTest, self).setUp()
         self.http_achall = acme_util.HTTP01_A
         self.dns_achall = acme_util.DNS01_A
+        self.dns_achall_2 = acme_util.DNS01_A_2
         self.tls_sni_achall = acme_util.TLSSNI01_A
-        self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall]
+        self.achalls = [self.http_achall, self.dns_achall, self.tls_sni_achall, self.dns_achall_2]
         for d in ["config_dir", "work_dir", "in_progress"]:
             os.mkdir(os.path.join(self.tempdir, d))
             # "backup_dir" and "temp_checkpoint_dir" get created in
@@ -74,12 +76,14 @@ class AuthenticatorTest(test_util.TempDirTestCase):
     def test_script_perform(self):
         self.config.manual_public_ip_logging_ok = True
         self.config.manual_auth_hook = (
-            'echo ${CERTBOT_DOMAIN}; '
-            'echo ${CERTBOT_TOKEN:-notoken}; '
-            'echo ${CERTBOT_CERT_PATH:-nocert}; '
-            'echo ${CERTBOT_KEY_PATH:-nokey}; '
-            'echo ${CERTBOT_SNI_DOMAIN:-nosnidomain}; '
-            'echo ${CERTBOT_VALIDATION:-novalidation};')
+            '{0} -c "from __future__ import print_function;'
+            'import os;  print(os.environ.get(\'CERTBOT_DOMAIN\'));'
+            'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));'
+            'print(os.environ.get(\'CERTBOT_CERT_PATH\', \'nocert\'));'
+            'print(os.environ.get(\'CERTBOT_KEY_PATH\', \'nokey\'));'
+            'print(os.environ.get(\'CERTBOT_SNI_DOMAIN\', \'nosnidomain\'));'
+            'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"'
+            .format(sys.executable))
         dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}'.format(
             self.dns_achall.domain, 'notoken',
             'nocert', 'nokey', 'nosnidomain',
@@ -127,6 +131,7 @@ class AuthenticatorTest(test_util.TempDirTestCase):
                         achall.validation(achall.account_key) in args[0])
             self.assertFalse(kwargs['wrap'])
 
+    @test_util.broken_on_windows
     def test_cleanup(self):
         self.config.manual_public_ip_logging_ok = True
         self.config.manual_auth_hook = 'echo foo;'

certbot/plugins/util.py

@@ -9,18 +9,19 @@ logger = logging.getLogger(__name__)
 def get_prefixes(path):
     """Retrieves all possible path prefixes of a path, in descending order
     of length. For instance,
-        /a/b/c/ => ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/']
+        (linux) /a/b/c returns ['/a/b/c', '/a/b', '/a', '/']
+        (windows) C:\\a\\b\\c returns ['C:\\a\\b\\c', 'C:\\a\\b', 'C:\\a', 'C:']
     :param str path: the path to break into prefixes
 
     :returns: all possible path prefixes of given path in descending order
     :rtype: `list` of `str`
     """
-    prefix = path
+    prefix = os.path.normpath(path)
     prefixes = []
     while len(prefix) > 0:
         prefixes.append(prefix)
         prefix, _ = os.path.split(prefix)
-        # break once we hit '/'
+        # break once we hit the root path
         if prefix == prefixes[-1]:
             break
     return prefixes
@@ -51,6 +52,6 @@ def path_surgery(cmd):
         return True
     else:
         expanded = " expanded" if any(added) else ""
-        logger.warning("Failed to find executable %s in%s PATH: %s", cmd,
-                       expanded, path)
+        logger.debug("Failed to find executable %s in%s PATH: %s", cmd,
+                     expanded, path)
         return False

certbot/plugins/util_test.py

@@ -4,21 +4,21 @@ import unittest
 
 import mock
 
-
 class GetPrefixTest(unittest.TestCase):
     """Tests for certbot.plugins.get_prefixes."""
     def test_get_prefix(self):
         from certbot.plugins.util import get_prefixes
-        self.assertEqual(get_prefixes("/a/b/c/"), ['/a/b/c/', '/a/b/c', '/a/b', '/a', '/'])
-        self.assertEqual(get_prefixes("/"), ["/"])
-        self.assertEqual(get_prefixes("a"), ["a"])
+        self.assertEqual(
+            get_prefixes('/a/b/c'),
+            [os.path.normpath(path) for path in ['/a/b/c', '/a/b', '/a', '/']])
+        self.assertEqual(get_prefixes('/'), [os.path.normpath('/')])
+        self.assertEqual(get_prefixes('a'), ['a'])
 
 class PathSurgeryTest(unittest.TestCase):
     """Tests for certbot.plugins.path_surgery."""
 
-    @mock.patch("certbot.plugins.util.logger.warning")
     @mock.patch("certbot.plugins.util.logger.debug")
-    def test_path_surgery(self, mock_debug, mock_warn):
+    def test_path_surgery(self, mock_debug):
         from certbot.plugins.util import path_surgery
         all_path = {"PATH": "/usr/local/bin:/bin/:/usr/sbin/:/usr/local/sbin/"}
         with mock.patch.dict('os.environ', all_path):
@@ -26,14 +26,12 @@ class PathSurgeryTest(unittest.TestCase):
                 mock_exists.return_value = True
                 self.assertEqual(path_surgery("eg"), True)
                 self.assertEqual(mock_debug.call_count, 0)
-                self.assertEqual(mock_warn.call_count, 0)
                 self.assertEqual(os.environ["PATH"], all_path["PATH"])
         no_path = {"PATH": "/tmp/"}
         with mock.patch.dict('os.environ', no_path):
             path_surgery("thingy")
-            self.assertEqual(mock_debug.call_count, 1)
-            self.assertEqual(mock_warn.call_count, 1)
-            self.assertTrue("Failed to find" in mock_warn.call_args[0][0])
+            self.assertEqual(mock_debug.call_count, 2)
+            self.assertTrue("Failed to find" in mock_debug.call_args[0][0])
             self.assertTrue("/usr/local/bin" in os.environ["PATH"])
             self.assertTrue("/tmp" in os.environ["PATH"])
 

certbot/plugins/webroot.py

@@ -170,7 +170,9 @@ to serve all files under specified web root ({0})."""
             old_umask = os.umask(0o022)
             try:
                 stat_path = os.stat(path)
-                for prefix in sorted(util.get_prefixes(self.full_roots[name]), key=len):
+                # We ignore the last prefix in the next iteration,
+                # as it does not correspond to a folder path ('/' or 'C:')
+                for prefix in sorted(util.get_prefixes(self.full_roots[name])[:-1], key=len):
                     try:
                         # This is coupled with the "umask" call above because
                         # os.mkdir's "mode" parameter may not always work:
@@ -180,7 +182,7 @@ to serve all files under specified web root ({0})."""
                         # Set owner as parent directory if possible
                         try:
                             os.chown(prefix, stat_path.st_uid, stat_path.st_gid)
-                        except OSError as exception:
+                        except (OSError, AttributeError) as exception:
                             logger.info("Unable to change owner and uid of webroot directory")
                             logger.debug("Error was: %s", exception)
                     except OSError as exception:

certbot/plugins/webroot_test.py

@@ -4,9 +4,9 @@ from __future__ import print_function
 
 import argparse
 import errno
+import json
 import os
 import shutil
-import stat
 import tempfile
 import unittest
 
@@ -17,6 +17,7 @@ import six
 from acme import challenges
 
 from certbot import achallenges
+from certbot import compat
 from certbot import errors
 from certbot.display import util as display_util
 
@@ -142,6 +143,7 @@ class AuthenticatorTest(unittest.TestCase):
             self.assertRaises(errors.PluginError, self.auth.perform, [])
         os.chmod(self.path, 0o700)
 
+    @test_util.skip_on_windows('On Windows, there is no chown.')
     @mock.patch("certbot.plugins.webroot.os.chown")
     def test_failed_chown(self, mock_chown):
         mock_chown.side_effect = OSError(errno.EACCES, "msg")
@@ -169,16 +171,14 @@ class AuthenticatorTest(unittest.TestCase):
         # Remove exec bit from permission check, so that it
         # matches the file
         self.auth.perform([self.achall])
-        path_permissions = stat.S_IMODE(os.stat(self.validation_path).st_mode)
-        self.assertEqual(path_permissions, 0o644)
+        self.assertTrue(compat.compare_file_modes(os.stat(self.validation_path).st_mode, 0o644))
 
         # Check permissions of the directories
 
         for dirpath, dirnames, _ in os.walk(self.path):
             for directory in dirnames:
                 full_path = os.path.join(dirpath, directory)
-                dir_permissions = stat.S_IMODE(os.stat(full_path).st_mode)
-                self.assertEqual(dir_permissions, 0o755)
+                self.assertTrue(compat.compare_file_modes(os.stat(full_path).st_mode, 0o755))
 
         parent_gid = os.stat(self.path).st_gid
         parent_uid = os.stat(self.path).st_uid
@@ -274,7 +274,7 @@ class WebrootActionTest(unittest.TestCase):
 
     def test_webroot_map_action(self):
         args = self.parser.parse_args(
-            ["--webroot-map", '{{"thing.com":"{0}"}}'.format(self.path)])
+            ["--webroot-map", json.dumps({'thing.com': self.path})])
         self.assertEqual(args.webroot_map["thing.com"], self.path)
 
     def test_domain_before_webroot(self):

certbot/renewal.py

@@ -301,7 +301,7 @@ def renew_cert(config, domains, le_client, lineage):
         domains = lineage.names()
     # The private key is the existing lineage private key if reuse_key is set.
     # Otherwise, generate a fresh private key by passing None.
-    new_key = lineage.privkey if config.reuse_key else None
+    new_key = os.path.normpath(lineage.privkey) if config.reuse_key else None
     new_cert, new_chain, new_key, _ = le_client.obtain_certificate(domains, new_key)
     if config.dry_run:
         logger.debug("Dry run: skipping updating lineage at %s",
@@ -359,7 +359,7 @@ def _renew_describe_results(config, renew_successes, renew_failures,
         notify_error(report(renew_failures, "failure"))
 
     if parse_failures:
-        notify("\nAdditionally, the following renewal configuration files "
+        notify("\nAdditionally, the following renewal configurations "
                "were invalid: ")
         notify(report(parse_failures, "parsefail"))
 

certbot/reverter.py

@@ -10,6 +10,7 @@ import traceback
 import six
 import zope.component
 
+from certbot import compat
 from certbot import constants
 from certbot import errors
 from certbot import interfaces
@@ -65,7 +66,7 @@ class Reverter(object):
         self.config = config
 
         util.make_or_verify_dir(
-            config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
+            config.backup_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(),
             self.config.strict_permissions)
 
     def revert_temporary_config(self):
@@ -219,7 +220,7 @@ class Reverter(object):
 
         """
         util.make_or_verify_dir(
-            cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
+            cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(),
             self.config.strict_permissions)
 
         op_fd, existing_filepaths = self._read_and_append(
@@ -433,7 +434,7 @@ class Reverter(object):
             cp_dir = self.config.in_progress_dir
 
         util.make_or_verify_dir(
-            cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(),
+            cp_dir, constants.CONFIG_DIRS_MODE, compat.os_geteuid(),
             self.config.strict_permissions)
 
         return cp_dir

certbot/storage.py

@@ -214,6 +214,26 @@ def get_link_target(link):
         target = os.path.join(os.path.dirname(link), target)
     return os.path.abspath(target)
 
+def _write_live_readme_to(readme_path, is_base_dir=False):
+    prefix = ""
+    if is_base_dir:
+        prefix = "[cert name]/"
+    with open(readme_path, "w") as f:
+        logger.debug("Writing README to %s.", readme_path)
+        f.write("This directory contains your keys and certificates.\n\n"
+                "`{prefix}privkey.pem`  : the private key for your certificate.\n"
+                "`{prefix}fullchain.pem`: the certificate file used in most server software.\n"
+                "`{prefix}chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.\n"
+                "`{prefix}cert.pem`     : will break many server configurations, and "
+                                    "should not be used\n"
+                "                 without reading further documentation (see link below).\n\n"
+                "WARNING: DO NOT MOVE OR RENAME THESE FILES!\n"
+                "         Certbot expects these files to remain in this location in order\n"
+                "         to function properly!\n\n"
+                "We recommend not moving these files. For more information, see the Certbot\n"
+                "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-"
+                                    "certificates.\n".format(prefix=prefix))
+
 
 def _relevant(option):
     """
@@ -1003,6 +1023,9 @@ class RenewableCert(object):
                 logger.debug("Creating directory %s.", i)
         config_file, config_filename = util.unique_lineage_name(
             cli_config.renewal_configs_dir, lineagename)
+        base_readme_path = os.path.join(cli_config.live_dir, README)
+        if not os.path.exists(base_readme_path):
+            _write_live_readme_to(base_readme_path, is_base_dir=True)
 
         # Determine where on disk everything will go
         # lineagename will now potentially be modified based on which
@@ -1045,21 +1068,7 @@ class RenewableCert(object):
 
         # Write a README file to the live directory
         readme_path = os.path.join(live_dir, README)
-        with open(readme_path, "w") as f:
-            logger.debug("Writing README to %s.", readme_path)
-            f.write("This directory contains your keys and certificates.\n\n"
-                    "`privkey.pem`  : the private key for your certificate.\n"
-                    "`fullchain.pem`: the certificate file used in most server software.\n"
-                    "`chain.pem`    : used for OCSP stapling in Nginx >=1.3.7.\n"
-                    "`cert.pem`     : will break many server configurations, and "
-                                        "should not be used\n"
-                    "                 without reading further documentation (see link below).\n\n"
-                    "WARNING: DO NOT MOVE THESE FILES!\n"
-                    "         Certbot expects these files to remain in this location in order\n"
-                    "         to function properly!\n\n"
-                    "We recommend not moving these files. For more information, see the Certbot\n"
-                    "User Guide at https://certbot.eff.org/docs/using.html#where-are-my-"
-                                        "certificates.\n")
+        _write_live_readme_to(readme_path)
 
         # Document what we've done in a new renewal config file
         config_file.close()

certbot/tests/account_test.py

@@ -116,6 +116,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
     def test_init_creates_dir(self):
         self.assertTrue(os.path.isdir(self.config.accounts_dir))
 
+    @test_util.broken_on_windows
     def test_save_and_restore(self):
         self.storage.save(self.acc, self.mock_client)
         account_path = os.path.join(self.config.accounts_dir, self.acc.id)
@@ -218,12 +219,14 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.assertEqual([], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_version_staging(self):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertEqual([self.acc], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_version_production(self):
         self._set_server('https://acme-v01.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
@@ -241,6 +244,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertEqual([], self.storage.find_all())
 
+    @test_util.broken_on_windows
     def test_upgrade_load(self):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
@@ -249,6 +253,15 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         account = self.storage.load(self.acc.id)
         self.assertEqual(prev_account, account)
 
+    @test_util.broken_on_windows
+    def test_upgrade_load_single_account(self):
+        self._set_server('https://acme-staging.api.letsencrypt.org/directory')
+        self.storage.save(self.acc, self.mock_client)
+        prev_account = self.storage.load(self.acc.id)
+        self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory')
+        account = self.storage.load(self.acc.id)
+        self.assertEqual(prev_account, account)
+
     def test_load_ioerror(self):
         self.storage.save(self.acc, self.mock_client)
         mock_open = mock.mock_open()
@@ -265,6 +278,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
                 errors.AccountStorageError, self.storage.save,
                     self.acc, self.mock_client)
 
+    @test_util.broken_on_windows
     def test_delete(self):
         self.storage.save(self.acc, self.mock_client)
         self.storage.delete(self.acc.id)
@@ -287,7 +301,7 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging.api.letsencrypt.org/directory')
         self.storage.save(self.acc, self.mock_client)
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
-        self.storage.find_all()
+        self.storage.load(self.acc.id)
 
         # delete starting at given server_url
         self._set_server(server_url)
@@ -299,14 +313,30 @@ class AccountFileStorageTest(test_util.ConfigTestCase):
         self._set_server('https://acme-staging-v02.api.letsencrypt.org/directory')
         self.assertRaises(errors.AccountNotFound, self.storage.load, self.acc.id)
 
+    @test_util.broken_on_windows
     def test_delete_folders_up(self):
         self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory')
         self._assert_symlinked_account_removed()
 
+    @test_util.broken_on_windows
     def test_delete_folders_down(self):
         self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory')
         self._assert_symlinked_account_removed()
 
+    def _set_server_and_stop_symlink(self, server_path):
+        self._set_server(server_path)
+        with open(os.path.join(self.config.accounts_dir, 'foo'), 'w') as f:
+            f.write('bar')
+
+    @test_util.broken_on_windows
+    def test_delete_shared_account_up(self):
+        self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory')
+        self._test_delete_folders('https://acme-staging.api.letsencrypt.org/directory')
+
+    @test_util.broken_on_windows
+    def test_delete_shared_account_down(self):
+        self._set_server_and_stop_symlink('https://acme-staging-v02.api.letsencrypt.org/directory')
+        self._test_delete_folders('https://acme-staging-v02.api.letsencrypt.org/directory')
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

certbot/tests/acme_util.py

@@ -21,6 +21,7 @@ HTTP01 = challenges.HTTP01(
 TLSSNI01 = challenges.TLSSNI01(
     token=jose.b64decode(b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJyPCt92wrDoA"))
 DNS01 = challenges.DNS01(token=b"17817c66b60ce2e4012dfad92657527a")
+DNS01_2 = challenges.DNS01(token=b"cafecafecafecafecafecafe0feedbac")
 
 CHALLENGES = [HTTP01, TLSSNI01, DNS01]
 
@@ -49,6 +50,7 @@ def chall_to_challb(chall, status):  # pylint: disable=redefined-outer-name
 TLSSNI01_P = chall_to_challb(TLSSNI01, messages.STATUS_PENDING)
 HTTP01_P = chall_to_challb(HTTP01, messages.STATUS_PENDING)
 DNS01_P = chall_to_challb(DNS01, messages.STATUS_PENDING)
+DNS01_P_2 = chall_to_challb(DNS01_2, messages.STATUS_PENDING)
 
 CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P]
 
@@ -57,6 +59,7 @@ CHALLENGES_P = [HTTP01_P, TLSSNI01_P, DNS01_P]
 HTTP01_A = auth_handler.challb_to_achall(HTTP01_P, JWK, "example.com")
 TLSSNI01_A = auth_handler.challb_to_achall(TLSSNI01_P, JWK, "example.net")
 DNS01_A = auth_handler.challb_to_achall(DNS01_P, JWK, "example.org")
+DNS01_A_2 = auth_handler.challb_to_achall(DNS01_P_2, JWK, "esimerkki.example.org")
 
 ACHALLENGES = [HTTP01_A, TLSSNI01_A, DNS01_A]
 

certbot/tests/cert_manager_test.py

@@ -204,7 +204,7 @@ class CertificatesTest(BaseCertManagerTest):
         shutil.rmtree(empty_tempdir)
 
     @mock.patch('certbot.cert_manager.ocsp.RevocationChecker.ocsp_revoked')
-    def test_report_human_readable(self, mock_revoked):
+    def test_report_human_readable(self, mock_revoked): #pylint: disable=too-many-statements
         mock_revoked.return_value = None
         from certbot import cert_manager
         import datetime, pytz
@@ -228,7 +228,7 @@ class CertificatesTest(BaseCertManagerTest):
         cert.target_expiry += datetime.timedelta(hours=2)
         # pylint: disable=protected-access
         out = get_report()
-        self.assertTrue('1 hour(s)' in out)
+        self.assertTrue('1 hour(s)' in out or '2 hour(s)' in out)
         self.assertTrue('VALID' in out and not 'INVALID' in out)
 
         cert.target_expiry += datetime.timedelta(days=1)

certbot/tests/cli_test.py

@@ -76,6 +76,7 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
         return output.getvalue()
 
+    @test_util.broken_on_windows
     @mock.patch("certbot.cli.flag_default")
     def test_cli_ini_domains(self, mock_flag_default):
         tmp_config = tempfile.NamedTemporaryFile()

certbot/tests/configuration_test.py

@@ -48,18 +48,23 @@ class NamespaceConfigTest(test_util.ConfigTestCase):
         mock_constants.TEMP_CHECKPOINT_DIR = 't'
 
         self.assertEqual(
-            self.config.accounts_dir, os.path.join(
-                self.config.config_dir, 'acc/acme-server.org:443/new'))
+            os.path.normpath(self.config.accounts_dir),
+            os.path.normpath(os.path.join(self.config.config_dir, 'acc/acme-server.org:443/new')))
         self.assertEqual(
-                self.config.backup_dir, os.path.join(self.config.work_dir, 'backups'))
+            os.path.normpath(self.config.backup_dir),
+            os.path.normpath(os.path.join(self.config.work_dir, 'backups')))
         self.assertEqual(
-                self.config.csr_dir, os.path.join(self.config.config_dir, 'csr'))
+            os.path.normpath(self.config.csr_dir),
+            os.path.normpath(os.path.join(self.config.config_dir, 'csr')))
         self.assertEqual(
-                self.config.in_progress_dir, os.path.join(self.config.work_dir, '../p'))
+            os.path.normpath(self.config.in_progress_dir),
+            os.path.normpath(os.path.join(self.config.work_dir, '../p')))
         self.assertEqual(
-                self.config.key_dir, os.path.join(self.config.config_dir, 'keys'))
+            os.path.normpath(self.config.key_dir),
+            os.path.normpath(os.path.join(self.config.config_dir, 'keys')))
         self.assertEqual(
-                self.config.temp_checkpoint_dir, os.path.join(self.config.work_dir, 't'))
+            os.path.normpath(self.config.temp_checkpoint_dir),
+            os.path.normpath(os.path.join(self.config.work_dir, 't')))
 
     def test_absolute_paths(self):
         from certbot.configuration import NamespaceConfig

certbot/tests/crypto_util_test.py

@@ -140,7 +140,7 @@ class ImportCSRFileTest(unittest.TestCase):
              util.CSR(file=csrfile,
                       data=data_pem,
                       form="pem"),
-             ["Example.com"],),
+             ["Example.com"]),
             self._call(csrfile, data))
 
     def test_pem_csr(self):
@@ -376,7 +376,6 @@ class NotAfterTest(unittest.TestCase):
 
 class Sha256sumTest(unittest.TestCase):
     """Tests for certbot.crypto_util.notAfter"""
-
     def test_sha256sum(self):
         from certbot.crypto_util import sha256sum
         self.assertEqual(sha256sum(CERT_PATH),

certbot/tests/display/completer_test.py

@@ -1,6 +1,9 @@
 """Test certbot.display.completer."""
 import os
-import readline
+try:
+    import readline # pylint: disable=import-error
+except ImportError:
+    import certbot.display.dummy_readline as readline # type: ignore
 import string
 import sys
 import unittest
@@ -9,9 +12,9 @@ import mock
 from six.moves import reload_module  # pylint: disable=import-error
 
 from acme.magic_typing import List  # pylint: disable=unused-import, no-name-in-module
-from certbot.tests.util import TempDirTestCase
+import certbot.tests.util as test_util
 
-class CompleterTest(TempDirTestCase):
+class CompleterTest(test_util.TempDirTestCase):
     """Test certbot.display.completer.Completer."""
 
     def setUp(self):
@@ -47,6 +50,8 @@ class CompleterTest(TempDirTestCase):
         completion = my_completer.complete(self.tempdir, num_paths)
         self.assertEqual(completion, None)
 
+    @unittest.skipIf('readline' not in sys.modules,
+                     reason='Not relevant if readline is not available.')
     def test_import_error(self):
         original_readline = sys.modules['readline']
         sys.modules['readline'] = None
@@ -91,7 +96,7 @@ class CompleterTest(TempDirTestCase):
 
 def enable_tab_completion(unused_command):
     """Enables readline tab completion using the system specific syntax."""
-    libedit = 'libedit' in readline.__doc__
+    libedit = readline.__doc__ is not None and 'libedit' in readline.__doc__
     command = 'bind ^I rl_complete' if libedit else 'tab: complete'
     readline.parse_and_bind(command)