Merge remote-tracking branch 'github/letsencrypt/master' into cli-config-fixes

Conflicts:
	letsencrypt/constants.py
	letsencrypt/interfaces.py

README.rst

@@ -1,7 +1,7 @@
 About the Let's Encrypt Client
 ==============================
 
-|build-status| |coverage| |docs|
+|build-status| |coverage| |docs| |container|
 
 In short: getting and installing SSL/TLS certificates made easy (`watch demo video`_).
 
@@ -43,6 +43,10 @@ server automatically!::
    :target: https://readthedocs.org/projects/letsencrypt/
    :alt: Documentation status
 
+.. |container| image:: https://quay.io/repository/letsencrypt/lets-encrypt-preview/status
+   :target: https://quay.io/repository/letsencrypt/lets-encrypt-preview
+   :alt: Docker Repository on Quay.io
+
 .. _watch demo video: https://www.youtube.com/watch?v=Gas_sSB-5SU
 
 

docs/api/reporter.rst

@@ -0,0 +1,5 @@
+:mod:`letsencrypt.reporter`
+---------------------------
+
+.. automodule:: letsencrypt.reporter
+   :members:

letsencrypt/cli.py

@@ -1,6 +1,7 @@
 """Let's Encrypt CLI."""
 # TODO: Sanity check all input.  Be sure to avoid shell code etc...
 import argparse
+import atexit
 import logging
 import os
 import sys
@@ -20,6 +21,7 @@ from letsencrypt import errors
 from letsencrypt import interfaces
 from letsencrypt import le_util
 from letsencrypt import log
+from letsencrypt import reporter
 
 from letsencrypt.display import util as display_util
 from letsencrypt.display import ops as display_ops
@@ -240,15 +242,28 @@ def create_parser(plugins):
     add("--version", action="version", version="%(prog)s {0}".format(
         letsencrypt.__version__))
     add("-v", "--verbose", dest="verbose_count", action="count",
-        default=flag_default("verbose_count"))
+        default=flag_default("verbose_count"), help="This flag can be used "
+        "multiple times to incrementally increase the verbosity of output, "
+        "e.g. -vvv.")
     add("--no-confirm", dest="no_confirm", action="store_true",
         help="Turn off confirmation screens, currently used for --revoke")
     add("-e", "--agree-tos", dest="tos", action="store_true",
         help="Skip the end user license agreement screen.")
     add("-t", "--text", dest="text_mode", action="store_true",
         help="Use the text output instead of the curses UI.")
-    add("--test-mode", action="store_true", help=config_help("test_mode"),
-        default=flag_default("test_mode"))
+
+    testing_group = parser.add_argument_group(
+        "testing", description="The following flags are meant for "
+        "testing purposes only! Do NOT change them, unless you "
+        "really know what you're doing!")
+    testing_group.add_argument(
+        "--no-verify-ssl", action="store_true",
+        help=config_help("no_verify_ssl"),
+        default=flag_default("no_verify_ssl"))
+    # TODO: apache and nginx plugins do NOT respect it
+    testing_group.add_argument(
+        "--dvsni-port", type=int, help=config_help("dvsni_port"),
+        default=flag_default("dvsni_port"))
 
     # positional arg shadows --domains, instead of appending, and
     # --domains is useful, because it can be stored in config
@@ -378,6 +393,11 @@ def main(args=sys.argv[1:]):
         displayer = display_util.NcursesDisplay()
     zope.component.provideUtility(displayer)
 
+    # Reporter
+    report = reporter.Reporter()
+    zope.component.provideUtility(report)
+    atexit.register(report.print_messages)
+
     # Logging
     level = -args.verbose_count * 10
     logger = logging.getLogger()

letsencrypt/client.py

@@ -64,7 +64,7 @@ class Client(object):
         # TODO: Allow for other alg types besides RS256
         self.network = network2.Network(
             config.server, jwk.JWKRSA.load(self.account.key.pem),
-            verify_ssl=(not config.test_mode))
+            verify_ssl=config.no_verify_ssl)
 
         self.config = config
 

letsencrypt/constants.py

@@ -15,7 +15,8 @@ CLI_DEFAULTS = dict(
     rollback_checkpoints=0,
     config_dir="/etc/letsencrypt",
     work_dir="/var/lib/letsencrypt",
-    test_mode=False,
+    no_verify_ssl=False,
+    dvsni_port=challenges.DVSNI.PORT,
 )
 """Defaults for CLI flags and `.IConfig` attributes."""
 
@@ -91,7 +92,3 @@ RENEWER_CONFIG_FILENAME = "renewer.conf"
 NETSTAT = "/bin/netstat"
 """Location of netstat binary for checking whether a listener is already
 running on the specified port (Linux-specific)."""
-
-BOULDER_TEST_MODE_CHALLENGE_PORT = 5001
-"""Port that Boulder will connect on for validations in test mode."""
-

letsencrypt/interfaces.py

@@ -177,8 +177,11 @@ class IConfig(zope.interface.Interface):
     renewer_config_file = zope.interface.Attribute(
         "Location of renewal configuration file.")
 
-    test_mode = zope.interface.Attribute(
-        "Test mode. Disables certificate verification.")
+    no_verify_ssl = zope.interface.Attribute(
+        "Disable SSL certificate verification.")
+    dvsni_port = zope.interface.Attribute(
+        "Port number to perform DVSNI challenge. "
+        "Boulder in testing mode defaults to 5001.")
 
 
 class IInstaller(IPlugin):
@@ -346,3 +349,30 @@ class IValidator(zope.interface.Interface):
 
     def hsts(name):
         """Verify HSTS header is enabled."""
+
+
+class IReporter(zope.interface.Interface):
+    """Interface to collect and display information to the user."""
+
+    HIGH_PRIORITY = zope.interface.Attribute(
+        "Used to denote high priority messages")
+    MEDIUM_PRIORITY = zope.interface.Attribute(
+        "Used to denote medium priority messages")
+    LOW_PRIORITY = zope.interface.Attribute(
+        "Used to denote low priority messages")
+
+    def add_message(self, msg, priority, on_crash=False):
+        """Adds msg to the list of messages to be printed.
+
+        :param str msg: Message to be displayed to the user.
+
+        :param int priority: One of HIGH_PRIORITY, MEDIUM_PRIORITY, or
+            LOW_PRIORITY.
+
+        :param bool on_crash: Whether or not the message should be printed if
+            the program exits abnormally.
+
+        """
+
+    def print_messages(self):
+        """Prints messages to the user and clears the message queue."""

letsencrypt/plugins/standalone/authenticator.py

@@ -15,7 +15,6 @@ import zope.interface
 from acme import challenges
 
 from letsencrypt import achallenges
-from letsencrypt import constants
 from letsencrypt import interfaces
 
 from letsencrypt.plugins import common
@@ -379,10 +378,8 @@ class StandaloneAuthenticator(common.Plugin):
                 results_if_failure.append(False)
         if not self.tasks:
             raise ValueError("nothing for .perform() to do")
-        port = challenges.DVSNI.PORT
-        if self.config and self.config.test_mode:
-            port = constants.BOULDER_TEST_MODE_CHALLENGE_PORT
-        if self.already_listening(port):
+
+        if self.already_listening(self.config.dvsni_port):
             # If we know a process is already listening on this port,
             # tell the user, and don't even attempt to bind it.  (This
             # test is Linux-specific and won't indicate that the port
@@ -390,7 +387,7 @@ class StandaloneAuthenticator(common.Plugin):
             return results_if_failure
         # Try to do the authentication; note that this creates
         # the listener subprocess via os.fork()
-        if self.start_listener(port, key):
+        if self.start_listener(self.config.dvsni_port, key):
             return results_if_success
         else:
             # TODO: This should probably raise a DVAuthError exception
@@ -427,8 +424,9 @@ class StandaloneAuthenticator(common.Plugin):
     def more_info(self):  # pylint: disable=no-self-use
         """Human-readable string that describes the Authenticator."""
         return ("The Standalone Authenticator uses PyOpenSSL to listen "
-                "on port 443 and perform DVSNI challenges. Once a certificate "
-                "is attained, it will be saved in the "
-                "(TODO) current working directory.{0}{0}"
-                "TCP port 443 must be available in order to use the "
-                "Standalone Authenticator.".format(os.linesep))
+                "on port {port} and perform DVSNI challenges. Once a "
+                "certificate is attained, it will be saved in the "
+                "(TODO) current working directory.{linesep}{linesep}"
+                "TCP port {port} must be available in order to use the "
+                "Standalone Authenticator.".format(
+                    linesep=os.linesep, port=self.config.dvsni_port))

letsencrypt/plugins/standalone/tests/authenticator_test.py

@@ -22,6 +22,7 @@ KEY = le_util.Key("foo", pkg_resources.resource_string(
     "acme.jose", os.path.join("testdata", "rsa512_key.pem")))
 PRIVATE_KEY = OpenSSL.crypto.load_privatekey(
     OpenSSL.crypto.FILETYPE_PEM, KEY.pem)
+CONFIG = mock.Mock(dvsni_port=5001)
 
 
 # Classes based on to allow interrupting infinite loop under test
@@ -59,7 +60,7 @@ class ChallPrefTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     def test_chall_pref(self):
         self.assertEqual(self.authenticator.get_chall_pref("example.com"),
@@ -71,7 +72,7 @@ class SNICallbackTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
         self.cert = achallenges.DVSNI(
             challb=acme_util.DVSNI_P,
             domain="example.com", key=KEY).gen_cert_and_response()[0]
@@ -109,7 +110,7 @@ class ClientSignalHandlerTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
         self.authenticator.tasks = {"foononce.acme.invalid": "stuff"}
         self.authenticator.child_pid = 12345
 
@@ -138,7 +139,7 @@ class SubprocSignalHandlerTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
         self.authenticator.tasks = {"foononce.acme.invalid": "stuff"}
         self.authenticator.child_pid = 12345
         self.authenticator.parent_pid = 23456
@@ -190,7 +191,7 @@ class AlreadyListeningTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     @mock.patch("letsencrypt.plugins.standalone.authenticator.psutil."
                 "net_connections")
@@ -293,7 +294,7 @@ class PerformTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
         self.achall1 = achallenges.DVSNI(
             challb=acme_util.chall_to_challb(
@@ -327,7 +328,8 @@ class PerformTest(unittest.TestCase):
         self.assertTrue(isinstance(result[0], challenges.ChallengeResponse))
         self.assertTrue(isinstance(result[1], challenges.ChallengeResponse))
         self.assertFalse(result[2])
-        self.authenticator.start_listener.assert_called_once_with(443, KEY)
+        self.authenticator.start_listener.assert_called_once_with(
+            CONFIG.dvsni_port, KEY)
 
     def test_cannot_perform(self):
         """What happens if start_listener() returns False."""
@@ -342,7 +344,8 @@ class PerformTest(unittest.TestCase):
         self.assertTrue(isinstance(result, list))
         self.assertEqual(len(result), 3)
         self.assertEqual(result, [None, None, False])
-        self.authenticator.start_listener.assert_called_once_with(443, KEY)
+        self.authenticator.start_listener.assert_called_once_with(
+            CONFIG.dvsni_port, KEY)
 
     def test_perform_with_pending_tasks(self):
         self.authenticator.tasks = {"foononce.acme.invalid": "cert_data"}
@@ -367,7 +370,7 @@ class StartListenerTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     @mock.patch("letsencrypt.plugins.standalone.authenticator."
                 "Crypto.Random.atfork")
@@ -402,7 +405,7 @@ class DoParentProcessTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     @mock.patch("letsencrypt.plugins.standalone.authenticator."
                 "zope.component.getUtility")
@@ -444,7 +447,7 @@ class DoChildProcessTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
         self.cert = achallenges.DVSNI(
             challb=acme_util.chall_to_challb(
                 challenges.DVSNI(r=("x" * 32), nonce="abcdef"), "pending"),
@@ -530,7 +533,7 @@ class CleanupTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import \
             StandaloneAuthenticator
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
         self.achall = achallenges.DVSNI(
             challb=acme_util.chall_to_challb(
                 challenges.DVSNI(r="whee", nonce="foononce"), "pending"),
@@ -562,7 +565,7 @@ class MoreInfoTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import (
             StandaloneAuthenticator)
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     def test_more_info(self):
         """Make sure exceptions aren't raised."""
@@ -574,7 +577,7 @@ class InitTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.standalone.authenticator import (
             StandaloneAuthenticator)
-        self.authenticator = StandaloneAuthenticator(config=None, name=None)
+        self.authenticator = StandaloneAuthenticator(config=CONFIG, name=None)
 
     def test_prepare(self):
         """Make sure exceptions aren't raised.

letsencrypt/renewer.py

@@ -117,6 +117,14 @@ def main(config=None):
         rc_config = configobj.ConfigObj(
             os.path.join(config["renewal_configs_dir"], i))
         try:
+            # TODO: Before trying to initialize the RenewableCert object,
+            #       we could check here whether the combination of the config
+            #       and the rc_config together disables all autorenewal and
+            #       autodeployment applicable to this cert.  In that case, we
+            #       can simply continue and don't need to instantiate a
+            #       RenewableCert object for this cert at all, which could
+            #       dramatically improve performance for large deployments
+            #       where autorenewal is widely turned off.
             cert = storage.RenewableCert(rc_config)
         except ValueError:
             # This indicates an invalid renewal configuration file, such

letsencrypt/reporter.py

@@ -0,0 +1,74 @@
+"""Collects and displays information to the user."""
+import collections
+import Queue
+import sys
+import textwrap
+
+import zope.interface
+
+from letsencrypt import interfaces
+
+
+class Reporter(object):
+    """Collects and displays information to the user.
+
+    :ivar `Queue.PriorityQueue` messages: Messages to be displayed to
+        the user.
+
+    """
+    zope.interface.implements(interfaces.IReporter)
+
+    HIGH_PRIORITY = 0
+    """High priority constant. See `add_message`."""
+    MEDIUM_PRIORITY = 1
+    """Medium priority constant. See `add_message`."""
+    LOW_PRIORITY = 2
+    """Low priority constant. See `add_message`."""
+
+    _RESET = '\033[0m'
+    _BOLD = '\033[1m'
+    _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash')
+
+    def __init__(self):
+        self.messages = Queue.PriorityQueue()
+
+    def add_message(self, msg, priority, on_crash=False):
+        """Adds msg to the list of messages to be printed.
+
+        :param str msg: Message to be displayed to the user.
+
+        :param int priority: One of `HIGH_PRIORITY`, `MEDIUM_PRIORITY`,
+            or `LOW_PRIORITY`.
+
+        :param bool on_crash: Whether or not the message should be
+            printed if the program exits abnormally.
+
+        """
+        assert self.HIGH_PRIORITY <= priority <= self.LOW_PRIORITY
+        self.messages.put(self._msg_type(priority, msg, on_crash))
+
+    def print_messages(self):
+        """Prints messages to the user and clears the message queue.
+
+        If there is an unhandled exception, only messages for which
+        ``on_crash`` is ``True`` are printed.
+
+        """
+        bold_on = False
+        if not self.messages.empty():
+            no_exception = sys.exc_info()[0] is None
+            bold_on = sys.stdout.isatty()
+            if bold_on:
+                sys.stdout.write(self._BOLD)
+            print 'IMPORTANT NOTES:'
+            wrapper = textwrap.TextWrapper(initial_indent=' - ',
+                                           subsequent_indent=(' ' * 3))
+        while not self.messages.empty():
+            msg = self.messages.get()
+            if no_exception or msg.on_crash:
+                if bold_on and msg.priority > self.HIGH_PRIORITY:
+                    sys.stdout.write(self._RESET)
+                    bold_on = False
+                print wrapper.fill(msg.text)
+        if bold_on:
+            sys.stdout.write(self._RESET)

letsencrypt/tests/acme_util.py

@@ -1,4 +1,4 @@
-"""Class helps construct valid ACME messages for testing."""
+"""ACME utilities for testing."""
 import datetime
 import itertools
 import os

letsencrypt/tests/continuity_auth_test.py

@@ -1,4 +1,4 @@
-"""Test the ContinuityAuthenticator dispatcher."""
+"""Test for letsencrypt.continuity_auth."""
 import unittest
 
 import mock

letsencrypt/tests/notify_test.py

@@ -1,4 +1,4 @@
-"""Tests for letsencrypt/notify.py"""
+"""Tests for letsencrypt.notify."""
 
 import mock
 import socket

letsencrypt/tests/proof_of_possession_test.py

@@ -1,4 +1,4 @@
-"""Tests for proof_of_possession.py"""
+"""Tests for letsencrypt.proof_of_possession."""
 import Crypto.PublicKey.RSA
 import os
 import pkg_resources

letsencrypt/tests/reporter_test.py

@@ -0,0 +1,75 @@
+"""Tests for letsencrypt.reporter."""
+import StringIO
+import sys
+import unittest
+
+
+class ReporterTest(unittest.TestCase):
+    """Tests for letsencrypt.reporter.Reporter."""
+
+    def setUp(self):
+        from letsencrypt import reporter
+        self.reporter = reporter.Reporter()
+
+        self.old_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+    def tearDown(self):
+        sys.stdout = self.old_stdout
+
+    def test_tty_print_empty(self):
+        sys.stdout.isatty = lambda: True
+        self.test_no_tty_print_empty()
+
+    def test_no_tty_print_empty(self):
+        self.reporter.print_messages()
+        self.assertEqual(sys.stdout.getvalue(), "")
+        try:
+            raise ValueError
+        except ValueError:
+            self.reporter.print_messages()
+        self.assertEqual(sys.stdout.getvalue(), "")
+
+    def test_tty_successful_exit(self):
+        sys.stdout.isatty = lambda: True
+        self._successful_exit_common()
+
+    def test_no_tty_successful_exit(self):
+        self._successful_exit_common()
+
+    def test_tty_unsuccessful_exit(self):
+        sys.stdout.isatty = lambda: True
+        self._unsuccessful_exit_common()
+
+    def test_no_tty_unsuccessful_exit(self):
+        self._unsuccessful_exit_common()
+
+    def _successful_exit_common(self):
+        self._add_messages()
+        self.reporter.print_messages()
+        output = sys.stdout.getvalue()
+        self.assertTrue("IMPORTANT NOTES:" in output)
+        self.assertTrue("High" in output)
+        self.assertTrue("Med" in output)
+        self.assertTrue("Low" in output)
+
+    def _unsuccessful_exit_common(self):
+        self._add_messages()
+        try:
+            raise ValueError
+        except ValueError:
+            self.reporter.print_messages()
+        output = sys.stdout.getvalue()
+        self.assertTrue("IMPORTANT NOTES:" in output)
+        self.assertTrue("High" in output)
+        self.assertTrue("Med" not in output)
+        self.assertTrue("Low" not in output)
+
+    def _add_messages(self):
+        self.reporter.add_message("High", self.reporter.HIGH_PRIORITY, True)
+        self.reporter.add_message("Med", self.reporter.MEDIUM_PRIORITY)
+        self.reporter.add_message("Low", self.reporter.LOW_PRIORITY)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover