Start implementing some renewal hook flags

Also some refactoring:
  - split renewal out of _auth_from_domains into renewal.renew_cert
  - split main._csr_obtain_cert out of main.obtain_cert

letsencrypt/cli.py

@@ -692,7 +692,23 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
         " used to create obtain or most recently successfully renew each"
         " certificate lineage. You can try it with `--dry-run` first. For"
         " more fine-grained control, you can renew individual lineages with"
-        " the `certonly` subcommand.")
+        " the `certonly` subcommand. Hooks are available to run commands "
+        " before and after renewal; see XXX for more information on these.")
+
+    helpful.add(
+        "renew", "--pre-hook",
+        help="Command to be run in a shell before obtaining any certificates. Intended"
+        " primarily for renewal, where it can be used to temporarily shut down a"
+        " webserver that might conflict with the standalone plugin. This will "
+        " only be called if a certificate is actually to be obtained/renewed. ")
+    helpful.add(
+        "renew", "--post-hook",
+        help="Command to be run in a shell after attempting to obtain/renew "
+        " certificates. Can be used to deploy renewed certificates, or to restart"
+        " any servers that were stopped by --pre-hook.")
+    helpful.add(
+        "renew", "--renew-hook",
+        help="Command to be run in a shell once for each renewed certificate")
 
     helpful.add_deprecated_argument("--agree-dev-preview", 0)
 

letsencrypt/errors.py

@@ -25,6 +25,10 @@ class CertStorageError(Error):
     """Generic `.CertStorage` error."""
 
 
+class HookCommandNotFound(Error):
+    """Failed to find a hook command in the PATH."""
+
+
 # Auth Handler Errors
 class AuthorizationError(Error):
     """Authorization error."""

letsencrypt/hooks.py

@@ -0,0 +1,96 @@
+"""Facilities for implementing hooks that call shell commands."""
+from __future__ import print_function
+
+import logging
+import os
+
+from subprocess import Popen, PIPE
+
+from letsencrypt import errors
+
+logger = logging.getLogger(__name__)
+
+def validate_hooks(config):
+    """Check hook commands are executable."""
+    _validate_hook(config.pre_hook)
+    _validate_hook(config.post_hook)
+    _validate_hook(config.renew_hook)
+
+def _prog(shell_cmd):
+    """Extract the program run by a shell command"""
+    cmd = _which(shell_cmd)
+    return os.path.basename(cmd) if cmd else None
+
+def _validate_hook(shell_cmd):
+    """Check that a command provided as a hook is plausibly executable.
+
+    :raises .errors.HookCommandNotFound: if the command is not found
+    """
+    cmd = shell_cmd.partition(" ")[0]
+    if not _prog(cmd):
+        path = os.environ["PATH"]
+        msg = "Unable to find command {0} in the PATH.\n(PATH is {1})".format(
+            cmd, path)
+        raise errors.HookCommandNotFound(msg)
+
+    return True
+
+def pre_hook(config):
+    "Run pre-hook if it's defined and hasn't been run."
+    if config.pre_hook and not pre_hook.already:
+        logger.info("Running pre-hook command: %s", config.pre_hook)
+        _run_hook(config.pre_hook)
+        pre_hook.already = True
+
+pre_hook.already = False
+
+def post_hook(config, final=False):
+    """Run post hook if defined.
+
+    If the verb is renew, we might have more certs to renew, so we wait until
+    we're called with final=True before actually doing anything.
+    """
+    if config.post_hook:
+        if final or config.verb != "renew":
+            logger.info("Running post-hook command: %s", config.post_hook)
+            _run_hook(config.post_hook)
+
+def renew_hook(config, domains, lineage_path):
+    "Run post-renewal hook if defined."
+    if config.renew_hook:
+        os.environ["RENEWED_DOMAINS"] = " ".join(domains)
+        os.environ["RENEWED_LINEAGE"] = lineage_path
+        _run_hook(config.renew_hook)
+
+def _run_hook(shell_cmd):
+    """Run a hook command.
+
+    :returns: stderr if there was any"""
+
+    cmd = Popen(shell_cmd, shell=True, stdout=PIPE, stderr=PIPE)
+    _out, err = cmd.communicate()
+    if cmd.returncode != 0:
+        logger.error('Hook command "%s" returned error code %d', shell_cmd, cmd.returncode)
+    if err:
+        logger.error('Error output from %s:\n%s', _prog(shell_cmd), err)
+
+def _which(program):
+    """Test if program is in the path."""
+    # Borrowed from:
+    # https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python
+    # XXX May need more porting to handle .exe extensions on Windows
+    def _is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, _fname = os.path.split(program)
+    if fpath:
+        if _is_exe(program):
+            return program
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            exe_file = os.path.join(path, program)
+            if _is_exe(exe_file):
+                return exe_file
+
+    return None

letsencrypt/main.py

@@ -8,7 +8,6 @@ import sys
 import time
 import traceback
 
-import OpenSSL
 import zope.component
 
 from acme import jose
@@ -23,6 +22,7 @@ from letsencrypt import colored_logging
 from letsencrypt import configuration
 from letsencrypt import constants
 from letsencrypt import errors
+from letsencrypt import hooks
 from letsencrypt import interfaces
 from letsencrypt import le_util
 from letsencrypt import log
@@ -52,28 +52,6 @@ def _suggest_donation_if_appropriate(config, action):
     reporter_util.add_message(msg, reporter_util.LOW_PRIORITY)
 
 
-def _avoid_invalidating_lineage(config, lineage, original_server):
-    "Do not renew a valid cert with one from a staging server!"
-    def _is_staging(srv):
-        return srv == constants.STAGING_URI or "staging" in srv
-
-    # Some lineages may have begun with --staging, but then had production certs
-    # added to them
-    latest_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
-                                                  open(lineage.cert).read())
-    # all our test certs are from happy hacker fake CA, though maybe one day
-    # we should test more methodically
-    now_valid = "fake" not in repr(latest_cert.get_issuer()).lower()
-
-    if _is_staging(config.server):
-        if not _is_staging(original_server) or now_valid:
-            if not config.break_my_certs:
-                names = ", ".join(lineage.names())
-                raise errors.Error(
-                    "You've asked to renew/replace a seemingly valid certificate with "
-                    "a test certificate (domains: {0}). We will not do that "
-                    "unless you use the --break-my-certs flag!".format(names))
-
 
 def _report_successful_dry_run(config):
     reporter_util = zope.component.getUtility(interfaces.IReporter)
@@ -82,6 +60,7 @@ def _report_successful_dry_run(config):
                                   reporter_util.HIGH_PRIORITY, on_crash=False)
 
 
+
 def _auth_from_domains(le_client, config, domains, lineage=None):
     """Authenticate and enroll certificate."""
     # Note: This can raise errors... caught above us though. This is now
@@ -105,31 +84,18 @@ def _auth_from_domains(le_client, config, domains, lineage=None):
         # The lineage already exists; allow the caller to try installing
         # it without getting a new certificate at all.
         return lineage, "reinstall"
-    elif action == "renew":
+
-        original_server = lineage.configuration["renewalparams"]["server"]
+    hooks.pre_hook(config)
-        _avoid_invalidating_lineage(config, lineage, original_server)
+    try:
-        # TODO: schoen wishes to reuse key - discussion
+        if action == "renew":
-        # https://github.com/letsencrypt/letsencrypt/pull/777/files#r40498574
+            renewal.renew_cert(config, domains, le_client, lineage)
-        new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains)
+        elif action == "newcert":
-        # TODO: Check whether it worked! <- or make sure errors are thrown (jdk)
+            # TREAT AS NEW REQUEST
-        if config.dry_run:
+            lineage = le_client.obtain_and_enroll_certificate(domains)
-            logger.info("Dry run: skipping updating lineage at %s",
+            if lineage is False:
-                        os.path.dirname(lineage.cert))
+                raise errors.Error("Certificate could not be obtained")
-        else:
+    finally:
-            lineage.save_successor(
+        hooks.post_hook(config)
-                lineage.latest_common_version(), OpenSSL.crypto.dump_certificate(
-                    OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped),
-                new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain),
-                configuration.RenewerConfiguration(config.namespace))
-            lineage.update_all_links_to(lineage.latest_common_version())
-        # TODO: Check return value of save_successor
-        # TODO: Also update lineage renewal config with any relevant
-        #       configuration values from this attempt? <- Absolutely (jdkasten)
-    elif action == "newcert":
-        # TREAT AS NEW REQUEST
-        lineage = le_client.obtain_and_enroll_certificate(domains)
-        if lineage is False:
-            raise errors.Error("Certificate could not be obtained")
 
     if not config.dry_run and not config.verb == "renew":
         _report_new_cert(lineage.cert, lineage.fullchain)
@@ -142,7 +108,8 @@ def _handle_subset_cert_request(config, domains, cert):
 
     :param storage.RenewableCert cert:
 
-    :returns: Tuple of (string, cert_or_None) as per _treat_as_renewal
+    :returns: Tuple of (stringa action, cert_or_None) as per _treat_as_renewal
+              action can be: "newcert" | "renew" | "reinstall"
     :rtype: tuple
 
     """
@@ -183,7 +150,8 @@ def _handle_identical_cert_request(config, cert):
 
     :param storage.RenewableCert cert:
 
-    :returns: Tuple of (string, cert_or_None) as per _treat_as_renewal
+    :returns: Tuple of (string action, cert_or_None) as per _treat_as_renewal
+              action can be: "newcert" | "renew" | "reinstall"
     :rtype: tuple
 
     """
@@ -507,41 +475,53 @@ def run(config, plugins):  # pylint: disable=too-many-branches,too-many-locals
     _suggest_donation_if_appropriate(config, action)
 
 
+def _csr_obtain_cert(config, le_client):
+    """Obtain a cert using a user-supplied CSR
+
+    This works differently in the CSR case (for now) because we don't
+    have the privkey, and therefore can't construct the files for a lineage.
+    So we just save the cert & chain to disk :/
+    """
+    csr, typ = config.actual_csr
+    certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ)
+    if config.dry_run:
+        logger.info(
+            "Dry run: skipping saving certificate to %s", config.cert_path)
+    else:
+        cert_path, _, cert_fullchain = le_client.save_certificate(
+            certr, chain, config.cert_path, config.chain_path, config.fullchain_path)
+        _report_new_cert(cert_path, cert_fullchain)
+
+
 def obtain_cert(config, plugins, lineage=None):
-    """Implements "certonly": authenticate & obtain cert, but do not install it."""
+    """Authenticate & obtain cert, but do not install it.
-    # pylint: disable=too-many-locals
+
+    This implements the 'certonly' subcommand, and is also called from within the
+    'renew' command."""
+
+    # SETUP: Select plugins and construct a client instance
     try:
         # installers are used in auth mode to determine domain names
-        installer, authenticator = plug_sel.choose_configurator_plugins(config, plugins, "certonly")
+        installer, auth = plug_sel.choose_configurator_plugins(config, plugins, "certonly")
     except errors.PluginSelectionError as e:
         logger.info("Could not choose appropriate plugin: %s", e)
         raise
+    le_client = _init_le_client(config, auth, installer)
 
-    # TODO: Handle errors from _init_le_client?
+    # SHOWTIME: Possibly obtain/renew a cert, and set action to renew | newcert | reinstall
-    le_client = _init_le_client(config, authenticator, installer)
+    if config.csr is None: # the common case
-
-    action = "newcert"
-    # This is a special case; cert and chain are simply saved
-    if config.csr is not None:
-        assert lineage is None, "Did not expect a CSR with a RenewableCert"
-        csr, typ = config.actual_csr
-        certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ)
-        if config.dry_run:
-            logger.info(
-                "Dry run: skipping saving certificate to %s", config.cert_path)
-        else:
-            cert_path, _, cert_fullchain = le_client.save_certificate(
-                certr, chain, config.cert_path, config.chain_path, config.fullchain_path)
-            _report_new_cert(cert_path, cert_fullchain)
-    else:
         domains = _find_domains(config, installer)
         _, action = _auth_from_domains(le_client, config, domains, lineage)
+    else:
+        assert lineage is None, "Did not expect a CSR with a RenewableCert"
+        _csr_obtain_cert(config, le_client)
+        action = "newcert"
 
+    # POSTPRODUCTION: Cleanup, deployment & reporting
     if config.dry_run:
         _report_successful_dry_run(config)
     elif config.verb == "renew":
         if installer is None:
-            # Tell the user that the server was not restarted.
             print("new certificate deployed without reload, fullchain is",
                   lineage.fullchain)
         else:
@@ -553,10 +533,13 @@ def obtain_cert(config, plugins, lineage=None):
                   config.installer, "server; fullchain is", lineage.fullchain)
     _suggest_donation_if_appropriate(config, action)
 
+
 def renew(config, unused_plugins):
     """Renew previously-obtained certificates."""
-    renewal.renew_all_lineages(config)
+    try:
-
+        renewal.renew_all_lineages(config)
+    finally:
+        hooks.post_hook(config, final=True)
 
 
 def setup_log_file_handler(config, logfile, fmt):

letsencrypt/renewal.py

@@ -9,8 +9,13 @@ import traceback
 import six
 import zope.component
 
+import OpenSSL
+
 from letsencrypt import configuration
 from letsencrypt import cli
+from letsencrypt import constants
+
+from letsencrypt import crypto_util
 from letsencrypt import errors
 from letsencrypt import storage
 from letsencrypt.plugins import disco as plugins_disco
@@ -199,6 +204,48 @@ def should_renew(config, lineage):
     return False
 
 
+def _avoid_invalidating_lineage(config, lineage, original_server):
+    "Do not renew a valid cert with one from a staging server!"
+    def _is_staging(srv):
+        return srv == constants.STAGING_URI or "staging" in srv
+
+    # Some lineages may have begun with --staging, but then had production certs
+    # added to them
+    latest_cert = OpenSSL.crypto.load_certificate(
+        OpenSSL.crypto.FILETYPE_PEM, open(lineage.cert).read())
+    # all our test certs are from happy hacker fake CA, though maybe one day
+    # we should test more methodically
+    now_valid = "fake" not in repr(latest_cert.get_issuer()).lower()
+
+    if _is_staging(config.server):
+        if not _is_staging(original_server) or now_valid:
+            if not config.break_my_certs:
+                names = ", ".join(lineage.names())
+                raise errors.Error(
+                    "You've asked to renew/replace a seemingly valid certificate with "
+                    "a test certificate (domains: {0}). We will not do that "
+                    "unless you use the --break-my-certs flag!".format(names))
+
+
+def renew_cert(config, domains, le_client, lineage):
+    "Renew a certificate lineage."
+    original_server = lineage.configuration["renewalparams"]["server"]
+    _avoid_invalidating_lineage(config, lineage, original_server)
+    new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains)
+    if config.dry_run:
+        logger.info("Dry run: skipping updating lineage at %s",
+                    os.path.dirname(lineage.cert))
+    else:
+        prior_version = lineage.latest_common_version()
+        new_cert = OpenSSL.crypto.dump_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped)
+        new_chain = crypto_util.dump_pyopenssl_chain(new_chain)
+        renewal_conf = configuration.RenewerConfiguration(config.namespace)
+        lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, renewal_conf)
+        lineage.update_all_links_to(lineage.latest_common_version())
+    # TODO: Check return value of save_successor
+
+
 def _renew_describe_results(config, renew_successes, renew_failures,
                             renew_skipped, parse_failures):
     def _status(msgs, category):

letsencrypt/tests/cli_test.py

@@ -579,11 +579,11 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
                     mock_init.return_value = mock_client
                     get_utility_path = 'letsencrypt.main.zope.component.getUtility'
                     with mock.patch(get_utility_path) as mock_get_utility:
-                        with mock.patch('letsencrypt.main.OpenSSL') as mock_ssl:
+                        with mock.patch('letsencrypt.main.renewal.OpenSSL') as mock_ssl:
                             mock_latest = mock.MagicMock()
                             mock_latest.get_issuer.return_value = "Fake fake"
                             mock_ssl.crypto.load_certificate.return_value = mock_latest
-                            with mock.patch('letsencrypt.main.crypto_util'):
+                            with mock.patch('letsencrypt.main.renewal.crypto_util'):
                                 if not args:
                                     args = ['-d', 'isnot.org', '-a', 'standalone', 'certonly']
                                 if extra_args: