Merge pull request #2745 from letsencrypt/quiet

Implement --quiet / -q

letsencrypt/account.py

@@ -98,7 +98,7 @@ def report_new_account(acc, config):
         recovery_msg = ("If you lose your account credentials, you can "
                         "recover through e-mails sent to {0}.".format(
                             ", ".join(acc.regr.body.emails)))
-        reporter.add_message(recovery_msg, reporter.HIGH_PRIORITY)
+        reporter.add_message(recovery_msg, reporter.MEDIUM_PRIORITY)
 
 
 class AccountMemoryStorage(interfaces.AccountStorage):

letsencrypt/cli.py

@@ -639,6 +639,13 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
              "regardless of whether it is near expiry. (Often "
              "--keep-until-expiring is more appropriate). Also implies "
              "--expand.")
+    helpful.add(
+        "automation", "--allow-subset-of-names", action="store_true",
+        help="When performing domain validation, do not consider it a failure "
+             "if authorizations can not be obtained for a strict subset of "
+             "the requested domains. This may be useful for allowing renewals for "
+             "multiple domains to succeed even if some domains no longer point "
+             "at this system. This option cannot be used with --csr.")
     helpful.add(
         "automation", "--agree-tos", dest="tos", action="store_true",
         help="Agree to the Let's Encrypt Subscriber Agreement")
@@ -656,6 +663,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
         "automation", "--no-self-upgrade", action="store_true",
         help="(letsencrypt-auto only) prevent the letsencrypt-auto script from"
              " upgrading itself to newer released versions")
+    helpful.add(
+        "automation", "-q", "--quiet", dest="quiet", action="store_true",
+        help="Silence all output except errors. Useful for automation via cron."
+             "Implies --non-interactive.")
 
     helpful.add_group(
         "testing", description="The following flags are meant for "
@@ -716,12 +727,6 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False):
         "security", "--strict-permissions", action="store_true",
         help="Require that all configuration files are owned by the current "
              "user; only needed if your config is somewhere unsafe like /tmp/")
-    helpful.add(
-        "automation", "--allow-subset-of-names",
-        action="store_true",
-        help="When performing domain validation, do not consider it a failure "
-             "if authorizations can not be obtained for a strict subset of "
-             "the requested domains. This option cannot be used with --csr.")
 
     helpful.add_group(
         "renew", description="The 'renew' subcommand will attempt to renew all"

letsencrypt/hooks.py

@@ -62,7 +62,7 @@ def renew_hook(config, domains, lineage_path):
             os.environ["RENEWED_LINEAGE"] = lineage_path
             _run_hook(config.renew_hook)
         else:
-            print("Dry run: skipping renewal hook command: {0}".format(config.renew_hook))
+            logger.warning("Dry run: skipping renewal hook command: %s", config.renew_hook)
 
 def _run_hook(shell_cmd):
     """Run a hook command.

letsencrypt/main.py

@@ -34,7 +34,6 @@ from letsencrypt.display import util as display_util, ops as display_ops
 from letsencrypt.plugins import disco as plugins_disco
 from letsencrypt.plugins import selection as plug_sel
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -518,19 +517,20 @@ def obtain_cert(config, plugins, lineage=None):
         action = "newcert"
 
     # POSTPRODUCTION: Cleanup, deployment & reporting
+    notify = zope.component.getUtility(interfaces.IDisplay).notification
     if config.dry_run:
         _report_successful_dry_run(config)
     elif config.verb == "renew":
         if installer is None:
-            print("new certificate deployed without reload, fullchain is",
+            notify("new certificate deployed without reload, fullchain is {0}".format(
-                  lineage.fullchain)
+                   lineage.fullchain), pause=False)
         else:
             # In case of a renewal, reload server to pick up new certificate.
             # In principle we could have a configuration option to inhibit this
             # from happening.
             installer.restart()
-            print("new certificate deployed with reload of",
+            notify("new certificate deployed with reload of {0} server; fullchain is {1}".format(
-                  config.installer, "server; fullchain is", lineage.fullchain)
+                   config.installer, lineage.fullchain), pause=False)
     _suggest_donation_if_appropriate(config, action)
 
 
@@ -672,7 +672,10 @@ def main(cli_args=sys.argv[1:]):
     sys.excepthook = functools.partial(_handle_exception, config=config)
 
     # Displayer
-    if config.noninteractive_mode:
+    if config.quiet:
+        config.noninteractive_mode = True
+        displayer = display_util.NoninteractiveDisplay(open(os.devnull, "w"))
+    elif config.noninteractive_mode:
         displayer = display_util.NoninteractiveDisplay(sys.stdout)
     elif config.text_mode:
         displayer = display_util.FileDisplay(sys.stdout)
@@ -684,7 +687,7 @@ def main(cli_args=sys.argv[1:]):
     zope.component.provideUtility(displayer)
 
     # Reporter
-    report = reporter.Reporter()
+    report = reporter.Reporter(config)
     zope.component.provideUtility(report)
     atexit.register(report.atexit_print_messages)
 

letsencrypt/renewal.py

@@ -17,6 +17,7 @@ from letsencrypt import constants
 
 from letsencrypt import crypto_util
 from letsencrypt import errors
+from letsencrypt import interfaces
 from letsencrypt import hooks
 from letsencrypt import storage
 from letsencrypt.plugins import disco as plugins_disco
@@ -241,48 +242,59 @@ def renew_cert(config, domains, le_client, lineage):
             OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped)
         new_chain = crypto_util.dump_pyopenssl_chain(new_chain)
         renewal_conf = configuration.RenewerConfiguration(config.namespace)
+        # TODO: Check return value of save_successor
         lineage.save_successor(prior_version, new_cert, new_key.pem, new_chain, renewal_conf)
         lineage.update_all_links_to(lineage.latest_common_version())
 
     hooks.renew_hook(config, domains, lineage.live_dir)
-    # TODO: Check return value of save_successor
 
 
+def report(msgs, category):
+    "Format a results report for a category of renewal outcomes"
+    lines = ("%s (%s)" % (m, category) for m in msgs)
+    return "  " + "\n  ".join(lines)
+
 def _renew_describe_results(config, renew_successes, renew_failures,
                             renew_skipped, parse_failures):
-    def _status(msgs, category):
+
-        return "  " + "\n  ".join("%s (%s)" % (m, category) for m in msgs)
+    out = []
+    notify = out.append
+
     if config.dry_run:
-        print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry")
+        notify("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry")
-        print("**          (The test certificates below have not been saved.)")
+        notify("**          (The test certificates below have not been saved.)")
-    print()
+    notify("")
     if renew_skipped:
-        print("The following certs are not due for renewal yet:")
+        notify("The following certs are not due for renewal yet:")
-        print(_status(renew_skipped, "skipped"))
+        notify(report(renew_skipped, "skipped"))
     if not renew_successes and not renew_failures:
-        print("No renewals were attempted.")
+        notify("No renewals were attempted.")
     elif renew_successes and not renew_failures:
-        print("Congratulations, all renewals succeeded. The following certs "
+        notify("Congratulations, all renewals succeeded. The following certs "
-              "have been renewed:")
+               "have been renewed:")
-        print(_status(renew_successes, "success"))
+        notify(report(renew_successes, "success"))
     elif renew_failures and not renew_successes:
-        print("All renewal attempts failed. The following certs could not be "
+        notify("All renewal attempts failed. The following certs could not be "
-              "renewed:")
+               "renewed:")
-        print(_status(renew_failures, "failure"))
+        notify(report(renew_failures, "failure"))
     elif renew_failures and renew_successes:
-        print("The following certs were successfully renewed:")
+        notify("The following certs were successfully renewed:")
-        print(_status(renew_successes, "success"))
+        notify(report(renew_successes, "success"))
-        print("\nThe following certs could not be renewed:")
+        notify("\nThe following certs could not be renewed:")
-        print(_status(renew_failures, "failure"))
+        notify(report(renew_failures, "failure"))
 
     if parse_failures:
-        print("\nAdditionally, the following renewal configuration files "
+        notify("\nAdditionally, the following renewal configuration files "
-              "were invalid: ")
+               "were invalid: ")
-        print(_status(parse_failures, "parsefail"))
+        notify(parse_failures, "parsefail")
 
     if config.dry_run:
-        print("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry")
+        notify("** DRY RUN: simulating 'letsencrypt renew' close to cert expiry")
-        print("**          (The test certificates above have not been saved.)")
+        notify("**          (The test certificates above have not been saved.)")
+
+    if config.quiet and not (renew_failures or parse_failures):
+        return
+    print("\n".join(out))
 
 
 def renew_all_lineages(config):
@@ -302,7 +314,8 @@ def renew_all_lineages(config):
     renew_skipped = []
     parse_failures = []
     for renewal_file in renewal_conf_files(renewer_config):
-        print("Processing " + renewal_file)
+        disp = zope.component.getUtility(interfaces.IDisplay)
+        disp.notification("Processing " + renewal_file, pause=False)
         lineage_config = copy.deepcopy(config)
 
         # Note that this modifies config (to add back the configuration

letsencrypt/reporter.py

@@ -35,8 +35,9 @@ class Reporter(object):
 
     _msg_type = collections.namedtuple('ReporterMsg', 'priority text on_crash')
 
-    def __init__(self):
+    def __init__(self, config):
         self.messages = queue.PriorityQueue()
+        self.config = config
 
     def add_message(self, msg, priority, on_crash=True):
         """Adds msg to the list of messages to be printed.
@@ -76,9 +77,10 @@ class Reporter(object):
         if not self.messages.empty():
             no_exception = sys.exc_info()[0] is None
             bold_on = sys.stdout.isatty()
-            if bold_on:
+            if not self.config.quiet:
-                print(le_util.ANSI_SGR_BOLD)
+                if bold_on:
-            print('IMPORTANT NOTES:')
+                    print(le_util.ANSI_SGR_BOLD)
+                print('IMPORTANT NOTES:')
             first_wrapper = textwrap.TextWrapper(
                 initial_indent=' - ', subsequent_indent=(' ' * 3))
             next_wrapper = textwrap.TextWrapper(
@@ -86,14 +88,20 @@ class Reporter(object):
                 subsequent_indent=first_wrapper.subsequent_indent)
         while not self.messages.empty():
             msg = self.messages.get()
+            if self.config.quiet:
+                # In --quiet mode, we only print high priority messages that
+                # are flagged for crash cases
+                if not (msg.priority == self.HIGH_PRIORITY and msg.on_crash):
+                    continue
             if no_exception or msg.on_crash:
                 if bold_on and msg.priority > self.HIGH_PRIORITY:
-                    sys.stdout.write(le_util.ANSI_SGR_RESET)
+                    if not self.config.quiet:
-                    bold_on = False
+                        sys.stdout.write(le_util.ANSI_SGR_RESET)
+                        bold_on = False
                 lines = msg.text.splitlines()
                 print(first_wrapper.fill(lines[0]))
                 if len(lines) > 1:
                     print("\n".join(
                         next_wrapper.fill(line) for line in lines[1:]))
-        if bold_on:
+        if bold_on and not self.config.quiet:
             sys.stdout.write(le_util.ANSI_SGR_RESET)

letsencrypt/tests/cli_test.py

@@ -57,30 +57,21 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         # pylint: disable=protected-access
         cli._parser = cli.set_by_cli.detector = None
 
-    def _call(self, args):
+    def _call(self, args, stdout=None):
         "Run the cli with output streams and actual client mocked out"
         with mock.patch('letsencrypt.main.client') as client:
-            ret, stdout, stderr = self._call_no_clientmock(args)
+            ret, stdout, stderr = self._call_no_clientmock(args, stdout)
             return ret, stdout, stderr, client
 
-    def _call_no_clientmock(self, args):
+    def _call_no_clientmock(self, args, stdout=None):
         "Run the client with output streams mocked out"
         args = self.standard_args + args
-        with mock.patch('letsencrypt.main.sys.stdout') as stdout:
+
+        toy_stdout = stdout if stdout else six.StringIO()
+        with mock.patch('letsencrypt.main.sys.stdout', new=toy_stdout):
             with mock.patch('letsencrypt.main.sys.stderr') as stderr:
                 ret = main.main(args[:])  # NOTE: parser can alter its args!
-        return ret, stdout, stderr
+        return ret, toy_stdout, stderr
-
-    def _call_stdout(self, args):
-        """
-        Variant of _call that preserves stdout so that it can be mocked by the
-        caller.
-        """
-        args = self.standard_args + args
-        with mock.patch('letsencrypt.main.sys.stderr') as stderr:
-            with mock.patch('letsencrypt.main.client') as client:
-                ret = main.main(args[:])  # NOTE: parser can alter its args!
-        return ret, None, stderr, client
 
     def test_no_flags(self):
         with mock.patch('letsencrypt.main.run') as mock_run:
@@ -91,10 +82,9 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         "Run a command, and return the ouput string for scrutiny"
 
         output = six.StringIO()
-        with mock.patch('letsencrypt.main.sys.stdout', new=output):
+        self.assertRaises(SystemExit, self._call, args, output)
-            self.assertRaises(SystemExit, self._call_stdout, args)
+        out = output.getvalue()
-            out = output.getvalue()
+        return out
-            return out
 
     def test_help(self):
         self.assertRaises(SystemExit, self._call, ['--help'])
@@ -284,7 +274,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         plugins.visible.assert_called_once_with()
         plugins.visible().ifaces.assert_called_once_with(ifaces)
         filtered = plugins.visible().ifaces()
-        stdout.write.called_once_with(str(filtered))
+        self.assertEqual(stdout.getvalue().strip(), str(filtered))
 
     @mock.patch('letsencrypt.main.plugins_disco')
     @mock.patch('letsencrypt.main.cli.HelpfulArgumentParser.determine_help_topics')
@@ -299,7 +289,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         self.assertEqual(filtered.init.call_count, 1)
         filtered.verify.assert_called_once_with(ifaces)
         verified = filtered.verify()
-        stdout.write.called_once_with(str(verified))
+        self.assertEqual(stdout.getvalue().strip(), str(verified))
 
     @mock.patch('letsencrypt.main.plugins_disco')
     @mock.patch('letsencrypt.main.cli.HelpfulArgumentParser.determine_help_topics')
@@ -316,7 +306,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         verified.prepare.assert_called_once_with()
         verified.available.assert_called_once_with()
         available = verified.available()
-        stdout.write.called_once_with(str(available))
+        self.assertEqual(stdout.getvalue().strip(), str(available))
 
     def test_certonly_abspath(self):
         cert = 'cert'
@@ -374,7 +364,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         try:
             self._call(['--csr', CSR])
         except errors.Error as e:
-            assert "Please try the certonly" in e.message
+            assert "Please try the certonly" in repr(e)
             return
         assert False, "Expected supplying --csr to fail with default verb"
 
@@ -571,6 +561,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         mock_certr = mock.MagicMock()
         mock_key = mock.MagicMock(pem='pem_key')
         mock_client = mock.MagicMock()
+        stdout = None
         mock_client.obtain_certificate.return_value = (mock_certr, 'chain',
                                                        mock_key, 'csr')
         try:
@@ -590,7 +581,7 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
                                 if extra_args:
                                     args += extra_args
                                 try:
-                                    ret, _, _, _ = self._call(args)
+                                    ret, stdout, _, _ = self._call(args)
                                     if ret:
                                         print("Returned", ret)
                                         raise AssertionError(ret)
@@ -613,10 +604,10 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
                 with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf:
                     self.assertTrue(log_out in lf.read())
 
-        return mock_lineage, mock_get_utility
+        return mock_lineage, mock_get_utility, stdout
 
     def test_certonly_renewal(self):
-        lineage, get_utility = self._test_renewal_common(True, [])
+        lineage, get_utility, _ = self._test_renewal_common(True, [])
         self.assertEqual(lineage.save_successor.call_count, 1)
         lineage.update_all_links_to.assert_called_once_with(
             lineage.latest_common_version())
@@ -626,17 +617,18 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 
     def test_certonly_renewal_triggers(self):
         # --dry-run should force renewal
-        _, get_utility = self._test_renewal_common(False, ['--dry-run', '--keep'],
+        _, get_utility, _ = self._test_renewal_common(False, ['--dry-run', '--keep'],
-                                                   log_out="simulating renewal")
+                                                      log_out="simulating renewal")
         self.assertEqual(get_utility().add_message.call_count, 1)
         self.assertTrue('dry run' in get_utility().add_message.call_args[0][0])
 
-        _, _ = self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'],
+        self._test_renewal_common(False, ['--renew-by-default', '-tvv', '--debug'],
-                                        log_out="Auto-renewal forced")
+                                  log_out="Auto-renewal forced")
         self.assertEqual(get_utility().add_message.call_count, 1)
 
-        _, _ = self._test_renewal_common(False, ['-tvv', '--debug', '--keep'],
+        self._test_renewal_common(False, ['-tvv', '--debug', '--keep'],
-                                        log_out="not yet due", should_renew=False)
+                                  log_out="not yet due", should_renew=False)
+
 
     def _dump_log(self):
         with open(os.path.join(self.logs_dir, "letsencrypt.log")) as lf:
@@ -661,6 +653,19 @@ class CLITest(unittest.TestCase):  # pylint: disable=too-many-public-methods
         args = ["renew", "--dry-run", "-tvv"]
         self._test_renewal_common(True, [], args=args, should_renew=True)
 
+    def test_quiet_renew(self):
+        self._make_test_renewal_conf('sample-renewal.conf')
+        args = ["renew", "--dry-run"]
+        _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True)
+        out = stdout.getvalue()
+        self.assertTrue("renew" in out)
+
+        args = ["renew", "--dry-run", "-q"]
+        _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True)
+        out = stdout.getvalue()
+        self.assertEqual("", out)
+
+
     @mock.patch("letsencrypt.cli.set_by_cli")
     def test_ancient_webroot_renewal_conf(self, mock_set_by_cli):
         mock_set_by_cli.return_value = False

letsencrypt/tests/hook_test.py

@@ -3,7 +3,6 @@
 
 import os
 import unittest
-import sys
 
 import mock
 
@@ -48,11 +47,13 @@ class HookTest(unittest.TestCase):
         self.assertEqual(hooks._prog("funky"), None)
 
     def _test_a_hook(self, config, hook_function, calls_expected):
-        with mock.patch('letsencrypt.hooks.logger'):
+        with mock.patch('letsencrypt.hooks.logger') as mock_logger:
+            mock_logger.warning = mock.MagicMock()
             with mock.patch('letsencrypt.hooks._run_hook') as mock_run_hook:
                 hook_function(config)
                 hook_function(config)
                 self.assertEqual(mock_run_hook.call_count, calls_expected)
+            return mock_logger.warning
 
     def test_pre_hook(self):
         config = mock.MagicMock(pre_hook="true")
@@ -78,13 +79,8 @@ class HookTest(unittest.TestCase):
             self.assertEqual(os.environ["RENEWED_LINEAGE"], "thing")
 
             config = mock.MagicMock(renew_hook="true", dry_run=True)
-            if sys.version_info < (2, 7):
+            mock_warn = self._test_a_hook(config, rhook, 0)
-                # the print() function is not mockable in py26
+            self.assertEqual(mock_warn.call_count, 2)
-                self._test_a_hook(config, rhook, 0)
-            else:
-                with mock.patch("letsencrypt.hooks.print") as mock_print:
-                    self._test_a_hook(config, rhook, 0)
-                    self.assertEqual(mock_print.call_count, 2)
 
     @mock.patch('letsencrypt.hooks.Popen')
     def test_run_hook(self, mock_popen):

letsencrypt/tests/reporter_test.py

@@ -1,4 +1,5 @@
 """Tests for letsencrypt.reporter."""
+import mock
 import sys
 import unittest
 
@@ -10,7 +11,7 @@ class ReporterTest(unittest.TestCase):
 
     def setUp(self):
         from letsencrypt import reporter
-        self.reporter = reporter.Reporter()
+        self.reporter = reporter.Reporter(mock.MagicMock(quiet=False))
 
         self.old_stdout = sys.stdout
         sys.stdout = six.StringIO()