From 81b3a98346051ed7fad67d922515df41ac644a3e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 2 Feb 2016 17:23:01 -0800 Subject: [PATCH 01/22] Added write_renewal_config function --- letsencrypt/storage.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index e41805459..5da9de932 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -50,6 +50,43 @@ def add_time_interval(base_time, interval, textparser=parsedatetime.Calendar()): return textparser.parseDT(interval, base_time, tzinfo=tzinfo)[0] +def write_renewal_config(filename, target, cli_config): + """Writes a renewal config file with the specified name and values. + + :param str filename: Absolute path to the config file + :param dict target: Maps ALL_FOUR to their symlink paths + :param .RenewerConfiguration cli_config: parsed command line + arguments + + :returns: Configuration object for the new config file + :rtype: configobj.ConfigObj + + """ + # create_empty creates a new config file if filename does not exist + config = configobj.ConfigObj(filename, create_empty=True) + for kind in ALL_FOUR: + config[kind] = target[kind] + + # XXX: We clearly need a more general and correct way of getting + # options into the configobj for the RenewableCert instance. + # This is a quick-and-dirty way to do it to allow integration + # testing to start. (Note that the config parameter to new_lineage + # ideally should be a ConfigObj, but in this case a dict will be + # accepted in practice.) + renewalparams = vars(cli_config.namespace) + if renewalparams: + config["renewalparams"] = renewalparams + config.comments["renewalparams"] = ["", + "Options and defaults used" + " in the renewal process"] + + # TODO: add human-readable comments explaining other available + # parameters + logger.debug("Writing new config %s.", filename) + config.write() + return config + + class RenewableCert(object): # pylint: disable=too-many-instance-attributes """Renewable certificate. From c818b4f68924c793a37ce3d77325e04153b3a4f1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 2 Feb 2016 18:02:31 -0800 Subject: [PATCH 02/22] Refactor new_lineage --- letsencrypt/client.py | 18 ++++------------ letsencrypt/storage.py | 35 +++++-------------------------- letsencrypt/tests/renewer_test.py | 27 +++++++++--------------- 3 files changed, 19 insertions(+), 61 deletions(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index b7d486ba1..d8b8ca8c1 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -282,23 +282,13 @@ class Client(object): """ certr, chain, key, _ = self.obtain_certificate(domains) - # XXX: We clearly need a more general and correct way of getting - # options into the configobj for the RenewableCert instance. - # This is a quick-and-dirty way to do it to allow integration - # testing to start. (Note that the config parameter to new_lineage - # ideally should be a ConfigObj, but in this case a dict will be - # accepted in practice.) - params = vars(self.config.namespace) - config = {} - cli_config = configuration.RenewerConfiguration(self.config.namespace) - - if (cli_config.config_dir != constants.CLI_DEFAULTS["config_dir"] or - cli_config.work_dir != constants.CLI_DEFAULTS["work_dir"]): + if (self.config.config_dir != constants.CLI_DEFAULTS["config_dir"] or + self.config.work_dir != constants.CLI_DEFAULTS["work_dir"]): logger.warning( "Non-standard path(s), might not work with crontab installed " "by your operating system package manager") - if cli_config.dry_run: + if self.config.dry_run: logger.info("Dry run: Skipping creating new lineage for %s", domains[0]) return None @@ -307,7 +297,7 @@ class Client(object): domains[0], OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped), key.pem, crypto_util.dump_pyopenssl_chain(chain), - params, config, cli_config) + configuration.RenewerConfiguration(self.config.namespace)) def save_certificate(self, certr, chain_cert, cert_path, chain_path, fullchain_path): diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 5da9de932..8cc26d5b5 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -626,9 +626,8 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes return False @classmethod - def new_lineage(cls, lineagename, cert, privkey, chain, - renewalparams=None, config=None, cli_config=None): - # pylint: disable=too-many-locals,too-many-arguments + def new_lineage(cls, lineagename, cert, privkey, chain, cli_config): + # pylint: disable=too-many-locals """Create a new certificate lineage. Attempts to create a certificate lineage -- enrolled for @@ -648,26 +647,13 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :param str cert: the initial certificate version in PEM format :param str privkey: the private key in PEM format :param str chain: the certificate chain in PEM format - :param configobj.ConfigObj renewalparams: parameters that - should be used when instantiating authenticator and installer - objects in the future to attempt to renew this cert or deploy - new versions of it - :param configobj.ConfigObj config: renewal configuration - defaults, affecting, for example, the locations of the - directories where the associated files will be saved :param .RenewerConfiguration cli_config: parsed command line arguments :returns: the newly-created RenewalCert object - :rtype: :class:`storage.renewableCert`""" - - config = config_with_defaults(config) - # This attempts to read the renewer config file and augment or replace - # the renewer defaults with any options contained in that file. If - # renewer_config_file is undefined or if the file is nonexistent or - # empty, this .merge() will have no effect. - config.merge(configobj.ConfigObj(cli_config.renewer_config_file)) + :rtype: :class:`storage.renewableCert` + """ # Examine the configuration and find the new lineage's name for i in (cli_config.renewal_configs_dir, cli_config.archive_dir, cli_config.live_dir): @@ -722,18 +708,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # Document what we've done in a new renewal config file config_file.close() - new_config = configobj.ConfigObj(config_filename, create_empty=True) - for kind in ALL_FOUR: - new_config[kind] = target[kind] - if renewalparams: - new_config["renewalparams"] = renewalparams - new_config.comments["renewalparams"] = ["", - "Options and defaults used" - " in the renewal process"] - # TODO: add human-readable comments explaining other available - # parameters - logger.debug("Writing new config %s.", config_filename) - new_config.write() + new_config = write_renewal_config(config_filename, target, cli_config) return cls(new_config.filename, cli_config) def save_successor(self, prior_version, new_cert, new_privkey, new_chain): diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 3c8e3cb95..0628fc073 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -553,8 +553,7 @@ class RenewableCertTests(BaseRenewableCertTest): """Test for new_lineage() class method.""" from letsencrypt import storage result = storage.RenewableCert.new_lineage( - "the-lineage.com", "cert", "privkey", "chain", None, - self.defaults, self.cli_config) + "the-lineage.com", "cert", "privkey", "chain", self.cli_config) # This consistency check tests most relevant properties about the # newly created cert lineage. # pylint: disable=protected-access @@ -565,27 +564,23 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(f.read(), "cert" + "chain") # Let's do it again and make sure it makes a different lineage result = storage.RenewableCert.new_lineage( - "the-lineage.com", "cert2", "privkey2", "chain2", None, - self.defaults, self.cli_config) + "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config) self.assertTrue(os.path.exists(os.path.join( self.cli_config.renewal_configs_dir, "the-lineage.com-0001.conf"))) # Now trigger the detection of already existing files os.mkdir(os.path.join( self.cli_config.live_dir, "the-lineage.com-0002")) self.assertRaises(errors.CertStorageError, - storage.RenewableCert.new_lineage, - "the-lineage.com", "cert3", "privkey3", "chain3", - None, self.defaults, self.cli_config) + storage.RenewableCert.new_lineage, "the-lineage.com", + "cert3", "privkey3", "chain3", self.cli_config) os.mkdir(os.path.join(self.cli_config.archive_dir, "other-example.com")) self.assertRaises(errors.CertStorageError, storage.RenewableCert.new_lineage, - "other-example.com", "cert4", "privkey4", "chain4", - None, self.defaults, self.cli_config) + "other-example.com", "cert4", + "privkey4", "chain4", self.cli_config) # Make sure it can accept renewal parameters - params = {"stuff": "properties of stuff", "great": "awesome"} result = storage.RenewableCert.new_lineage( - "the-lineage.com", "cert2", "privkey2", "chain2", - params, self.defaults, self.cli_config) + "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config) # TODO: Conceivably we could test that the renewal parameters actually # got saved @@ -597,8 +592,7 @@ class RenewableCertTests(BaseRenewableCertTest): shutil.rmtree(self.cli_config.live_dir) storage.RenewableCert.new_lineage( - "the-lineage.com", "cert2", "privkey2", "chain2", - None, self.defaults, self.cli_config) + "the-lineage.com", "cert2", "privkey2", "chain2", self.cli_config) self.assertTrue(os.path.exists( os.path.join( self.cli_config.renewal_configs_dir, "the-lineage.com.conf"))) @@ -612,9 +606,8 @@ class RenewableCertTests(BaseRenewableCertTest): from letsencrypt import storage mock_uln.return_value = "this_does_not_end_with_dot_conf", "yikes" self.assertRaises(errors.CertStorageError, - storage.RenewableCert.new_lineage, - "example.com", "cert", "privkey", "chain", - None, self.defaults, self.cli_config) + storage.RenewableCert.new_lineage, "example.com", + "cert", "privkey", "chain", self.cli_config) def test_bad_kind(self): self.assertRaises( From ca8b4751adbb1edefa6880133ee849a147d44e5d Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 15:13:17 -0800 Subject: [PATCH 03/22] Update config in save_successor --- letsencrypt/storage.py | 36 ++++++++++++++++++++++++++++++- letsencrypt/tests/storage_test.py | 25 ++++++++++++--------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 8cc26d5b5..70a4ea302 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -87,6 +87,31 @@ def write_renewal_config(filename, target, cli_config): return config +def update_configuration(lineagename, target, cli_config): + """Modifies lineagename's config to contain the specified values. + + :param str lineagename: Name of the lineage being modified + :param dict target: Maps ALL_FOUR to their symlink paths + :param .RenewerConfiguration cli_config: parsed command line + arguments + + :returns: Configuration object for the updated config file + :rtype: configobj.ConfigObj + + """ + config_filename = os.path.join( + cli_config.renewal_configs_dir, lineagename) + ".conf" + temp_filename = config_filename + ".new" + + # If an existing tempfile exists, delete it + if os.path.exists(temp_filename): + os.unlink(temp_filename) + write_renewal_config(temp_filename, target, cli_config) + os.rename(temp_filename, config_filename) + + return configobj.ConfigObj(config_filename) + + class RenewableCert(object): # pylint: disable=too-many-instance-attributes """Renewable certificate. @@ -711,7 +736,8 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes new_config = write_renewal_config(config_filename, target, cli_config) return cls(new_config.filename, cli_config) - def save_successor(self, prior_version, new_cert, new_privkey, new_chain): + def save_successor(self, prior_version, new_cert, + new_privkey, new_chain, cli_config): """Save new cert and chain as a successor of a prior version. Returns the new version number that was created. @@ -727,6 +753,8 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes :param str new_privkey: the new private key, in PEM format, or ``None``, if the private key has not changed :param str new_chain: the new chain, in PEM format + :param .RenewerConfiguration cli_config: parsed command line + arguments :returns: the new version number that was created :rtype: int @@ -775,4 +803,10 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes with open(target["fullchain"], "w") as f: logger.debug("Writing full chain to %s.", target["fullchain"]) f.write(new_cert + new_chain) + + # Update renewal config file + self.configfile = update_configuration( + self.lineagename, target, cli_config) + self.configuration = config_with_defaults(self.configfile) + return target_version diff --git a/letsencrypt/tests/storage_test.py b/letsencrypt/tests/storage_test.py index e8ef28932..963df60c2 100644 --- a/letsencrypt/tests/storage_test.py +++ b/letsencrypt/tests/storage_test.py @@ -504,8 +504,9 @@ class RenewableCertTests(BaseRenewableCertTest): with open(where, "w") as f: f.write(kind) self.test_rc.update_all_links_to(3) - self.assertEqual(6, self.test_rc.save_successor(3, "new cert", None, - "new chain")) + self.assertEqual( + 6, self.test_rc.save_successor(3, "new cert", None, + "new chain", self.cli_config)) with open(self.test_rc.version("cert", 6)) as f: self.assertEqual(f.read(), "new cert") with open(self.test_rc.version("chain", 6)) as f: @@ -516,10 +517,12 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(os.path.islink(self.test_rc.version("privkey", 3))) self.assertTrue(os.path.islink(self.test_rc.version("privkey", 6))) # Let's try two more updates - self.assertEqual(7, self.test_rc.save_successor(6, "again", None, - "newer chain")) - self.assertEqual(8, self.test_rc.save_successor(7, "hello", None, - "other chain")) + self.assertEqual( + 7, self.test_rc.save_successor(6, "again", None, + "newer chain", self.cli_config)) + self.assertEqual( + 8, self.test_rc.save_successor(7, "hello", None, + "other chain", self.cli_config)) # All of the subsequent versions should link directly to the original # privkey. for i in (6, 7, 8): @@ -532,8 +535,9 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version(kind), 3) # Test updating from latest version rather than old version self.test_rc.update_all_links_to(8) - self.assertEqual(9, self.test_rc.save_successor(8, "last", None, - "attempt")) + self.assertEqual( + 9, self.test_rc.save_successor(8, "last", None, + "attempt", self.cli_config)) for kind in ALL_FOUR: self.assertEqual(self.test_rc.available_versions(kind), range(1, 10)) @@ -542,8 +546,9 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(f.read(), "last" + "attempt") # Test updating when providing a new privkey. The key should # be saved in a new file rather than creating a new symlink. - self.assertEqual(10, self.test_rc.save_successor(9, "with", "a", - "key")) + self.assertEqual( + 10, self.test_rc.save_successor(9, "with", "a", + "key", self.cli_config)) self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) From c9909810353fc9ebd8de85e491e4f68c12436364 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 15:18:31 -0800 Subject: [PATCH 04/22] Pass config in call to save_successor --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 92e985313..832cc9117 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -428,8 +428,8 @@ def _auth_from_domains(le_client, config, domains): lineage.save_successor( lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), - new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain)) - + new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain), + configuration.RenewerConfiguration(config)) lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor # TODO: Also update lineage renewal config with any relevant From b60583e416610f59226907cb9857ea78e3868493 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 15:47:46 -0800 Subject: [PATCH 05/22] Test deleting stray .new file --- letsencrypt/tests/storage_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/letsencrypt/tests/storage_test.py b/letsencrypt/tests/storage_test.py index 963df60c2..9d402089c 100644 --- a/letsencrypt/tests/storage_test.py +++ b/letsencrypt/tests/storage_test.py @@ -544,6 +544,10 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertEqual(self.test_rc.current_version(kind), 8) with open(self.test_rc.version("fullchain", 9)) as f: self.assertEqual(f.read(), "last" + "attempt") + temp_config_file = os.path.join(self.cli_config.renewal_configs_dir, + self.test_rc.lineagename) + ".conf.new" + with open(temp_config_file, "w") as f: + f.write("We previously crashed while writing me :(") # Test updating when providing a new privkey. The key should # be saved in a new file rather than creating a new symlink. self.assertEqual( @@ -551,6 +555,7 @@ class RenewableCertTests(BaseRenewableCertTest): "key", self.cli_config)) self.assertTrue(os.path.exists(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) + self.assertFalse(os.path.exists(temp_config_file)) def test_new_lineage(self): """Test for new_lineage() class method.""" From db9a931fb050f526b5e37b56e0a8887ff7f86f88 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 16:10:47 -0800 Subject: [PATCH 06/22] Update cli_config in save_successor --- letsencrypt/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index 70a4ea302..e338c890a 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -766,6 +766,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes # if needed (ensuring their permissions are correct) # Figure out what the new version is and hence where to save things + self.cli_config = cli_config target_version = self.next_free_version() archive = self.cli_config.archive_dir prefix = os.path.join(archive, self.lineagename) From 04f13a9cf83bf1bae9ca24086fd15947ef264a0f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 16:22:52 -0800 Subject: [PATCH 07/22] Use symlinks not their targets --- letsencrypt/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt/storage.py b/letsencrypt/storage.py index e338c890a..67d40a58f 100644 --- a/letsencrypt/storage.py +++ b/letsencrypt/storage.py @@ -805,9 +805,10 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes logger.debug("Writing full chain to %s.", target["fullchain"]) f.write(new_cert + new_chain) + symlinks = dict((kind, self.configuration[kind]) for kind in ALL_FOUR) # Update renewal config file self.configfile = update_configuration( - self.lineagename, target, cli_config) + self.lineagename, symlinks, cli_config) self.configuration = config_with_defaults(self.configfile) return target_version From 41bb67f7a269ba82995c3dd1ff9194072aea0290 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 Feb 2016 16:29:36 -0800 Subject: [PATCH 08/22] Use .namespace --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 832cc9117..8c33ddfd0 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -429,7 +429,7 @@ def _auth_from_domains(le_client, config, domains): lineage.latest_common_version(), OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, new_certr.body.wrapped), new_key.pem, crypto_util.dump_pyopenssl_chain(new_chain), - configuration.RenewerConfiguration(config)) + configuration.RenewerConfiguration(config.namespace)) lineage.update_all_links_to(lineage.latest_common_version()) # TODO: Check return value of save_successor # TODO: Also update lineage renewal config with any relevant From df111febfd0237930157bbd7583a8f16d361f0cd Mon Sep 17 00:00:00 2001 From: TheNavigat Date: Thu, 4 Feb 2016 05:48:00 +0200 Subject: [PATCH 09/22] Fixing parameter type for obtain_certificate's domains parameter --- letsencrypt/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/client.py b/letsencrypt/client.py index c2dfca1bf..dee5866ea 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -249,7 +249,7 @@ class Client(object): `.register` must be called before `.obtain_certificate` - :param set domains: domains to get a certificate + :param list domains: domains to get a certificate :returns: `.CertificateResource`, certificate chain (as returned by `.fetch_chain`), and newly generated private key From 8fdcb772d9047318c8b678a41af4eb8f8f095452 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 09:29:11 -0800 Subject: [PATCH 10/22] return failure --- tests/letstest/scripts/test_renew_standalone.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh index 955cb104a..b9f00efe0 100755 --- a/tests/letstest/scripts/test_renew_standalone.sh +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -20,3 +20,7 @@ ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem if [ $? -ne 0 ] ; then FAIL=1 fi + +if [ "$FAIL" = 1 ] ; then + exit 1 +fi From dc9a51b2e68c65ecda197842835d13e8cd94fbc3 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 09:38:45 -0800 Subject: [PATCH 11/22] make a robust test script --- .../letstest/scripts/test_renew_standalone.sh | 55 ++++++++++++++----- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/tests/letstest/scripts/test_renew_standalone.sh b/tests/letstest/scripts/test_renew_standalone.sh index b9f00efe0..d90ae9ab6 100755 --- a/tests/letstest/scripts/test_renew_standalone.sh +++ b/tests/letstest/scripts/test_renew_standalone.sh @@ -1,26 +1,55 @@ #!/bin/bash -x -# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution - -# with curl, instance metadata available from EC2 metadata service: -#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname) -#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4) -#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4) +# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL +# are dynamically set at execution +# run letsencrypt-apache2 via letsencrypt-auto cd letsencrypt -./letsencrypt-auto certonly -v --standalone --debug \ - --text --agree-dev-preview --agree-tos \ - --renew-by-default --redirect \ - --register-unsafely-without-email \ - --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -./letsencrypt-auto renew --renew-by-default +export SUDO=sudo +if [ -f /etc/debian_version ] ; then + echo "Bootstrapping dependencies for Debian-based OSes..." + $SUDO bootstrap/_deb_common.sh +elif [ -f /etc/redhat-release ] ; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + $SUDO bootstrap/_rpm_common.sh +else + echo "Dont have bootstrapping for this OS!" + exit 1 +fi -ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem +bootstrap/dev/venv.sh +sudo venv/bin/letsencrypt certonly --debug --standalone -t --agree-dev-preview --agree-tos \ + --renew-by-default --redirect --register-unsafely-without-email \ + --domain $PUBLIC_HOSTNAME --server $BOULDER_URL -v if [ $? -ne 0 ] ; then FAIL=1 fi +if [ "$OS_TYPE" = "ubuntu" ] ; then + venv/bin/tox -e apacheconftest +else + echo Not running hackish apache tests on $OS_TYPE +fi + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +sudo venv/bin/letsencrypt renew --renew-by-default + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + +ls /etc/letsencrypt/archive/$PUBLIC_HOSTNAME | grep -q 2.pem + +if [ $? -ne 0 ] ; then + FAIL=1 +fi + +# return error if any of the subtests failed if [ "$FAIL" = 1 ] ; then exit 1 fi From 375543eb3208bd8171d1709000c1965d8a71b4d1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 14:43:05 -0800 Subject: [PATCH 12/22] Hoping to see if integration test is really renewing --- tests/boulder-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 5d9ed4859..8c5a93e39 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -51,7 +51,7 @@ common renew sed -i "4arenew_before_expiry = 10 years" "$root/conf/renewal/le1.wtf.conf" common renew -# letsencrypt-renewer $store_flags +ls "$root/conf/archive/le1.wtf" # dir="$root/conf/archive/le1.wtf" # for x in cert chain fullchain privkey; # do From e14feb2919ecdda071e743137f2d6c0fb3a6cd6b Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 15:44:54 -0800 Subject: [PATCH 13/22] renew should imply noninteractive --- letsencrypt/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 3a68eab99..88c23c24b 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -859,7 +859,6 @@ def renew(cli_config, unused_plugins): # XXX: does this succeed in making a fully independent config object # each time? config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) - config.noninteractive_mode = True # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). @@ -1700,6 +1699,8 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) + elif config.renew: + displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() zope.component.provideUtility(displayer) From e2e0dddaa4d31ed7676d17ff709c2f77b4e4f7b1 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 16:04:53 -0800 Subject: [PATCH 14/22] Responding to comments (logger.debug, reject --csr in renew) --- letsencrypt/cli.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 3a68eab99..c43eeaadb 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -229,7 +229,7 @@ def _find_duplicative_certs(config, domains): candidate_lineage = storage.RenewableCert(renewal_file, cli_config) except (errors.CertStorageError, IOError): logger.warning("Renewal conf file %s is broken. Skipping.", renewal_file) - logger.info("Traceback was:\n%s", traceback.format_exc()) + logger.debug("Traceback was:\n%s", traceback.format_exc()) continue # TODO: Handle these differently depending on whether they are # expired or still valid? @@ -248,8 +248,10 @@ def _find_duplicative_certs(config, domains): def _treat_as_renewal(config, domains): - """Determine whether there are duplicated names and how to handle them - (renew, reinstall, newcert, or no action). + """Determine whether there are duplicated names and how to handle + them (renew, reinstall, newcert, or raising an error to stop + the client run if the user chooses to cancel the operation when + prompted). :returns: Two-element tuple containing desired new-certificate behavior as a string token ("reinstall", "renew", or "newcert"), plus either @@ -852,6 +854,10 @@ def renew(cli_config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") + if cli_config.csr is not None: + raise errors.Error("Currently, the renew verb cannot be used when " + "specifying a CSR file. Please try the certonly " + "command instead.") for renewal_file in _renewal_conf_files(cli_config): if not renewal_file.endswith(".conf"): continue From 893918de00453a4552c64cc43eba1bdba14d8b66 Mon Sep 17 00:00:00 2001 From: Seth Schoen Date: Thu, 4 Feb 2016 16:07:56 -0800 Subject: [PATCH 15/22] Check for config.verb == "renew" rather than config.renew --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 5443e0ac7..9036804cb 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1705,7 +1705,7 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) - elif config.renew: + elif config.verb == "renew": displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From 8dfb2a1d4c9ace1d8153a3d7a3b6c84916cba6d7 Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Thu, 4 Feb 2016 16:09:42 -0800 Subject: [PATCH 16/22] check verb --- letsencrypt/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 88c23c24b..570484f46 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -1699,7 +1699,7 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) - elif config.renew: + elif config.verb == renew: displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() From dace0aecfaee06af3c9e5cbfcc49ceebade3cb56 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 16:23:39 -0800 Subject: [PATCH 17/22] I missed these --- letsencrypt/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 8c33ddfd0..a4fa409c1 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -159,7 +159,7 @@ def _determine_account(config): acc = accounts[0] else: # no account registered yet if config.email is None and not config.register_unsafely_without_email: - config.email = display_ops.get_email() + config.namespace.email = display_ops.get_email() def _tos_cb(regr): if config.tos: @@ -181,7 +181,7 @@ def _determine_account(config): raise errors.Error( "Unable to register an account with ACME server") - config.account = acc.id + config.namespace.account = acc.id return acc, acme From 7dd1ea4dcfc2160e616af4fe27e628b61c46758e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 17:30:52 -0800 Subject: [PATCH 18/22] Kill this now plz --- letsencrypt/configuration.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/letsencrypt/configuration.py b/letsencrypt/configuration.py index 04053c8c3..37eaba3bd 100644 --- a/letsencrypt/configuration.py +++ b/letsencrypt/configuration.py @@ -84,15 +84,10 @@ class RenewerConfiguration(object): def __init__(self, namespace): self.namespace = namespace - # We're done setting up the attic. Now pull up the ladder after ourselves... - self.__setattr__ = self.__setattr_implementation__ def __getattr__(self, name): return getattr(self.namespace, name) - def __setattr_implementation__(self, var, value): - return self.namespace.__setattr__(var, value) - @property def archive_dir(self): # pylint: disable=missing-docstring return os.path.join(self.namespace.config_dir, constants.ARCHIVE_DIR) From b2dae6cae27335276653ec6c466e1fe6a093cc00 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:10:39 -0800 Subject: [PATCH 19/22] Fixed it? --- letsencrypt/cli.py | 64 +++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 204327e13..e4f841d50 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -727,7 +727,15 @@ def install(config, plugins): le_client.enhance_config(domains, config) -def _restore_required_config_elements(full_path, config, renewalparams): +def _restore_required_config_elements(config, renewalparams): + """Sets non-plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ # string-valued items to add if they're present for config_item in STR_CONFIG_ITEMS: if config_item in renewalparams: @@ -744,12 +752,18 @@ def _restore_required_config_elements(full_path, config, renewalparams): value = int(renewalparams[config_item]) setattr(config.namespace, config_item, value) except ValueError: - logger.warning("Renewal configuration file %s specifies " - "a non-numeric value for %s. Skipping.", - full_path, config_item) - raise + raise errors.Error( + "Expected a numeric value for {0}".format(config_item)) def _restore_plugin_configs(config, renewalparams): + """Sets plugin specific values in config from renewalparams + + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param configobj.Section renewalparams: Parameters from the renewal + configuration file that defines this lineage + + """ # Now use parser to get plugin-prefixed items with correct types # XXX: the current approach of extracting only prefixed items # related to the actually-used installer and authenticator @@ -782,7 +796,7 @@ def _restore_plugin_configs(config, renewalparams): return True -def _reconstitute(full_path, config): +def _reconstitute(config, full_path): """Try to instantiate a RenewableCert, updating config with relevant items. This is specifically for use in renewal and enforces several checks @@ -790,12 +804,18 @@ def _reconstitute(full_path, config): request. The config argument is modified by including relevant options read from the renewal configuration file. + :param configuration.NamespaceConfig config: configuration for the + current lineage + :param str full_path: Absolute path to the configuration file that + defines this lineage + :returns: the RenewableCert object or None if a fatal error occurred :rtype: `storage.RenewableCert` or NoneType - """ + """ try: - renewal_candidate = storage.RenewableCert(full_path, config) + renewal_candidate = storage.RenewableCert( + full_path, configuration.RenewerConfiguration(config)) except (errors.CertStorageError, IOError): logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.info("Traceback was:\n%s", traceback.format_exc()) @@ -812,11 +832,12 @@ def _reconstitute(full_path, config): # Now restore specific values along with their data types, if # those elements are present. try: - _restore_required_config_elements(full_path, config, renewalparams) + _restore_required_config_elements(config, renewalparams) _restore_plugin_configs(config, renewalparams) - except ValueError: - # There was a data type error which has already been - # logged. + except (ValueError, errors.Error) as error: + logger.warning( + "An error occured while parsing %s. The error was %s. " + "Skipping the file.", full_path, error.message) return None # webroot_map is, uniquely, a dict, and the general-purpose @@ -843,10 +864,9 @@ def _renewal_conf_files(config): return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) -def renew(cli_config, unused_plugins): +def renew(config, unused_plugins): """Renew previously-obtained certificates.""" - cli_config = configuration.RenewerConfiguration(cli_config) - if cli_config.domains != []: + if config.domains != []: raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " "to be renewed; individual domains cannot be " @@ -854,22 +874,24 @@ def renew(cli_config, unused_plugins): "renew specific certificates, use the certonly " "command. The renew verb may provide other options " "for selecting certificates to renew in the future.") - if cli_config.csr is not None: + if config.csr is not None: raise errors.Error("Currently, the renew verb cannot be used when " "specifying a CSR file. Please try the certonly " "command instead.") - for renewal_file in _renewal_conf_files(cli_config): + renewer_config = configuration.RenewerConfiguration(config) + for renewal_file in _renewal_conf_files(renewer_config): if not renewal_file.endswith(".conf"): continue print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - config = configuration.RenewerConfiguration(copy.deepcopy(cli_config)) + lineage_config = configuration.RenewerConfiguration( + copy.deepcopy(config)) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(renewal_file, config) + renewal_candidate = _reconstitute(renewal_file, lineage_config) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -882,14 +904,14 @@ def renew(cli_config, unused_plugins): # already been logged. Go on to the next config. continue # XXX: ensure that each call here replaces the previous one - zope.component.provideUtility(config) + zope.component.provideUtility(lineage_config) print("Trying...") # Because obtain_cert itself indirectly decides whether to renew # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(config, plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From 36a42d18304a502393a769e5fb37025abd6a5232 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:15:39 -0800 Subject: [PATCH 20/22] Tracebacks are useful --- letsencrypt/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index e4f841d50..2838e8395 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -838,6 +838,7 @@ def _reconstitute(config, full_path): logger.warning( "An error occured while parsing %s. The error was %s. " "Skipping the file.", full_path, error.message) + logger.debug("Traceback was:\n%s", traceback.format_exc()) return None # webroot_map is, uniquely, a dict, and the general-purpose From b4f1d94d096e735cbaeca35a381977a7460fabbe Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:21:36 -0800 Subject: [PATCH 21/22] less nesting + fixed argument order --- letsencrypt/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 2838e8395..370024a25 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -892,7 +892,7 @@ def renew(config, unused_plugins): # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). try: - renewal_candidate = _reconstitute(renewal_file, lineage_config) + renewal_candidate = _reconstitute(lineage_config, renewal_file) except Exception as e: # pylint: disable=broad-except # reconstitute encountered an unanticipated problem. logger.warning("Renewal configuration file %s produced an " @@ -912,7 +912,8 @@ def renew(config, unused_plugins): # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config.namespace, + plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config From 8c4721531886499f5d9f7c86590b706b56b74e66 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 Feb 2016 18:28:10 -0800 Subject: [PATCH 22/22] More unnesting --- letsencrypt/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 370024a25..14298a645 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -886,8 +886,7 @@ def renew(config, unused_plugins): print("Processing " + renewal_file) # XXX: does this succeed in making a fully independent config object # each time? - lineage_config = configuration.RenewerConfiguration( - copy.deepcopy(config)) + lineage_config = copy.deepcopy(config) # Note that this modifies config (to add back the configuration # elements from within the renewal configuration file). @@ -912,8 +911,7 @@ def renew(config, unused_plugins): # or not, we couldn't currently make a UI/logging distinction at # this stage to indicate whether renewal was actually attempted # (or successful). - obtain_cert(lineage_config.namespace, - plugins_disco.PluginsRegistry.find_all(), + obtain_cert(lineage_config, plugins_disco.PluginsRegistry.find_all(), renewal_candidate) def revoke(config, unused_plugins): # TODO: coop with renewal config