diff --git a/CHANGELOG.md b/CHANGELOG.md index 3189f7686..cec3eded3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). ### Added -* +* Noninteractive renewals with `certbot renew` (those not started from a + terminal) now randomly sleep 1-480 seconds before beginning work in + order to spread out load spikes on the server side. ### Changed diff --git a/certbot/compat.py b/certbot/compat.py index d42febe81..7e509206f 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -172,3 +172,30 @@ def compare_file_modes(mode1, mode2): # Windows specific: most of mode bits are ignored on Windows. Only check user R/W rights. return (stat.S_IMODE(mode1) & stat.S_IREAD == stat.S_IMODE(mode2) & stat.S_IREAD and stat.S_IMODE(mode1) & stat.S_IWRITE == stat.S_IMODE(mode2) & stat.S_IWRITE) + +WINDOWS_DEFAULT_FOLDERS = { + 'config': 'C:\\Certbot', + 'work': 'C:\\Certbot\\lib', + 'logs': 'C:\\Certbot\\log', +} +LINUX_DEFAULT_FOLDERS = { + 'config': '/etc/letsencrypt', + 'work': '/var/letsencrypt/lib', + 'logs': '/var/letsencrypt/log', +} + +def get_default_folder(folder_type): + """ + Return the relevant default folder for the current OS + + :param str folder_type: The type of folder to retrieve (config, work or logs) + + :returns: The relevant default folder. + :rtype: str + + """ + if 'fcntl' in sys.modules: + # Linux specific + return LINUX_DEFAULT_FOLDERS[folder_type] + # Windows specific + return WINDOWS_DEFAULT_FOLDERS[folder_type] diff --git a/certbot/constants.py b/certbot/constants.py index a2de2d27a..5bf68589d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -4,7 +4,7 @@ import os import pkg_resources from acme import challenges - +from certbot import compat SETUPTOOLS_PLUGINS_ENTRY_POINT = "certbot.plugins" """Setuptools entry point group name for plugins.""" @@ -14,7 +14,7 @@ OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT = "letsencrypt.plugins" CLI_DEFAULTS = dict( config_files=[ - "/etc/letsencrypt/cli.ini", + os.path.join(compat.get_default_folder('config'), 'cli.ini'), # http://freedesktop.org/wiki/Software/xdg-user-dirs/ os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), @@ -85,9 +85,9 @@ CLI_DEFAULTS = dict( auth_cert_path="./cert.pem", auth_chain_path="./chain.pem", key_path=None, - config_dir="/etc/letsencrypt", - work_dir="/var/lib/letsencrypt", - logs_dir="/var/log/letsencrypt", + config_dir=compat.get_default_folder('config'), + work_dir=compat.get_default_folder('work'), + logs_dir=compat.get_default_folder('logs'), server="https://acme-v02.api.letsencrypt.org/directory", # Plugins parsers diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 1e15a8474..3dae1070b 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -4,9 +4,11 @@ import os import zope.component +from certbot import compat from certbot import errors from certbot import interfaces from certbot import util + from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -33,7 +35,8 @@ def get_email(invalid=False, optional=True): unsafe_suggestion = ("\n\nIf you really want to skip this, you can run " "the client with --register-unsafely-without-email " "but make sure you then backup your account key from " - "/etc/letsencrypt/accounts\n\n") + "{0}\n\n".format(os.path.join( + compat.get_default_folder('config'), 'accounts'))) if optional: if invalid: msg += unsafe_suggestion diff --git a/certbot/main.py b/certbot/main.py index 5d5251dd2..692dafeed 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,7 +4,9 @@ from __future__ import print_function import functools import logging.handlers import os +import random import sys +import time import configobj import josepy as jose @@ -1243,6 +1245,16 @@ def renew(config, unused_plugins): :rtype: None """ + if not sys.stdin.isatty(): + # Noninteractive renewals include a random delay in order to spread + # out the load on the certificate authority servers, even if many + # users all pick the same time for renewals. This delay precedes + # running any hooks, so that side effects of the hooks (such as + # shutting down a web service) aren't prolonged unnecessarily. + sleep_time = random.randint(1, 60*8) + logger.info("Non-interactive renewal: random delay of %s seconds", sleep_time) + time.sleep(sleep_time) + try: renewal.handle_renewal_request(config) finally: diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index f42319c4f..adfeff872 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -520,6 +520,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met '--work-dir', self.config.work_dir, '--logs-dir', self.config.logs_dir, '--text'] + self.mock_sleep = mock.patch('time.sleep').start() + def tearDown(self): # Reset globals in cli reload_module(cli) @@ -944,8 +946,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met @mock.patch('certbot.crypto_util.notAfter') @test_util.patch_get_utility() def test_certonly_new_request_success(self, mock_get_utility, mock_notAfter): - cert_path = '/etc/letsencrypt/live/foo.bar' - key_path = '/etc/letsencrypt/live/baz.qux' + cert_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/foo.bar')) + key_path = os.path.normpath(os.path.join(self.config.config_dir, 'live/baz.qux')) date = '1970-01-01' mock_notAfter().date.return_value = date @@ -975,7 +977,8 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met reuse_key=False): # pylint: disable=too-many-locals,too-many-arguments,too-many-branches cert_path = test_util.vector_path('cert_512.pem') - chain_path = '/etc/letsencrypt/live/foo.bar/fullchain.pem' + chain_path = os.path.normpath(os.path.join(self.config.config_dir, + 'live/foo.bar/fullchain.pem')) mock_lineage = mock.MagicMock(cert=cert_path, fullchain=chain_path, cert_path=cert_path, fullchain_path=chain_path) mock_lineage.should_autorenew.return_value = due_for_renewal @@ -1092,6 +1095,26 @@ class MainTest(test_util.ConfigTestCase): # pylint: disable=too-many-public-met args = ["renew", "--reuse-key"] self._test_renewal_common(True, [], args=args, should_renew=True, reuse_key=True) + @mock.patch('sys.stdin') + def test_noninteractive_renewal_delay(self, stdin): + stdin.isatty.return_value = False + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, should_renew=True) + self.assertEqual(self.mock_sleep.call_count, 1) + # in main.py: + # sleep_time = random.randint(1, 60*8) + sleep_call_arg = self.mock_sleep.call_args[0][0] + self.assertTrue(1 <= sleep_call_arg <= 60*8) + + @mock.patch('sys.stdin') + def test_interactive_no_renewal_delay(self, stdin): + stdin.isatty.return_value = True + test_util.make_lineage(self.config.config_dir, 'sample-renewal.conf') + args = ["renew", "--dry-run", "-tvv"] + self._test_renewal_common(True, [], args=args, should_renew=True) + self.assertEqual(self.mock_sleep.call_count, 0) + @mock.patch('certbot.renewal.should_renew') def test_renew_skips_recent_certs(self, should_renew): should_renew.return_value = False diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 3f5eec88d..f8baf9052 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -912,6 +912,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -1017,43 +1046,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse @@ -1224,13 +1249,11 @@ UNLIKELY_EOF cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -1269,7 +1292,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1281,7 +1304,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1357,10 +1380,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -1374,7 +1395,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1387,7 +1408,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1406,6 +1427,7 @@ def main(): if __name__ == '__main__': exit(main()) + UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: @@ -1649,7 +1671,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 6d2977832..f431e32e4 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -451,6 +451,35 @@ OldVenvExists() { [ -n "$OLD_VENV_PATH" -a -f "$OLD_VENV_PATH/bin/letsencrypt" ] } +# Given python path, version 1 and version 2, check if version 1 is outdated compared to version 2. +# An unofficial version provided as version 1 (eg. 0.28.0.dev0) will be treated +# specifically by printing "UNOFFICIAL". Otherwise, print "OUTDATED" if version 1 +# is outdated, and "UP_TO_DATE" if not. +# This function relies only on installed python environment (2.x or 3.x) by certbot-auto. +CompareVersions() { + "$1" - "$2" "$3" << "UNLIKELY_EOF" +import sys +from distutils.version import StrictVersion + +try: + current = StrictVersion(sys.argv[1]) +except ValueError: + sys.stdout.write('UNOFFICIAL') + sys.exit() + +try: + remote = StrictVersion(sys.argv[2]) +except ValueError: + sys.stdout.write('UP_TO_DATE') + sys.exit() + +if current < remote: + sys.stdout.write('OUTDATED') +else: + sys.stdout.write('UP_TO_DATE') +UNLIKELY_EOF +} + if [ "$1" = "--le-auto-phase2" ]; then # Phase 2: Create venv, install LE, and run. @@ -635,7 +664,12 @@ UNLIKELY_EOF error "WARNING: couldn't find Python $MIN_PYTHON_VERSION+ to check for updates." elif ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then error "WARNING: unable to check for updates." - elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + fi + + LE_VERSION_STATE=`CompareVersions "$LE_PYTHON" "$LE_AUTO_VERSION" "$REMOTE_VERSION"` + if [ "$LE_VERSION_STATE" = "UNOFFICIAL" ]; then + say "Unofficial certbot-auto version detected, self-upgrade is disabled: $LE_AUTO_VERSION" + elif [ "$LE_VERSION_STATE" = "OUTDATED" ]; then say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 78f430210..983d2bb95 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -21,43 +21,39 @@ pycparser==2.14 \ asn1crypto==0.22.0 \ --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \ --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a -cffi==1.10.0 \ - --hash=sha256:446699c10f3c390633d0722bc19edbc7ac4b94761918a4a4f7908a24e86ebbd0 \ - --hash=sha256:562326fc7f55a59ef3fef5e82908fe938cdc4bbda32d734c424c7cd9ed73e93a \ - --hash=sha256:7f732ad4a30db0b39400c3f7011249f7d0701007d511bf09604729aea222871f \ - --hash=sha256:94fb8410c6c4fc48e7ea759d3d1d9ca561171a88d00faddd4aa0306f698ad6a0 \ - --hash=sha256:587a5043df4b00a2130e09fed42da02a4ed3c688bd9bf07a3ac89d2271f4fb07 \ - --hash=sha256:ec08b88bef627ec1cea210e1608c85d3cf44893bcde74e41b7f7dbdfd2c1bad6 \ - --hash=sha256:a41406f6d62abcdf3eef9fd998d8dcff04fd2a7746644143045feeebd76352d1 \ - --hash=sha256:b560916546b2f209d74b82bdbc3223cee9a165b0242fa00a06dfc48a2054864a \ - --hash=sha256:e74896774e437f4715c57edeb5cf3d3a40d7727f541c2c12156617b5a15d1829 \ - --hash=sha256:9a31c18ba4881a116e448c52f3f5d3e14401cf7a9c43cc88f06f2a7f5428da0e \ - --hash=sha256:80796ea68e11624a0279d3b802f88a7fe7214122b97a15a6c97189934a2cc776 \ - --hash=sha256:f4019826a2dec066c909a1f483ef0dcf9325d6740cc0bd15308942b28b0930f7 \ - --hash=sha256:7248506981eeba23888b4140a69a53c4c0c0a386abcdca61ed8dd790a73e64b9 \ - --hash=sha256:a8955265d146e86fe2ce116394be4eaf0cb40314a79b19f11c4fa574cd639572 \ - --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \ - --hash=sha256:c1d8b3d8dcb5c23ac1a8bf56422036f3f305a3c5a8bc8c354256579a1e2aa2c1 \ - --hash=sha256:9e389615bcecb8c782a87939d752340bb0a3a097e90bae54d7f0915bc12f45bd \ - --hash=sha256:d09ff358f75a874f69fa7d1c2b4acecf4282a950293fcfcf89aa606da8a9a500 \ - --hash=sha256:b69b4557aae7de18b7c174a917fe19873529d927ac592762d9771661875bbd40 \ - --hash=sha256:5de52b081a2775e76b971de9d997d85c4457fc0a09079e12d66849548ae60981 \ - --hash=sha256:e7d88fecb7b6250a1fd432e6dc64890342c372fce13dbfe4bb6f16348ad00c14 \ - --hash=sha256:1426e67e855ef7f5030c9184f4f1a9f4bfa020c31c962cd41fd129ec5aef4a6a \ - --hash=sha256:267dd2c66a5760c5f4d47e2ebcf8eeac7ef01e1ae6ae7a6d0d241a290068bc38 \ - --hash=sha256:e553eb489511cacf19eda6e52bc9e151316f0d721724997dda2c4d3079b778db \ - --hash=sha256:98b89b2c57f97ce2db7aeba60db173c84871d73b40e41a11ea95de1500ddc57e \ - --hash=sha256:e2b7e090188833bc58b2ae03fb864c22688654ebd2096bcf38bc860c4f38a3d8 \ - --hash=sha256:afa7d8b8d38ad40db8713ee053d41b36d87d6ae5ec5ad36f9210b548a18dc214 \ - --hash=sha256:4fc9c2ff7924b3a1fa326e1799e5dd58cac585d7fb25fe53ccaa1333b0453d65 \ - --hash=sha256:937db39a1ec5af3003b16357b2042bba67c88d43bc11aaa203fa8a5924524209 \ - --hash=sha256:ab22285797631df3b513b2cd3ecdc51cd8e3d36788e3991d93d0759d6883b027 \ - --hash=sha256:96e599b924ef009aa867f725b3249ee51d76489f484d3a45b4bd219c5ec6ed59 \ - --hash=sha256:bea842a0512be6a8007e585790bccd5d530520fc025ce63b03e139be373b0063 \ - --hash=sha256:e7175287f7fe7b1cc203bb958b17db40abd732690c1e18e700f10e0843a58598 \ - --hash=sha256:285ab352552f52f1398c912556d4d36d4ea9b8450e5c65d03809bf9886755533 \ - --hash=sha256:5576644b859197da7bbd8f8c7c2fb5dcc6cd505cadb42992d5f104c013f8a214 \ - --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5 +cffi==1.11.5 \ + --hash=sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50 \ + --hash=sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596 \ + --hash=sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef \ + --hash=sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743 \ + --hash=sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f \ + --hash=sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31 \ + --hash=sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04 \ + --hash=sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6 \ + --hash=sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3 \ + --hash=sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6 \ + --hash=sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b \ + --hash=sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca \ + --hash=sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e \ + --hash=sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb \ + --hash=sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd \ + --hash=sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1 \ + --hash=sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917 \ + --hash=sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359 \ + --hash=sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f \ + --hash=sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95 \ + --hash=sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801 \ + --hash=sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257 \ + --hash=sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184 \ + --hash=sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc \ + --hash=sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085 \ + --hash=sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93 \ + --hash=sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2 \ + --hash=sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30 \ + --hash=sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5 \ + --hash=sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e \ + --hash=sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b \ + --hash=sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4 ConfigArgParse==0.12.0 \ --hash=sha256:28cd7d67669651f2a4518367838c49539457504584a139709b2b8f6c208ef339 \ --no-binary ConfigArgParse diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index f21d36657..727040c3c 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -1,12 +1,10 @@ #!/usr/bin/env python """A small script that can act as a trust root for installing pip >=8 - Embed this in your project, and your VCS checkout is all you have to trust. In a post-peep era, this lets you claw your way to a hash-checking version of pip, with which you can install the rest of your dependencies safely. All it assumes is Python 2.6 or better and *some* version of pip already installed. If anything goes wrong, it will exit with a non-zero status code. - """ # This is here so embedded copies are MIT-compliant: # Copyright (c) 2016 Erik Rose @@ -45,7 +43,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info, executable +from sys import exit, version_info from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +55,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 2, 0, 0 +__version__ = 1, 5, 1 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -133,10 +131,8 @@ def hashed_download(url, temp, digest): def get_index_base(): """Return the URL to the dir containing the "packages" folder. - Try to wring something out of PIP_INDEX_URL, if set. Hack "/simple" off the end if it's there; that is likely to give us the right dir. - """ env_var = environ.get('PIP_INDEX_URL', '').rstrip('/') if env_var: @@ -150,7 +146,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) + pip_version = StrictVersion(check_output(['pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +159,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + + check_output('pip install --no-index --no-deps -U ' + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +177,4 @@ def main(): if __name__ == '__main__': - exit(main()) \ No newline at end of file + exit(main()) diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index c5109e208..16c478f20 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -27,6 +27,19 @@ def tests_dir(): return dirname(abspath(__file__)) +def copy_stable(src, dst): + """ + Copy letsencrypt-auto, and replace its current version to its equivalent stable one. + This is needed to test correctly the self-upgrade functionality. + """ + copy(src, dst) + with open(dst, 'r') as file: + filedata = file.read() + filedata = re.sub(r'LE_AUTO_VERSION="(.*)\.dev0"', r'LE_AUTO_VERSION="\1"', filedata) + with open(dst, 'w') as file: + file.write(filedata) + + sys.path.insert(0, dirname(tests_dir())) from build import build as build_le_auto @@ -343,7 +356,7 @@ class AutoTests(TestCase): 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), 'v99.9.9/letsencrypt-auto.sig': signed('something else')} with serving(resources) as base_url: - copy(LE_AUTO_PATH, le_auto_path) + copy_stable(LE_AUTO_PATH, le_auto_path) try: out, err = run_le_auto(le_auto_path, venv_dir, base_url) except CalledProcessError as exc: diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index f18a64065..0b9a91ffd 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -10,7 +10,7 @@ VERSION=$(letsencrypt-auto-source/version.py) export VENV_ARGS="-p $PYTHON" # setup venv -tools/_venv_common.sh --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt +tools/_venv_common.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . ./venv/bin/activate # build sdists diff --git a/tools/_venv_common.py b/tools/_venv_common.py index c44d05bf7..ecd438f94 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -1,4 +1,14 @@ #!/usr/bin/env python +"""Aids in creating a developer virtual environment for Certbot. + +When this module is run as a script, it takes the arguments that should +be passed to pip to install the Certbot packages as command line +arguments. The virtual environment will be created with the name "venv" +in the current working directory and will use the default version of +Python for the virtualenv executable in your PATH. You can change the +name of the virtual environment by setting the environment variable +VENV_NAME. +""" from __future__ import print_function @@ -9,6 +19,7 @@ import time import subprocess import sys import re +import shlex VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') @@ -91,18 +102,18 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(command): - print(command) - subprocess.check_call(command, shell=True) +def subprocess_with_print(cmd, env=os.environ, shell=False): + print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) + subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_python(venv_path): +def get_venv_bin_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return python_linux + return os.path.abspath(os.path.dirname(python_linux)) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return python_windows + return os.path.abspath(os.path.dirname(python_windows)) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' @@ -110,27 +121,45 @@ def get_venv_python(venv_path): def main(venv_name, venv_args, args): + """Creates a virtual environment and installs packages. + + :param str venv_name: The name or path at where the virtual + environment should be created. + :param str venv_args: Command line arguments for virtualenv + :param str args: Command line arguments that should be given to pip + to install packages + """ + for path in glob.glob('*.egg-info'): if os.path.isdir(path): shutil.rmtree(path) else: os.remove(path) + env_venv_name = os.environ.get('VENV_NAME') + if env_venv_name: + print('Creating venv at {0}' + ' as specified in VENV_NAME'.format(env_venv_name)) + venv_name = env_venv_name + if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' - .format(sys.executable, venv_name, venv_args)) + command = [sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', venv_name] + command.extend(shlex.split(venv_args)) + subprocess_with_print(command) - python_executable = get_venv_python(venv_name) - - subprocess_with_print('"{0}" {1}'.format( - python_executable, - os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) - subprocess_with_print('"{0}" {1} {2}'.format( - python_executable, - os.path.normpath('./tools/pip_install.py'), - ' '.join(args))) + # We execute the following commands in the context of the virtual environment, to install + # the packages in it. To do so, we append the venv binary to the PATH that will be used for + # these commands. With this trick, correct python executable will be selected. + new_environ = os.environ.copy() + new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) + subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), + env=new_environ, shell=True) + subprocess_with_print("python -m pip install --upgrade 'setuptools>=30.3'", + env=new_environ, shell=True) + subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), + env=new_environ, shell=True) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -150,6 +179,6 @@ def main(venv_name, venv_args, args): if __name__ == '__main__': - main(os.environ.get('VENV_NAME', 'venv'), - os.environ.get('VENV_ARGS', ''), + main('venv', + '', sys.argv[1:])