From 4edfb3ef65ee2522f5b09c88793095ebee886bab Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 00:35:09 +0100 Subject: [PATCH 01/11] [Windows] Handle file renaming when the destination path already exists (#6415) On Linux, you can invoke os.rename(src, dst) even if dst already exists. In this case, destination file will be atomically replaced by the source file. On Windows, this will lead to an OSError because changes are not atomic. This cause certbot renew to fail in particular, because the old certificate configuration needs to be replace by the new when a certificate is effectively renewed. One could use the cross-platform function os.replace, but it is available only on Python >= 3.3. This PR add a function in compat to handle correctly this case on Windows, and delegating everything else to os.rename. * Cross platform compatible os.rename (we can use os.replace if its python 3) * Use os.replace instead of custom non-atomic code. * Avoid errors for lint and mypy. Add a test. --- certbot/compat.py | 24 ++++++++++++++++++++++++ certbot/reverter.py | 2 +- certbot/storage.py | 3 ++- certbot/tests/compat_test.py | 21 +++++++++++++++++++++ certbot/tests/reverter_test.py | 2 +- 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 certbot/tests/compat_test.py diff --git a/certbot/compat.py b/certbot/compat.py index e3e1bc4e1..d42febe81 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -65,6 +65,30 @@ def os_geteuid(): # Windows specific return 0 +def os_rename(src, dst): + """ + Rename a file to a destination path and handles situations where the destination exists. + + :param str src: The current file path. + :param str dst: The new file path. + """ + try: + os.rename(src, dst) + except OSError as err: + # Windows specific, renaming a file on an existing path is not possible. + # On Python 3, the best fallback with atomic capabilities we have is os.replace. + if err.errno != errno.EEXIST: + # Every other error is a legitimate exception. + raise + if not hasattr(os, 'replace'): # pragma: no cover + # We should never go on this line. Either we are on Linux and os.rename has succeeded, + # either we are on Windows, and only Python >= 3.4 is supported where os.replace is + # available. + raise RuntimeError('Error: tried to run os_rename on Python < 3.3. ' + 'Certbot supports only Python 3.4 >= on Windows.') + getattr(os, 'replace')(src, dst) + + def readline_with_timeout(timeout, prompt): """ Read user input to return the first line entered, or raise after specified timeout. diff --git a/certbot/reverter.py b/certbot/reverter.py index 5d56615fd..919037358 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -576,7 +576,7 @@ class Reverter(object): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: - os.rename(self.config.in_progress_dir, final_dir) + compat.os_rename(self.config.in_progress_dir, final_dir) return except OSError: logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp) diff --git a/certbot/storage.py b/certbot/storage.py index c16ea35b8..4b8110072 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -14,6 +14,7 @@ import six import certbot from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import errors @@ -188,7 +189,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): # Save only the config items that are relevant to renewal values = relevant_values(vars(cli_config.namespace)) write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - os.rename(temp_filename, config_filename) + compat.os_rename(temp_filename, config_filename) return configobj.ConfigObj(config_filename) diff --git a/certbot/tests/compat_test.py b/certbot/tests/compat_test.py new file mode 100644 index 000000000..552aa5645 --- /dev/null +++ b/certbot/tests/compat_test.py @@ -0,0 +1,21 @@ +"""Tests for certbot.compat.""" +import os + +from certbot import compat +import certbot.tests.util as test_util + +class OsReplaceTest(test_util.TempDirTestCase): + """Test to ensure consistent behavior of os_rename method""" + + def test_os_rename_to_existing_file(self): + """Ensure that os_rename will effectively rename src into dst for all platforms.""" + src = os.path.join(self.tempdir, 'src') + dst = os.path.join(self.tempdir, 'dst') + open(src, 'w').close() + open(dst, 'w').close() + + # On Windows, a direct call to os.rename will fail because dst already exists. + compat.os_rename(src, dst) + + self.assertFalse(os.path.exists(src)) + self.assertTrue(os.path.exists(dst)) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 999c6225c..d04e3c641 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -356,7 +356,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @mock.patch("certbot.reverter.os.rename") + @mock.patch("certbot.reverter.compat.os_rename") def test_finalize_checkpoint_no_rename_directory(self, mock_rename): self.reverter.add_to_checkpoint(self.sets[0], "perm save") From e6e323e3ffd8d74251fa9ad46a6eed1bcffbfe38 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 7 Nov 2018 16:49:13 +0100 Subject: [PATCH 02/11] Update Lexicon to correct use of HTTP proxy on OVH provider (#6479) This PR update requirement of Lexicon to 2.7.14 on OVH plugin, to allow HTTP proxy to be used correctly when underlying OVH provider is invoked. * Update Lexicon to correct use of HTTP proxy on OVH provider * Update dev_constraints.txt * Update CHANGELOG.md --- CHANGELOG.md | 1 + certbot-dns-ovh/setup.py | 2 +- tools/dev_constraints.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df95c2d36..728c938fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Warn when using deprecated acme.challenges.TLSSNI01 * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins +* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies ### Fixed diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 1f3acbf62..89ff317d9 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -11,7 +11,7 @@ version = '0.28.0.dev0' install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.7.3', # Correct OVH integration tests + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 00ecee03e..380d49cb3 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.3 +dns-lexicon==2.7.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 From f3ff548a413a699760ab2239b6e11d65e083d540 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:02:25 -0800 Subject: [PATCH 03/11] Update changelog for 0.28.0 release. --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728c938fb..2f0fd40dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.28.0 - master +## 0.28.0 - 2018-11-7 ### Added @@ -18,6 +18,7 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins * OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies +* Default time the Linode plugin waits for DNS changes to propogate is now 1200 seconds. ### Fixed @@ -27,6 +28,30 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Stop caching the results of ipv6_info in http01.py * Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. +* The CloudXNS, DNSimple, DNS Made Easy, Gehirn, Linode, LuaDNS, NS1, OVH, and + Sakura Cloud DNS plugins are now compatible with Lexicon 3.0+. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-route53 +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/59?closed=1 ## 0.27.1 - 2018-09-06 From c1300a8e1b5b3d7a485895f9a9abe46d582b2975 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:57 -0800 Subject: [PATCH 04/11] Release 0.28.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 28 +++++++++--------- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 22 +++++++------- letsencrypt-auto | 28 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 16 +++++----- letsencrypt-auto-source/letsencrypt-auto | 26 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 +++++++-------- 26 files changed, 91 insertions(+), 91 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 85492d9a3..89337ad71 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aa5908f9e..805e1ff5e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 076c45e39..fe87317a7 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8dac3e047..bdda1bcde 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index b823cf98f..4fa1cd5ec 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3daf933cb..a1083f884 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index feb2a0d58..7d355f5e4 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index b6efcf360..6f6e9181f 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c268eaa8f..1a099d201 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index fc147f85c..1cec98e24 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 86d36bcb3..696de488d 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1d196403f..67592de76 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index a5c06d90e..38d2f144b 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 474093a5b..fa4bde85d 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 89ff317d9..ea2a9aa9f 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c009ef032..b1c1cd0ec 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2bae0c3d0..c42f976ea 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 9f8bfbbdb..3f54bb96f 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3c8a66ee5..d292fba31 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.28.0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index ab23926c9..eecca88e0 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0.dev0' +__version__ = '0.28.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4ed9f0731..d26da361b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -24,7 +24,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.1 + "". (default: CertbotACMEClient/0.28.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -261,7 +261,8 @@ manage: delete Clean up all files related to a certificate renew Renew all certificates (or one specified with --cert- name) - revoke Revoke a certificate specified with --cert-path + revoke Revoke a certificate specified with --cert-path or + --cert-name update_symlinks Recreate symlinks in your /etc/letsencrypt/live/ directory @@ -475,10 +476,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -492,16 +492,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 @@ -602,7 +602,7 @@ dns-linode: --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate before asking the ACME server to verify the DNS - record. (default: 960) + record. (default: 1200) --dns-linode-credentials DNS_LINODE_CREDENTIALS Linode credentials INI file. (default: None) diff --git a/letsencrypt-auto b/letsencrypt-auto index 076c45e39..fe87317a7 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 747d98e2d..57745758b 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X -dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ -nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn -ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa -I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk -AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm -8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== -=QRnC +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X +dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj +zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i +7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq +rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO +zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg +PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q== +=1hD2 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 740cb9c73..fe87317a7 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0.dev0" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index b717e359b2a2c65f6cb588a007f5024572c1e4ff..33f5b2c00007bde13d9f58cf7e48ad17d6998502 100644 GIT binary patch literal 256 zcmV+b0ssC{t4ydxm0o9{N03G=j7lT5c_4fvp=Ak0G{=zFl<_N_YV;M+5gCs_o{c#_ zk}R?1daPA6KJFO+@o7i}i@E(nx&;8|8w zwwanT<%NEK7%&NBc6_9#QU}r> literal 256 zcmV+b0ssEcvxB0LY3lA!OPssV3H<~y?AaAqjuWWrmMIj>00(?0Ek&mFUj?a=WHkII z-RNTHeD@0(kwW#&HKEPltP&F}Il@J^#^yCaE=%~OTz&S0VZ;xAiFO*AjYgLt@?f^0 z5%fBEr@1)1d9JccQ6BE>@C=b&m*{iB+J6X1CVdnJgQ4FUmIMh?O(XLrLz zjb^9RY>s^Yo_H0;@vfC^I1)~)4x~+{{pTYk(%Y}AY2=R!zDwo_S`Vp=%04-BF^8$F zx6n%y(2yohnk6h!P4Oc-;Nb&}2~R5^lWYM-Sp*8Vav}^+f?3H>xwNIK(r-rHpKXnf GZUdUtC3&a- diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index b9cd42694..401a8e25c 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb From 22858c6025c2697ea7842f97a91e366cbac8293a Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 13:22:59 -0800 Subject: [PATCH 05/11] Bump version to 0.29.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-dns-cloudflare/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-digitalocean/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 2 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-google/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-rfc2136/setup.py | 2 +- certbot-dns-route53/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 89337ad71..ad70c2947 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 805e1ff5e..e6f6f1e23 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index bdda1bcde..bfbbe0625 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4fa1cd5ec..d615fa999 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index a1083f884..f9880270a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 7d355f5e4..a9d46d128 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 6f6e9181f..ac7bb1090 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 1a099d201..ab944d40d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 1cec98e24..94ca74761 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 696de488d..aa1afdc93 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 67592de76..4c2571f96 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 38d2f144b..d0735d1ec 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index fa4bde85d..aebff5304 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index ea2a9aa9f..68ede8006 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index b1c1cd0ec..914bfb6e6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index c42f976ea..6318cbb81 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 3f54bb96f..5af4c8a00 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index d292fba31..0908c4c52 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot/__init__.py b/certbot/__init__.py index eecca88e0..f6b7defbd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0' +__version__ = '0.29.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index fe87317a7..b1a05f6e6 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0" +LE_AUTO_VERSION="0.29.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 63e0f5678457b78cfee7fa7f8337a4b8f1809fd8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 7 Nov 2018 15:56:29 -0800 Subject: [PATCH 06/11] update changelog for 0.29.0 --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0fd40dc..a25890929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.29.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/62?closed=1 + ## 0.28.0 - 2018-11-7 ### Added From 3d0e16ece3762d5bfe345fde0af286e26c54bacb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 02:16:16 +0100 Subject: [PATCH 07/11] [Windows|Unix] Rewrite bash scripts for tests into python (#6435) Certbot relies heavily on bash scripts to deploy a development environment and to execute tests. This is fine for Linux systems, including Travis, but problematic for Windows machines. This PR converts all theses scripts into Python, to make them platform independant. As a consequence, tox-win.ini is not needed anymore, and tox can be run indifferently on Windows or on Linux using a common tox.ini. AppVeyor is updated accordingly to execute tests for acme, certbot and all dns plugins. Other tests are not executed as they are for Docker, unsupported Apache/Nginx/Postfix plugins (for now) or not relevant for Windows (explicit Linux distribution tests or pylint). Another PR will be done on certbot website to update how a dev environment can be set up. * Replace several shell scripts by python equivalent. * Correction on tox coverage * Extend usage of new python scripts * Various corrections * Replace venv construction bash scripts by python equivalents * Update tox.ini * Unicode lines to compare files * Put modifications on letsencrypt-auto-source instead of generated scripts * Add executable permissions for Linux. * Merge tox win tests into main tox * Skip lock_test on Windows * Correct appveyor config * Update appveyor.yml * Explicit coverage py27 or py37 * Avoid to cover non supported certbot plugins on Windows * Update tox.ini * Remove specific warnings during CI * No cover on a debug code for tests only. * Update documentation and help script on venv/venv3.py * Customize help message for Windows * Quote correctly executable path with potential spaces in it. * Copy pipstrap from upstream --- .travis.yml | 4 +- Dockerfile-dev | 2 +- appveyor.yml | 25 +++- certbot-compatibility-test/Dockerfile | 5 +- certbot-postfix/README.rst | 2 +- certbot/tests/util.py | 13 +- docs/contributing.rst | 8 +- letsencrypt-auto-source/letsencrypt-auto | 11 +- .../pieces/bootstrappers/arch_common.sh | 2 +- letsencrypt-auto-source/pieces/pipstrap.py | 10 +- tests/letstest/scripts/test_apache2.sh | 2 +- tests/letstest/scripts/test_tox.sh | 2 +- tests/lock_test.py | 9 +- tests/modification-check.py | 124 ++++++++++++++++++ tests/modification-check.sh | 59 --------- tools/_venv_common.py | 67 ++++++++++ tools/_venv_common.sh | 26 ---- tools/install_and_test.py | 58 ++++++++ tools/install_and_test.sh | 29 ---- tools/merge_requirements.py | 16 +-- tools/pip_install.py | 90 +++++++++++++ tools/pip_install.sh | 44 ------- tools/pip_install_editable.py | 19 +++ tools/pip_install_editable.sh | 10 -- tools/readlink.py | 7 +- tools/venv.py | 58 ++++++++ tools/venv.sh | 34 ----- tools/venv3.py | 53 ++++++++ tools/venv3.sh | 33 ----- tox-win.ini | 13 -- tox.cover.py | 85 ++++++++++++ tox.cover.sh | 72 ---------- tox.ini | 28 ++-- 33 files changed, 643 insertions(+), 377 deletions(-) create mode 100755 tests/modification-check.py delete mode 100755 tests/modification-check.sh create mode 100755 tools/_venv_common.py delete mode 100755 tools/_venv_common.sh create mode 100755 tools/install_and_test.py delete mode 100755 tools/install_and_test.sh create mode 100755 tools/pip_install.py delete mode 100755 tools/pip_install.sh create mode 100755 tools/pip_install_editable.py delete mode 100755 tools/pip_install_editable.sh create mode 100755 tools/venv.py delete mode 100755 tools/venv.sh create mode 100755 tools/venv3.py delete mode 100755 tools/venv3.sh delete mode 100644 tox-win.ini create mode 100755 tox.cover.py delete mode 100755 tox.cover.sh diff --git a/.travis.yml b/.travis.yml index acdf365bd..2b8eafc13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=py27-cover FYI="py27 tests + code coverage" - sudo: required env: TOXENV=nginx_compat services: docker @@ -95,7 +95,7 @@ script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && codecov' +after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' notifications: email: false diff --git a/Dockerfile-dev b/Dockerfile-dev index 9e35ebec8..1ab56e081 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -16,6 +16,6 @@ RUN apt-get update && \ /tmp/* \ /var/tmp/* -RUN VENV_NAME="../venv" tools/venv.sh +RUN VENV_NAME="../venv" python tools/venv.py ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/appveyor.yml b/appveyor.yml index 796070081..725ecfbff 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,17 @@ -image: - # => Windows Server 2012 R2 - - Visual Studio 2015 - # => Windows Server 2016 - - Visual Studio 2017 +environment: + matrix: + - FYI: Python 3.4 on Windows Server 2012 R2 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - FYI: Python 3.4 on Windows Server 2016 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.5 on Windows Server 2016 + TOXENV: py35 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.7 on Windows Server 2016 + code coverage + TOXENV: py37-cover + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 branches: only: @@ -14,6 +23,7 @@ install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env + - "echo %APPVEYOR_BUILD_WORKER_IMAGE%" - "python --version" # Upgrade pip to avoid warnings - "python -m pip install --upgrade pip" @@ -23,7 +33,8 @@ install: build: off test_script: - - tox -c tox-win.ini -e py34,py35,py36,py37-cover + # Test env is set by TOXENV env variable + - tox on_success: - - codecov + - if exist .coverage codecov diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index 803b4a1b9..cbbdf7193 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... @@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/src/tools/pip_install_editable.sh \ +RUN /opt/certbot/venv/bin/python \ + /opt/certbot/src/tools/pip_install_editable.py \ /opt/certbot/src/acme \ /opt/certbot/src \ /opt/certbot/src/certbot-apache \ diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index e6367e365..1ae9cb980 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -7,7 +7,7 @@ feature requests for this plugin. To install this plugin, in the root of this repo, run:: - ./tools/venv.sh + python tools/venv.py source venv/bin/activate You can use this installer with any `authenticator plugin diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 822597dd4..8c5db2c2f 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -328,15 +328,16 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # Then we have various files which are not correctly closed at the time of tearDown. - # On Windows, it is visible for the same reasons as above. + # On Windows we have various files which are not correctly closed at the time of tearDown. # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. def onerror_handler(_, path, excinfo): """On error handler""" - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): diff --git a/docs/contributing.rst b/docs/contributing.rst index 58db251d4..ead4d7e2b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -38,13 +38,13 @@ Certbot. cd certbot ./certbot-auto --debug --os-packages-only - tools/venv.sh + python tools/venv.py -If you have Python3 available and want to use it, run the ``venv3.sh`` script. +If you have Python3 available and want to use it, run the ``venv3.py`` script. .. code-block:: shell - tools/venv3.sh + python tools/venv3.py .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -353,7 +353,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./tools/venv.sh``. + virtualenv. You can do this by running ``pip tools/venv.py``. (this is a **very important** step) 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors. 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index b1a05f6e6..12be26e19 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -594,7 +594,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 @@ -1260,7 +1260,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1272,7 +1272,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1365,7 +1365,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1378,7 +1378,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1397,7 +1397,6 @@ def main(): if __name__ == '__main__': exit(main()) - UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 5759336c5..c55527590 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -8,7 +8,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index d55d5bceb..f21d36657 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -45,7 +45,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -150,7 +150,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +163,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +181,4 @@ def main(): if __name__ == '__main__': - exit(main()) + exit(main()) \ No newline at end of file diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 6b5d63c80..4036e6efa 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then exit 1 fi -tools/_venv_common.sh -e acme[dev] -e .[dev,docs] -e certbot-apache +python tools/_venv_common.py -e acme[dev] -e .[dev,docs] -e certbot-apache sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 84e4bcd22..bb9126673 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -14,5 +14,5 @@ VENV_BIN=${VENV_PATH}/bin "$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./tools/venv.sh +python tools/venv.py venv/bin/tox -e py27 diff --git a/tests/lock_test.py b/tests/lock_test.py index b01cc5d58..0266cf029 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,4 +1,6 @@ """Tests to ensure the lock order is preserved.""" +from __future__ import print_function + import atexit import functools import logging @@ -235,4 +237,9 @@ def log_output(level, out, err): if __name__ == "__main__": - main() + if os.name != 'nt': + main() + else: + print( + 'Warning: lock_test cannot be executed on Windows, ' + 'as it relies on a Nginx distribution for Linux.') diff --git a/tests/modification-check.py b/tests/modification-check.py new file mode 100755 index 000000000..e00994b04 --- /dev/null +++ b/tests/modification-check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile +import shutil +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +def find_repo_path(): + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +# We do not use filecmp.cmp to take advantage of universal newlines +# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. +# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. +# But it will work correctly on Linux for any version, because every file tested will be LF. +def compare_files(path_1, path_2): + l1 = l2 = True + with open(path_1, 'r') as f1, open(path_2, 'r') as f2: + line = 1 + while l1 and l2: + line += 1 + l1 = f1.readline() + l2 = f2.readline() + if l1 != l2: + print('---') + print(( + 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' + .format(os.path.basename(path_1), os.path.basename(path_2), line))) + print('(1): {0}'.format(repr(l1))) + print('(2): {0}'.format(repr(l2))) + print('---') + return False + + return True + +def validate_scripts_content(repo_path, temp_cwd): + errors = False + + if not compare_files( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'letsencrypt-auto')): + print('Root certbot-auto and letsencrypt-auto differ.') + errors = True + else: + shutil.copyfile( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')) + shutil.copy(os.path.normpath(os.path.join( + repo_path, + 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) + + # Compare file against current version in the target branch + branch = os.environ.get('TRAVIS_BRANCH', 'master') + url = ( + 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' + .format(branch)) + urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) + + if compare_files( + os.path.join(temp_cwd, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were unchanged') + else: + # Compare file against the latest released version + latest_version = subprocess.check_output( + [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) + subprocess.call( + [sys.executable, 'fetch.py', '--le-auto-script', + 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) + if compare_files( + os.path.join(temp_cwd, 'letsencrypt-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were updated to the latest version.') + else: + print('Root *-auto have unexpected changes.') + errors = True + + return errors + +def main(): + repo_path = find_repo_path() + temp_cwd = tempfile.mkdtemp() + errors = False + + try: + errors = validate_scripts_content(repo_path, temp_cwd) + + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'original-lea') + ) + subprocess.call([sys.executable, os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/build.py'))]) + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'build-lea') + ) + shutil.copyfile( + os.path.join(temp_cwd, 'original-lea'), + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) + ) + + if not compare_files( + os.path.join(temp_cwd, 'original-lea'), + os.path.join(temp_cwd, 'build-lea')): + print('Script letsencrypt-auto-source/letsencrypt-auto ' + 'doesn\'t match output of build.py.') + errors = True + else: + print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') + finally: + shutil.rmtree(temp_cwd) + + return errors + +if __name__ == '__main__': + if main(): + sys.exit(1) diff --git a/tests/modification-check.sh b/tests/modification-check.sh deleted file mode 100755 index 0145b0228..000000000 --- a/tests/modification-check.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -e - -temp_dir=`mktemp -d` -trap "rm -rf $temp_dir" EXIT - -# cd to repo root -cd $(dirname $(dirname $(readlink -f $0))) -FLAG=false - -if ! cmp -s certbot-auto letsencrypt-auto; then - echo "Root certbot-auto and letsencrypt-auto differ." - FLAG=true -else - cp certbot-auto "$temp_dir/local-auto" - cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" - cd $temp_dir - - # Compare file against current version in the target branch - BRANCH=${TRAVIS_BRANCH:-master} - URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" - curl -sS $URL > certbot-auto - if cmp -s certbot-auto local-auto; then - echo "Root *-auto were unchanged." - else - # Compare file against the latest released version - python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" - if cmp -s letsencrypt-auto local-auto; then - echo "Root *-auto were updated to the latest version." - else - echo "Root *-auto have unexpected changes." - FLAG=true - fi - fi - cd ~- -fi - -# Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py - -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea -python letsencrypt-auto-source/build.py -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/build-lea -cp ${temp_dir}/original-lea letsencrypt-auto-source/letsencrypt-auto - -cd $temp_dir - -if ! cmp -s original-lea build-lea; then - echo "letsencrypt-auto-source/letsencrypt-auto doesn't match output of \ -build.py." - FLAG=true -else - echo "letsencrypt-auto-source/letsencrypt-auto matches output of \ -build.py." -fi - -rm -rf $temp_dir - -if $FLAG ; then - exit 1 -fi diff --git a/tools/_venv_common.py b/tools/_venv_common.py new file mode 100755 index 000000000..0c24664b3 --- /dev/null +++ b/tools/_venv_common.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import shutil +import glob +import time +import subprocess +import sys + +def subprocess_with_print(command): + print(command) + subprocess.call(command, shell=True) + +def get_venv_python(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_linux): + return python_linux + if os.path.isfile(python_windows): + return python_windows + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + +def main(venv_name, venv_args, args): + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + subprocess_with_print(' '.join([ + sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', + venv_name, venv_args])) + + python_executable = get_venv_python(venv_name) + + subprocess_with_print(' '.join([ + python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) + command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command.extend(args) + subprocess_with_print(' '.join(command)) + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + +if __name__ == '__main__': + main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh deleted file mode 100755 index 0f0ff7e28..000000000 --- a/tools/_venv_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -xe - -VENV_NAME=${VENV_NAME:-venv} - -# .egg-info directories tend to cause bizarre problems (e.g. `pip -e -# .` might unexpectedly install letshelp-certbot only, in case -# `python letshelp-certbot/setup.py build` has been called -# earlier) -rm -rf *.egg-info - -# virtualenv setup is NOT idempotent: shutil.Error: -# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and -# `venv/bin/python2` are the same file -mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true -virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS -. ./$VENV_NAME/bin/activate - -# Use pipstrap to update Python packaging tools to only update to a well tested -# version and to work around https://github.com/pypa/pip/issues/4817 on older -# systems. -python letsencrypt-auto-source/pieces/pipstrap.py -./tools/pip_install.sh "$@" - -set +x -echo "Please run the following command to activate developer environment:" -echo "source $VENV_NAME/bin/activate" diff --git a/tools/install_and_test.py b/tools/install_and_test.py new file mode 100755 index 000000000..b16181aa5 --- /dev/null +++ b/tools/install_and_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# pip installs the requested packages in editable mode and runs unit tests on +# them. Each package is installed and tested in the order they are provided +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.py for more information on the versions pinned +# to. +from __future__ import print_function + +import os +import sys +import tempfile +import shutil +import subprocess +import re + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + if os.environ.get('CERTBOT_NO_PIN') == '1': + command = [sys.executable, '-m', 'pip', '-q', '-e'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + + new_args = [] + for arg in args: + if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested.' + .format(arg))) + else: + new_args.append(arg) + + for requirement in new_args: + current_command = command[:] + current_command.append(requirement) + call_with_print(' '.join(current_command)) + pkg = re.sub(r'\[\w+\]', '', requirement) + + if pkg == '.': + pkg = 'certbot' + + temp_cwd = tempfile.mkdtemp() + try: + call_with_print(' '.join([ + sys.executable, '-m', 'pytest', '--numprocesses', 'auto', + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + finally: + shutil.rmtree(temp_cwd) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh deleted file mode 100755 index 819f683aa..000000000 --- a/tools/install_and_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -e -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. - -if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -q -e" -else - pip_install="$(dirname $0)/pip_install_editable.sh" -fi - -temp_cwd=$(mktemp -d) -trap "rm -rf $temp_cwd" EXIT - -set -x -for requirement in "$@" ; do - $pip_install $requirement - pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] - pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names - if [ $pkg = "." ]; then - pkg="certbot" - fi - cd "$temp_cwd" - pytest --numprocesses auto --quiet --pyargs $pkg - cd - -done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index c8fb95351..ad44a55d0 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys - def read_file(file_path): """Reads in a Python requirements file. @@ -32,17 +31,17 @@ def read_file(file_path): return d -def print_requirements(requirements): - """Prints requirements to stdout. +def output_requirements(requirements): + """Prepare print requirements to stdout. :param dict requirements: mapping from a project to its pinned version """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) + return '\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items())) -def merge_requirements_files(*files): +def main(*files): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier @@ -54,8 +53,9 @@ def merge_requirements_files(*files): d = {} for f in files: d.update(read_file(f)) - print_requirements(d) + return output_requirements(d) if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) + merged_requirements = main(*sys.argv[1:]) + print(merged_requirements) diff --git a/tools/pip_install.py b/tools/pip_install.py new file mode 100755 index 000000000..d09997bf5 --- /dev/null +++ b/tools/pip_install.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. + +from __future__ import print_function, absolute_import + +import subprocess +import os +import sys +import re +import shutil +import tempfile + +import merge_requirements as merge_module +import readlink + +def find_tools_path(): + return os.path.dirname(readlink.main(__file__)) + +def certbot_oldest_processing(tools_path, args, test_constraints): + if args[0] != '-e' or len(args) != 2: + raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' + 'with a single -e argument.') + # remove any extras such as [dev] + pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + # packages like acme don't have any local oldest requirements + if not os.path.isfile(requirements): + requirements = None + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + + return requirements + +def certbot_normal_processing(tools_path, test_constraints): + repo_path = os.path.dirname(tools_path) + certbot_requirements = os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + with open(certbot_requirements, 'r') as fd: + data = fd.readlines() + with open(test_constraints, 'w') as fd: + for line in data: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + fd.write('{0}{1}'.format(search.group(1), os.linesep)) + +def merge_requirements(tools_path, test_constraints, all_constraints): + merged_requirements = merge_module.main( + os.path.join(tools_path, 'dev_constraints.txt'), + test_constraints + ) + with open(all_constraints, 'w') as fd: + fd.write(merged_requirements) + +def call_with_print(command, cwd=None): + print(command) + subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + tools_path = find_tools_path() + working_dir = tempfile.mkdtemp() + try: + test_constraints = os.path.join(working_dir, 'test_constraints.txt') + all_constraints = os.path.join(working_dir, 'all_constraints.txt') + + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) + + merge_requirements(tools_path, test_constraints, all_constraints) + if requirements: + call_with_print(' '.join([ + sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, + '--requirement', requirements])) + + command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] + command.extend(args) + call_with_print(' '.join(command)) + finally: + shutil.rmtree(working_dir) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index 78e2afa17..000000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the -# top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If -# CERTBOT_OLDEST is set, this script must be run with `-e ` and -# no other arguments. - -# get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -all_constraints=$(mktemp) -test_constraints=$(mktemp) -trap "rm -f $all_constraints $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - if [ "$1" != "-e" -o "$#" -ne "2" ]; then - echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." - exit 1 - fi - pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] - requirements="$pkg_dir/local-oldest-requirements.txt" - # packages like acme don't have any local oldest requirements - if [ ! -f "$requirements" ]; then - unset requirements - fi - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi - -"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ - "$test_constraints" > "$all_constraints" - -set -x - -# install the requested packages using the pinned requirements as constraints -if [ -n "$requirements" ]; then - pip install -q --constraint "$all_constraints" --requirement "$requirements" -fi -pip install -q --constraint "$all_constraints" "$@" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py new file mode 100755 index 000000000..bdbdcadc5 --- /dev/null +++ b/tools/pip_install_editable.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# pip installs packages in editable mode using certbot-auto's requirements file +# as constraints + +from __future__ import absolute_import + +import sys + +import pip_install + +def main(args): + new_args = [] + for arg in args: + new_args.append('-e') + new_args.append(arg) + pip_install.main(new_args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh deleted file mode 100755 index 6130bf6e7..000000000 --- a/tools/pip_install_editable.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -# pip installs packages in editable mode using certbot-auto's requirements file -# as constraints - -args="" -for requirement in "$@" ; do - args="$args -e $requirement" -done - -"$(dirname $0)/pip_install.sh" $args diff --git a/tools/readlink.py b/tools/readlink.py index 02c74c48d..0199ce184 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -7,7 +7,12 @@ platforms. """ from __future__ import print_function + import os import sys -print(os.path.realpath(sys.argv[1])) +def main(link): + return os.path.realpath(link) + +if __name__ == '__main__': + print(main(sys.argv[1])) diff --git a/tools/venv.py b/tools/venv.py new file mode 100755 index 000000000..849c49119 --- /dev/null +++ b/tools/venv.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + command_python2_st_code = subprocess.call( + 'command -v python2', shell=True, stdout=fnull, stderr=fnull) + if not command_python2_st_code: + return '--python python2' + + command_python27_st_code = subprocess.call( + 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) + if not command_python27_st_code: + return '--python python2.7' + + raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) + +def main(): + if os.name == 'nt': + raise ValueError('Certbot for Windows is not supported on Python 2.x.') + + venv_args = get_venv_args() + + _venv_common.main('venv', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv.sh b/tools/venv.sh deleted file mode 100755 index 5692f9ebf..000000000 --- a/tools/venv.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -xe -# Developer virtualenv setup for Certbot client - -if command -v python2; then - export VENV_ARGS="--python python2" -elif command -v python2.7; then - export VENV_ARGS="--python python2.7" -else - echo "Couldn't find python2 or python2.7 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tools/venv3.py b/tools/venv3.py new file mode 100755 index 000000000..15db9495a --- /dev/null +++ b/tools/venv3.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + where_python3_st_code = subprocess.call( + 'where python3', shell=True, stdout=fnull, stderr=fnull) + command_python3_st_code = subprocess.call( + 'command -v python3', shell=True, stdout=fnull, stderr=fnull) + + if not where_python3_st_code or not command_python3_st_code: + return '--python python3' + + raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) + +def main(): + venv_args = get_venv_args() + + _venv_common.main('venv3', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + main() diff --git a/tools/venv3.sh b/tools/venv3.sh deleted file mode 100755 index 07512f370..000000000 --- a/tools/venv3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -xe -# Developer Python3 virtualenv setup for Certbot - -if command -v python3; then - export VENV_NAME="${VENV_NAME:-venv3}" - export VENV_ARGS="--python python3" -else - echo "Couldn't find python3 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tox-win.ini b/tox-win.ini deleted file mode 100644 index fe063c264..000000000 --- a/tox-win.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -skipsdist = True -envlist = py{34,35,36,37}-cover - -[testenv] -deps = -e acme[dev] - -e .[dev] -commands = pytest -n auto --pyargs acme - pytest -n auto --pyargs certbot - -[testenv:cover] -commands = pytest -n auto --cov acme --pyargs acme - pytest -n auto --cov certbot --cov-append --pyargs certbot diff --git a/tox.cover.py b/tox.cover.py new file mode 100755 index 000000000..8bbce2d09 --- /dev/null +++ b/tox.cover.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +import argparse +import subprocess +import os +import sys + +DEFAULT_PACKAGES = [ + 'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns', + 'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy', + 'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns', + 'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53', + 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] + +COVER_THRESHOLDS = { + 'certbot': 98, + 'acme': 100, + 'certbot_apache': 100, + 'certbot_dns_cloudflare': 98, + 'certbot_dns_cloudxns': 99, + 'certbot_dns_digitalocean': 98, + 'certbot_dns_dnsimple': 98, + 'certbot_dns_dnsmadeeasy': 99, + 'certbot_dns_gehirn': 97, + 'certbot_dns_google': 99, + 'certbot_dns_linode': 98, + 'certbot_dns_luadns': 98, + 'certbot_dns_nsone': 99, + 'certbot_dns_ovh': 97, + 'certbot_dns_rfc2136': 99, + 'certbot_dns_route53': 92, + 'certbot_dns_sakuracloud': 97, + 'certbot_nginx': 97, + 'certbot_postfix': 100, + 'letshelp_certbot': 100 +} + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def cover(package): + threshold = COVER_THRESHOLDS.get(package) + + if not threshold: + raise ValueError('Unrecognized package: {0}'.format(package)) + + pkg_dir = package.replace('_', '-') + + if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested/covered.' + .format(pkg_dir))) + return + + subprocess.call([ + sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', + '--numprocesses', 'auto', '--pyargs', package]) + subprocess.call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', + '{0}/*'.format(pkg_dir), '--show-missing']) + +def main(): + description = """ +This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order +to generate separate stats for each package. It should be removed once those +packages are moved to a separate repo. + +Option -e makes sure we fail fast and don't submit to codecov.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--packages', nargs='+') + + args = parser.parse_args() + + packages = args.packages or DEFAULT_PACKAGES + + # --cov-append is on, make sure stats are correct + try: + os.remove('.coverage') + except OSError: + pass + + for package in packages: + cover(package) + +if __name__ == '__main__': + main() diff --git a/tox.cover.sh b/tox.cover.sh deleted file mode 100755 index c68e757de..000000000 --- a/tox.cover.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -xe - -# USAGE: ./tox.cover.sh [package] -# -# This script is used by tox.ini (and thus Travis CI) in order to -# generate separate stats for each package. It should be removed once -# those packages are moved to separate repo. -# -# -e makes sure we fail fast and don't submit to codecov - -if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" -else - pkgs="$@" -fi - -cover () { - if [ "$1" = "certbot" ]; then - min=98 - elif [ "$1" = "acme" ]; then - min=100 - elif [ "$1" = "certbot_apache" ]; then - min=100 - elif [ "$1" = "certbot_dns_cloudflare" ]; then - min=98 - elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=99 - elif [ "$1" = "certbot_dns_digitalocean" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsimple" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=99 - elif [ "$1" = "certbot_dns_gehirn" ]; then - min=97 - elif [ "$1" = "certbot_dns_google" ]; then - min=99 - elif [ "$1" = "certbot_dns_linode" ]; then - min=98 - elif [ "$1" = "certbot_dns_luadns" ]; then - min=98 - elif [ "$1" = "certbot_dns_nsone" ]; then - min=99 - elif [ "$1" = "certbot_dns_ovh" ]; then - min=97 - elif [ "$1" = "certbot_dns_rfc2136" ]; then - min=99 - elif [ "$1" = "certbot_dns_route53" ]; then - min=92 - elif [ "$1" = "certbot_dns_sakuracloud" ]; then - min=97 - elif [ "$1" = "certbot_nginx" ]; then - min=97 - elif [ "$1" = "certbot_postfix" ]; then - min=100 - elif [ "$1" = "letshelp_certbot" ]; then - min=100 - else - echo "Unrecognized package: $1" - exit 1 - fi - - pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= --numprocesses "auto" --pyargs "$1" - coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing -} - -rm -f .coverage # --cov-append is on, make sure stats are correct -for pkg in $pkgs -do - cover $pkg -done diff --git a/tox.ini b/tox.ini index 9db06f78c..e38f1311e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,16 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},cover,lint +envlist = modification,py{34,35,36},py27-cover,lint [base] # pip installs the requested packages in editable mode -pip_install = {toxinidir}/tools/pip_install_editable.sh +pip_install = python {toxinidir}/tools/pip_install_editable.py # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. -install_and_test = {toxinidir}/tools/install_and_test.sh +install_and_test = python {toxinidir}/tools/install_and_test.py dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ @@ -38,7 +38,7 @@ all_packages = certbot-postfix \ letshelp-certbot install_packages = - {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} + python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} source_paths = acme/acme certbot @@ -64,7 +64,9 @@ source_paths = tests/lock_test.py [testenv] -passenv = TRAVIS +passenv = + TRAVIS + APPVEYOR commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -120,11 +122,17 @@ basepython = python2.7 commands = {[base]install_packages} -[testenv:cover] +[testenv:py27-cover] basepython = python2.7 commands = {[base]install_packages} - ./tox.cover.sh + python tox.cover.py + +[testenv:py37-cover] +basepython = python3.7 +commands = + {[base]install_packages} + python tox.cover.py [testenv:lint] basepython = python2.7 @@ -133,7 +141,7 @@ basepython = python2.7 # continue, but tox return code will reflect previous error commands = {[base]install_packages} - pylint --reports=n --rcfile=.pylintrc {[base]source_paths} + python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] basepython = python3 @@ -157,7 +165,7 @@ commands = # allow users to run the modification check by running `tox` [testenv:modification] commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py [testenv:apache_compat] commands = @@ -197,7 +205,7 @@ passenv = # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Trusty Docker image. commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = From 7352727a6507ee153dff485b5423940497186663 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 8 Nov 2018 17:35:07 +0100 Subject: [PATCH 08/11] [URGENT] Fix the CI system (#6485) It is about the exit codes that are returned from the various scripts in tools during tox execution. Indeed, tox relies on the non-zero exit code from a given script to know that something failed during the execution. Previously, theses scripts were in bash, and a bash script returns an exit code that is the higher code returned from any of the command executed by the script. So if any command return a non-zero (in particular pylint or pytest), then the script return also non-zero. Now that these scripts are converted into python, pylint and pytest are executed via subprocess, that returns the exit code as variables. But if theses codes are not handled explicitly, the python script itself will return zero if no python exception occured. As a consequence currently, Certbot CI system is unable to detect any test error or lint error, because there is no exception in this case, only exit codes from the binaries executed. This PR fixes that, by handling correctly the exit code from the most critical scripts, install_and_test.py and tox.cover.py, but also all the scripts that I converted into Python and that could be executed in the context of a shell (via tox or directly for instance). --- tools/_venv_common.py | 20 +++++++++++++------- tools/install_and_test.py | 14 +++++++++----- tools/pip_install.py | 15 ++++++++++----- tools/pip_install_editable.py | 5 +++-- tools/venv.py | 5 +++-- tools/venv3.py | 5 +++-- 6 files changed, 41 insertions(+), 23 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 0c24664b3..cce88f826 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -11,7 +11,7 @@ import sys def subprocess_with_print(command): print(command) - subprocess.call(command, shell=True) + return subprocess.call(command, shell=True) def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') @@ -35,17 +35,19 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ + exit_code = 0 + + exit_code = subprocess_with_print(' '.join([ sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + venv_name, venv_args])) or exit_code python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ + exit_code = subprocess_with_print(' '.join([ python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] + command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code command.extend(args) - subprocess_with_print(' '.join(command)) + exit_code = subprocess_with_print(' '.join(command)) or exit_code if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -63,5 +65,9 @@ def main(venv_name, venv_args, args): else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + return exit_code + if __name__ == '__main__': - main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:]) + sys.exit(main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:])) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b16181aa5..149ffc776 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -19,7 +19,7 @@ SKIP_PROJECTS_ON_WINDOWS = [ def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': @@ -37,10 +37,12 @@ def main(args): else: new_args.append(arg) + exit_code = 0 + for requirement in new_args: current_command = command[:] current_command.append(requirement) - call_with_print(' '.join(current_command)) + exit_code = call_with_print(' '.join(current_command)) or exit_code pkg = re.sub(r'\[\w+\]', '', requirement) if pkg == '.': @@ -48,11 +50,13 @@ def main(args): temp_cwd = tempfile.mkdtemp() try: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pytest', '--numprocesses', 'auto', - '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code finally: shutil.rmtree(temp_cwd) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install.py b/tools/pip_install.py index d09997bf5..273ce5ec2 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,11 +59,14 @@ def merge_requirements(tools_path, test_constraints, all_constraints): def call_with_print(command, cwd=None): print(command) - subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + + exit_code = 0 + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -76,15 +79,17 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ + exit_code = call_with_print(' '.join([ sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + '--requirement', requirements])) or exit_code command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] command.extend(args) - call_with_print(' '.join(command)) + exit_code = call_with_print(' '.join(command)) or exit_code finally: shutil.rmtree(working_dir) + return exit_code + if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index bdbdcadc5..35cc2264d 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -13,7 +13,8 @@ def main(args): for arg in args: new_args.append('-e') new_args.append(arg) - pip_install.main(new_args) + + return pip_install.main(new_args) if __name__ == '__main__': - main(sys.argv[1:]) + sys.exit(main(sys.argv[1:])) diff --git a/tools/venv.py b/tools/venv.py index 849c49119..2cc43251d 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -52,7 +53,7 @@ def main(): venv_args = get_venv_args() - _venv_common.main('venv', venv_args, REQUIREMENTS) + return _venv_common.main('venv', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) diff --git a/tools/venv3.py b/tools/venv3.py index 15db9495a..1bacc9c9a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import subprocess +import sys import _venv_common @@ -47,7 +48,7 @@ def get_venv_args(): def main(): venv_args = get_venv_args() - _venv_common.main('venv3', venv_args, REQUIREMENTS) + return _venv_common.main('venv3', venv_args, REQUIREMENTS) if __name__ == '__main__': - main() + sys.exit(main()) From ad885afdb8f51568dd247f1df5e9e1caf99ade12 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Sat, 10 Nov 2018 01:17:17 +0100 Subject: [PATCH 09/11] Correct venv3 detection on windows (#6490) A little typo in the _venv_common.py block the script to finish correctly once the virtual environment has been setup on Windows. This PR fixes that. --- tools/_venv_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index cce88f826..b180518f9 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -55,7 +55,7 @@ def main(venv_name, venv_args, args): print('Please run the following command to activate developer environment:') print('source {0}/bin/activate'.format(venv_name)) print('-------------------------------------------------------------------') - elif os.path.isdir(os.path.join(venv_args, 'Scripts')): + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') From b3d2ac5161b44d2f09c211048056248be224100a Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 14 Nov 2018 22:57:40 +0100 Subject: [PATCH 10/11] Fail-fast in test/cover/lint scripts (#6487) After #6485 and #6435, it appears that there is no good reason to not fail fast when test, cover or linting scripts are executed. This PR ensures to fail fast by invoking commands throught subprocess.check_call instead of subprocess.call, and by removing the handling of non-zero exit code at the end of theses scripts. As now coverage on Windows is executed with thresholds, I added specific thresholds for this platform. Because some portions of code that are done for Unix platform will not be executed on Windows. Note that coverage reports from Travis and AppVeyor are accumulated on Codecov. So if a file is covered up to 50 % on Linux, and all other parts are covered on Windows, then coverage is 100 % for Codecov. Note: that PR also fixes the ability of coverage tests to fail if thresholds are exceeded. * Use check_call to fail fast in all scripts related to tests/lint/coverage/deploy * Make specific coverage threshold for windows --- tests/modification-check.py | 4 +-- tools/_venv_common.py | 24 +++++++---------- tools/install_and_test.py | 14 ++++------ tools/pip_install.py | 14 ++++------ tools/pip_install_editable.py | 4 +-- tools/venv.py | 4 +-- tools/venv3.py | 4 +-- tox.cover.py | 49 ++++++++++++++++------------------- 8 files changed, 51 insertions(+), 66 deletions(-) diff --git a/tests/modification-check.py b/tests/modification-check.py index e00994b04..8abc0fbfe 100755 --- a/tests/modification-check.py +++ b/tests/modification-check.py @@ -70,7 +70,7 @@ def validate_scripts_content(repo_path, temp_cwd): # Compare file against the latest released version latest_version = subprocess.check_output( [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) - subprocess.call( + subprocess.check_call( [sys.executable, 'fetch.py', '--le-auto-script', 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) if compare_files( @@ -95,7 +95,7 @@ def main(): os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), os.path.join(temp_cwd, 'original-lea') ) - subprocess.call([sys.executable, os.path.normpath(os.path.join( + subprocess.check_call([sys.executable, os.path.normpath(os.path.join( repo_path, 'letsencrypt-auto-source/build.py'))]) shutil.copyfile( os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), diff --git a/tools/_venv_common.py b/tools/_venv_common.py index b180518f9..6134bd29d 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -11,13 +11,13 @@ import sys def subprocess_with_print(command): print(command) - return subprocess.call(command, shell=True) + subprocess.check_call(command, shell=True) def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') - python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_linux): return python_linux + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): return python_windows @@ -35,19 +35,17 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - exit_code = 0 - - exit_code = subprocess_with_print(' '.join([ + subprocess_with_print(' '.join([ sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) or exit_code + venv_name, venv_args])) python_executable = get_venv_python(venv_name) - exit_code = subprocess_with_print(' '.join([ + subprocess_with_print(' '.join([ python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code + command = [python_executable, os.path.normpath('./tools/pip_install.py')] command.extend(args) - exit_code = subprocess_with_print(' '.join(command)) or exit_code + subprocess_with_print(' '.join(command)) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -65,9 +63,7 @@ def main(venv_name, venv_args, args): else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) - return exit_code - if __name__ == '__main__': - sys.exit(main(os.environ.get('VENV_NAME', 'venv'), - os.environ.get('VENV_ARGS', ''), - sys.argv[1:])) + main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:]) diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 149ffc776..2842ec069 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -19,7 +19,7 @@ SKIP_PROJECTS_ON_WINDOWS = [ def call_with_print(command, cwd=None): print(command) - return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': @@ -37,12 +37,10 @@ def main(args): else: new_args.append(arg) - exit_code = 0 - for requirement in new_args: current_command = command[:] current_command.append(requirement) - exit_code = call_with_print(' '.join(current_command)) or exit_code + call_with_print(' '.join(current_command)) pkg = re.sub(r'\[\w+\]', '', requirement) if pkg == '.': @@ -50,13 +48,11 @@ def main(args): temp_cwd = tempfile.mkdtemp() try: - exit_code = call_with_print(' '.join([ + call_with_print(' '.join([ sys.executable, '-m', 'pytest', '--numprocesses', 'auto', - '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) finally: shutil.rmtree(temp_cwd) - return exit_code - if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/pip_install.py b/tools/pip_install.py index 273ce5ec2..2c4a47c21 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -59,14 +59,12 @@ def merge_requirements(tools_path, test_constraints, all_constraints): def call_with_print(command, cwd=None): print(command) - return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() - exit_code = 0 - try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') @@ -79,17 +77,15 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - exit_code = call_with_print(' '.join([ + call_with_print(' '.join([ sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) or exit_code + '--requirement', requirements])) command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] command.extend(args) - exit_code = call_with_print(' '.join(command)) or exit_code + call_with_print(' '.join(command)) finally: shutil.rmtree(working_dir) - return exit_code - if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index 35cc2264d..8eaf3a9fa 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -14,7 +14,7 @@ def main(args): new_args.append('-e') new_args.append(arg) - return pip_install.main(new_args) + pip_install.main(new_args) if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) + main(sys.argv[1:]) diff --git a/tools/venv.py b/tools/venv.py index 2cc43251d..8b6f92a56 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -53,7 +53,7 @@ def main(): venv_args = get_venv_args() - return _venv_common.main('venv', venv_args, REQUIREMENTS) + _venv_common.main('venv', venv_args, REQUIREMENTS) if __name__ == '__main__': - sys.exit(main()) + main() diff --git a/tools/venv3.py b/tools/venv3.py index 1bacc9c9a..9710806c5 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -48,7 +48,7 @@ def get_venv_args(): def main(): venv_args = get_venv_args() - return _venv_common.main('venv3', venv_args, REQUIREMENTS) + _venv_common.main('venv3', venv_args, REQUIREMENTS) if __name__ == '__main__': - sys.exit(main()) + main() diff --git a/tox.cover.py b/tox.cover.py index 8bbce2d09..ad6bb1a39 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -12,36 +12,33 @@ DEFAULT_PACKAGES = [ 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] COVER_THRESHOLDS = { - 'certbot': 98, - 'acme': 100, - 'certbot_apache': 100, - 'certbot_dns_cloudflare': 98, - 'certbot_dns_cloudxns': 99, - 'certbot_dns_digitalocean': 98, - 'certbot_dns_dnsimple': 98, - 'certbot_dns_dnsmadeeasy': 99, - 'certbot_dns_gehirn': 97, - 'certbot_dns_google': 99, - 'certbot_dns_linode': 98, - 'certbot_dns_luadns': 98, - 'certbot_dns_nsone': 99, - 'certbot_dns_ovh': 97, - 'certbot_dns_rfc2136': 99, - 'certbot_dns_route53': 92, - 'certbot_dns_sakuracloud': 97, - 'certbot_nginx': 97, - 'certbot_postfix': 100, - 'letshelp_certbot': 100 + 'certbot': {'linux': 98, 'windows': 94}, + 'acme': {'linux': 100, 'windows': 99}, + 'certbot_apache': {'linux': 100, 'windows': 100}, + 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, + 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, + 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsimple': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99}, + 'certbot_dns_gehirn': {'linux': 97, 'windows': 97}, + 'certbot_dns_google': {'linux': 99, 'windows': 99}, + 'certbot_dns_linode': {'linux': 98, 'windows': 98}, + 'certbot_dns_luadns': {'linux': 98, 'windows': 98}, + 'certbot_dns_nsone': {'linux': 99, 'windows': 99}, + 'certbot_dns_ovh': {'linux': 97, 'windows': 97}, + 'certbot_dns_rfc2136': {'linux': 99, 'windows': 99}, + 'certbot_dns_route53': {'linux': 92, 'windows': 92}, + 'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97}, + 'certbot_nginx': {'linux': 97, 'windows': 97}, + 'certbot_postfix': {'linux': 100, 'windows': 100}, + 'letshelp_certbot': {'linux': 100, 'windows': 100} } SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] def cover(package): - threshold = COVER_THRESHOLDS.get(package) - - if not threshold: - raise ValueError('Unrecognized package: {0}'.format(package)) + threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] pkg_dir = package.replace('_', '-') @@ -51,10 +48,10 @@ def cover(package): .format(pkg_dir))) return - subprocess.call([ + subprocess.check_call([ sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', '--numprocesses', 'auto', '--pyargs', package]) - subprocess.call([ + subprocess.check_call([ sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', '{0}/*'.format(pkg_dir), '--show-missing']) From 5073090a20fa59fae45b4d90bfb41635bc181911 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 16 Nov 2018 00:17:36 +0100 Subject: [PATCH 11/11] Update tools/venv3.py to support py launcher on Windows (#6493) Following some inconsistencies occurred during by developments, and in the light of #6508, it decided to wrote a PR that will take fully advantage of the conversion from bash to python to the development setup tools. This PR adresses several issues when trying to use the development setup tools (`tools/venv.py` and `tools/venv3.py`: * on Windows, `python` executable is not always in PATH (default behavior) * even if the option is checked, the `python` executable is not associated to the usually symlink `python3` on Windows * on Windows again, really powerful introspection of the available Python environments can be done with `py`, the Windows Python launcher * in general for all systems, `tools/venv.py` and `tools/venv3.py` ensures that the respective Python major version will be used to setup the virtual environment if available. * finally, the best and first candidate to test should be the Python executable used to launch the `tools/venv*.py` script. It was not relevant before because it was shell scripts, but do it is. The logic is shared in `_venv_common.py`, and will be called appropriately for both scripts. In priority decreasing order, python executable will be search and tested: * from the current Python executable, as exposed by `sys.executable` * from any python or pythonX (X as a python version like 2, 3 or 2.7 or 3.4) executable available in PATH * from the Windows Python launched `py` if available Individual changes were: * Update tools/venv3.py to support py launcher on Windows * Fix typo in help message * More explicit calls with space protection * Complete refactoring to take advantage of the python runtime, and control of the compatible version to use. --- tools/_venv_common.py | 104 ++++++++++++++++++++++++++++++++++++++---- tools/pip_install.py | 17 ++++--- tools/venv.py | 22 +-------- tools/venv3.py | 22 +-------- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 6134bd29d..c44d05bf7 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -8,11 +8,94 @@ import glob import time import subprocess import sys +import re + +VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') + + +class PythonExecutableNotFoundError(Exception): + pass + + +def find_python_executable(python_major): + # type: (int) -> str + """ + Find the relevant python executable that is of the given python major version. + Will test, in decreasing priority order: + * the current Python interpreter + * 'pythonX' executable in PATH (with X the given major version) if available + * 'python' executable in PATH if available + * Windows Python launcher 'py' executable in PATH if available + Incompatible python versions for Certbot will be evicted (eg. Python < 3.5 on Windows) + :param int python_major: the Python major version to target (2 or 3) + :rtype: str + :return: the relevant python executable path + :raise RuntimeError: if no relevant python executable path could be found + """ + python_executable_path = None + + # First try, current python executable + if _check_version('{0}.{1}.{2}'.format( + sys.version_info[0], sys.version_info[1], sys.version_info[2]), python_major): + return sys.executable + + # Second try, with python executables in path + versions_to_test = ['2.7', '2', ''] if python_major == 2 else ['3', ''] + for one_version in versions_to_test: + try: + one_python = 'python{0}'.format(one_version) + output = subprocess.check_output([one_python, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output.strip().split()[1], python_major): + return subprocess.check_output([one_python, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + # Last try, with Windows Python launcher + try: + env_arg = '-{0}'.format(python_major) + output_version = subprocess.check_output(['py', env_arg, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output_version.strip().split()[1], python_major): + return subprocess.check_output(['py', env_arg, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + if not python_executable_path: + raise RuntimeError('Error, no compatible Python {0} executable for Certbot could be found.' + .format(python_major)) + + +def _check_version(version_str, major_version): + search = VERSION_PATTERN.search(version_str) + + if not search: + return False + + version = (int(search.group(1)), int(search.group(2))) + + minimal_version_supported = (2, 7) + if major_version == 3 and os.name == 'nt': + minimal_version_supported = (3, 5) + elif major_version == 3: + minimal_version_supported = (3, 4) + + if version >= minimal_version_supported: + return True + + print('Incompatible python version for Certbot found: {0}'.format(version_str)) + return False + def subprocess_with_print(command): print(command) subprocess.check_call(command, shell=True) + def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): @@ -25,6 +108,7 @@ def get_venv_python(venv_path): 'Error, could not find python executable in venv path {0}: is it a valid venv ?' .format(venv_path))) + def main(venv_name, venv_args, args): for path in glob.glob('*.egg-info'): if os.path.isdir(path): @@ -35,17 +119,18 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ - sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' + .format(sys.executable, venv_name, venv_args)) python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ - python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] - command.extend(args) - subprocess_with_print(' '.join(command)) + subprocess_with_print('"{0}" {1}'.format( + python_executable, + os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) + subprocess_with_print('"{0}" {1} {2}'.format( + python_executable, + os.path.normpath('./tools/pip_install.py'), + ' '.join(args))) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -57,12 +142,13 @@ def main(venv_name, venv_args, args): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') - print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) print('---------------------------------------------------------------------------') else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + if __name__ == '__main__': main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), diff --git a/tools/pip_install.py b/tools/pip_install.py index 2c4a47c21..8878674c9 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -20,9 +20,11 @@ import tempfile import merge_requirements as merge_module import readlink + def find_tools_path(): return os.path.dirname(readlink.main(__file__)) + def certbot_oldest_processing(tools_path, args, test_constraints): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' @@ -37,6 +39,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): return requirements + def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( @@ -49,6 +52,7 @@ def certbot_normal_processing(tools_path, test_constraints): if search: fd.write('{0}{1}'.format(search.group(1), os.linesep)) + def merge_requirements(tools_path, test_constraints, all_constraints): merged_requirements = merge_module.main( os.path.join(tools_path, 'dev_constraints.txt'), @@ -57,10 +61,12 @@ def merge_requirements(tools_path, test_constraints, all_constraints): with open(all_constraints, 'w') as fd: fd.write(merged_requirements) + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() @@ -77,15 +83,14 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ - sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + call_with_print('"{0}" -m pip install -q --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) - command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] - command.extend(args) - call_with_print(' '.join(command)) + call_with_print('"{0}" -m pip install -q --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: shutil.rmtree(working_dir) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/tools/venv.py b/tools/venv.py index 8b6f92a56..93b012e76 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -1,11 +1,6 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - import os -import subprocess -import sys import _venv_common @@ -33,27 +28,14 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - command_python2_st_code = subprocess.call( - 'command -v python2', shell=True, stdout=fnull, stderr=fnull) - if not command_python2_st_code: - return '--python python2' - - command_python27_st_code = subprocess.call( - 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) - if not command_python27_st_code: - return '--python python2.7' - - raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) def main(): if os.name == 'nt': raise ValueError('Certbot for Windows is not supported on Python 2.x.') - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(2)) _venv_common.main('venv', venv_args, REQUIREMENTS) + if __name__ == '__main__': main() diff --git a/tools/venv3.py b/tools/venv3.py index 9710806c5..c2374ba5a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -1,12 +1,5 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - -import os -import subprocess -import sys - import _venv_common REQUIREMENTS = [ @@ -33,22 +26,11 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - where_python3_st_code = subprocess.call( - 'where python3', shell=True, stdout=fnull, stderr=fnull) - command_python3_st_code = subprocess.call( - 'command -v python3', shell=True, stdout=fnull, stderr=fnull) - - if not where_python3_st_code or not command_python3_st_code: - return '--python python3' - - raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) def main(): - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(3)) _venv_common.main('venv3', venv_args, REQUIREMENTS) + if __name__ == '__main__': main()