diff --git a/.travis.yml b/.travis.yml index acdf365bd..2b8eafc13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: sudo: required services: docker - python: "2.7" - env: TOXENV=cover FYI="this also tests py27" + env: TOXENV=py27-cover FYI="py27 tests + code coverage" - sudo: required env: TOXENV=nginx_compat services: docker @@ -95,7 +95,7 @@ script: - travis_retry tox - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)' -after_success: '[ "$TOXENV" == "cover" ] && codecov' +after_success: '[ "$TOXENV" == "py27-cover" ] && codecov' notifications: email: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 71177b0d6..3189f7686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,30 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). -## 0.28.0 - master +## 0.29.0 - master + +### Added + +* + +### Changed + +* + +### Fixed + +* Update code and dependencies to clean up Resource and Deprecation Warnings. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/62?closed=1 + +## 0.28.0 - 2018-11-7 ### Added @@ -17,6 +40,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Warn when using deprecated acme.challenges.TLSSNI01 * Log warning about TLS-SNI deprecation in Certbot * Stop preferring TLS-SNI in the Apache, Nginx, and standalone plugins +* OVH DNS plugin now relies on Lexicon>=2.7.14 to support HTTP proxies +* Default time the Linode plugin waits for DNS changes to propogate is now 1200 seconds. ### Fixed @@ -26,7 +51,30 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Stop caching the results of ipv6_info in http01.py * Test fix for Route53 plugin to prevent boto3 making outgoing connections. * The grammar used by Augeas parser in Apache plugin was updated to fix various parsing errors. -* Update code and dependencies to clean up Resource and Deprecation Warnings. +* The CloudXNS, DNSimple, DNS Made Easy, Gehirn, Linode, LuaDNS, NS1, OVH, and + Sakura Cloud DNS plugins are now compatible with Lexicon 3.0+. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-apache +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-route53 +* certbot-dns-sakuracloud +* certbot-nginx + +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/milestone/59?closed=1 ## 0.27.1 - 2018-09-06 diff --git a/Dockerfile-dev b/Dockerfile-dev index 9e35ebec8..1ab56e081 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -16,6 +16,6 @@ RUN apt-get update && \ /tmp/* \ /var/tmp/* -RUN VENV_NAME="../venv" tools/venv.sh +RUN VENV_NAME="../venv" python tools/venv.py ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/acme/setup.py b/acme/setup.py index 85492d9a3..ad70c2947 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,7 +3,7 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/appveyor.yml b/appveyor.yml index 796070081..725ecfbff 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,17 @@ -image: - # => Windows Server 2012 R2 - - Visual Studio 2015 - # => Windows Server 2016 - - Visual Studio 2017 +environment: + matrix: + - FYI: Python 3.4 on Windows Server 2012 R2 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + - FYI: Python 3.4 on Windows Server 2016 + TOXENV: py34 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.5 on Windows Server 2016 + TOXENV: py35 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - FYI: Python 3.7 on Windows Server 2016 + code coverage + TOXENV: py37-cover + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 branches: only: @@ -14,6 +23,7 @@ install: # Use Python 3.7 by default - "SET PATH=C:\\Python37;C:\\Python37\\Scripts;%PATH%" # Check env + - "echo %APPVEYOR_BUILD_WORKER_IMAGE%" - "python --version" # Upgrade pip to avoid warnings - "python -m pip install --upgrade pip" @@ -23,7 +33,8 @@ install: build: off test_script: - - tox -c tox-win.ini -e py34,py35,py36,py37-cover + # Test env is set by TOXENV env variable + - tox on_success: - - codecov + - if exist .coverage codecov diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index aa5908f9e..e6f6f1e23 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 076c45e39..fe87317a7 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index 803b4a1b9..cbbdf7193 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -14,7 +14,7 @@ RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only # the above is not likely to change, so by putting it further up the # Dockerfile we make sure we cache as much as possible -COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.sh tox.ini .pylintrc /opt/certbot/src/ +COPY setup.py README.rst CHANGELOG.md MANIFEST.in linter_plugin.py tox.cover.py tox.ini .pylintrc /opt/certbot/src/ # all above files are necessary for setup.py, however, package source # code directory has to be copied separately to a subdirectory... @@ -35,7 +35,8 @@ RUN virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH -RUN /opt/certbot/src/tools/pip_install_editable.sh \ +RUN /opt/certbot/venv/bin/python \ + /opt/certbot/src/tools/pip_install_editable.py \ /opt/certbot/src/acme \ /opt/certbot/src \ /opt/certbot/src/certbot-apache \ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 8dac3e047..bfbbe0625 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index b823cf98f..d615fa999 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 3daf933cb..f9880270a 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index feb2a0d58..a9d46d128 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index b6efcf360..ac7bb1090 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index c268eaa8f..ab944d40d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index fc147f85c..94ca74761 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 86d36bcb3..aa1afdc93 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 1d196403f..4c2571f96 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index a5c06d90e..d0735d1ec 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 474093a5b..aebff5304 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 1f3acbf62..68ede8006 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -4,14 +4,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.21.1', 'certbot>=0.21.1', - 'dns-lexicon>=2.7.3', # Correct OVH integration tests + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index c009ef032..914bfb6e6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 2bae0c3d0..6318cbb81 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 9f8bfbbdb..5af4c8a00 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 3c8a66ee5..0908c4c52 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.28.0.dev0' +version = '0.29.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-postfix/README.rst b/certbot-postfix/README.rst index e6367e365..1ae9cb980 100644 --- a/certbot-postfix/README.rst +++ b/certbot-postfix/README.rst @@ -7,7 +7,7 @@ feature requests for this plugin. To install this plugin, in the root of this repo, run:: - ./tools/venv.sh + python tools/venv.py source venv/bin/activate You can use this installer with any `authenticator plugin diff --git a/certbot/__init__.py b/certbot/__init__.py index ab23926c9..f6b7defbd 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.28.0.dev0' +__version__ = '0.29.0.dev0' diff --git a/certbot/compat.py b/certbot/compat.py index e3e1bc4e1..d42febe81 100644 --- a/certbot/compat.py +++ b/certbot/compat.py @@ -65,6 +65,30 @@ def os_geteuid(): # Windows specific return 0 +def os_rename(src, dst): + """ + Rename a file to a destination path and handles situations where the destination exists. + + :param str src: The current file path. + :param str dst: The new file path. + """ + try: + os.rename(src, dst) + except OSError as err: + # Windows specific, renaming a file on an existing path is not possible. + # On Python 3, the best fallback with atomic capabilities we have is os.replace. + if err.errno != errno.EEXIST: + # Every other error is a legitimate exception. + raise + if not hasattr(os, 'replace'): # pragma: no cover + # We should never go on this line. Either we are on Linux and os.rename has succeeded, + # either we are on Windows, and only Python >= 3.4 is supported where os.replace is + # available. + raise RuntimeError('Error: tried to run os_rename on Python < 3.3. ' + 'Certbot supports only Python 3.4 >= on Windows.') + getattr(os, 'replace')(src, dst) + + def readline_with_timeout(timeout, prompt): """ Read user input to return the first line entered, or raise after specified timeout. diff --git a/certbot/reverter.py b/certbot/reverter.py index 5d56615fd..919037358 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -576,7 +576,7 @@ class Reverter(object): timestamp = self._checkpoint_timestamp() final_dir = os.path.join(self.config.backup_dir, timestamp) try: - os.rename(self.config.in_progress_dir, final_dir) + compat.os_rename(self.config.in_progress_dir, final_dir) return except OSError: logger.warning("Extreme, unexpected race condition, retrying (%s)", timestamp) diff --git a/certbot/storage.py b/certbot/storage.py index 2a7ccee02..664ff172a 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -14,6 +14,7 @@ import six import certbot from certbot import cli +from certbot import compat from certbot import constants from certbot import crypto_util from certbot import errors @@ -188,7 +189,7 @@ def update_configuration(lineagename, archive_dir, target, cli_config): # Save only the config items that are relevant to renewal values = relevant_values(vars(cli_config.namespace)) write_renewal_config(config_filename, temp_filename, archive_dir, target, values) - os.rename(temp_filename, config_filename) + compat.os_rename(temp_filename, config_filename) return configobj.ConfigObj(config_filename) diff --git a/certbot/tests/compat_test.py b/certbot/tests/compat_test.py new file mode 100644 index 000000000..552aa5645 --- /dev/null +++ b/certbot/tests/compat_test.py @@ -0,0 +1,21 @@ +"""Tests for certbot.compat.""" +import os + +from certbot import compat +import certbot.tests.util as test_util + +class OsReplaceTest(test_util.TempDirTestCase): + """Test to ensure consistent behavior of os_rename method""" + + def test_os_rename_to_existing_file(self): + """Ensure that os_rename will effectively rename src into dst for all platforms.""" + src = os.path.join(self.tempdir, 'src') + dst = os.path.join(self.tempdir, 'dst') + open(src, 'w').close() + open(dst, 'w').close() + + # On Windows, a direct call to os.rename will fail because dst already exists. + compat.os_rename(src, dst) + + self.assertFalse(os.path.exists(src)) + self.assertTrue(os.path.exists(dst)) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index 999c6225c..d04e3c641 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -356,7 +356,7 @@ class TestFullCheckpointsReverter(test_util.ConfigTestCase): self.assertRaises( errors.ReverterError, self.reverter.finalize_checkpoint, "Title") - @mock.patch("certbot.reverter.os.rename") + @mock.patch("certbot.reverter.compat.os_rename") def test_finalize_checkpoint_no_rename_directory(self, mock_rename): self.reverter.add_to_checkpoint(self.sets[0], "perm save") diff --git a/certbot/tests/util.py b/certbot/tests/util.py index 822597dd4..8c5db2c2f 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -328,15 +328,16 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): """Execute after test""" - # Then we have various files which are not correctly closed at the time of tearDown. - # On Windows, it is visible for the same reasons as above. + # On Windows we have various files which are not correctly closed at the time of tearDown. # For know, we log them until a proper file close handling is written. + # Useful for development only, so no warning when we are on a CI process. def onerror_handler(_, path, excinfo): """On error handler""" - message = ('Following error occurred when deleting the tempdir {0}' - ' for path {1} during tearDown process: {2}' - .format(self.tempdir, path, str(excinfo))) - warnings.warn(message) + if not os.environ.get('APPVEYOR'): # pragma: no cover + message = ('Following error occurred when deleting the tempdir {0}' + ' for path {1} during tearDown process: {2}' + .format(self.tempdir, path, str(excinfo))) + warnings.warn(message) shutil.rmtree(self.tempdir, onerror=onerror_handler) class ConfigTestCase(TempDirTestCase): diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 4ed9f0731..d26da361b 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -24,7 +24,7 @@ obtain, install, and renew certificates: manage certificates: certificates Display information about certificates you have from Certbot - revoke Revoke a certificate (supply --cert-path) + revoke Revoke a certificate (supply --cert-path or --cert-name) delete Delete a certificate manage your account with Let's Encrypt: @@ -108,7 +108,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.27.1 + "". (default: CertbotACMEClient/0.28.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -261,7 +261,8 @@ manage: delete Clean up all files related to a certificate renew Renew all certificates (or one specified with --cert- name) - revoke Revoke a certificate specified with --cert-path + revoke Revoke a certificate specified with --cert-path or + --cert-name update_symlinks Recreate symlinks in your /etc/letsencrypt/live/ directory @@ -475,10 +476,9 @@ apache: Apache Web Server plugin - Beta --apache-enmod APACHE_ENMOD - Path to the Apache 'a2enmod' binary (default: a2enmod) + Path to the Apache 'a2enmod' binary (default: None) --apache-dismod APACHE_DISMOD - Path to the Apache 'a2dismod' binary (default: - a2dismod) + Path to the Apache 'a2dismod' binary (default: None) --apache-le-vhost-ext APACHE_LE_VHOST_EXT SSL vhost configuration extension (default: -le- ssl.conf) @@ -492,16 +492,16 @@ apache: /var/log/apache2) --apache-challenge-location APACHE_CHALLENGE_LOCATION Directory path for challenge configuration (default: - /etc/apache2) + /etc/apache2/other) --apache-handle-modules APACHE_HANDLE_MODULES Let installer handle enabling required modules for you - (Only Ubuntu/Debian currently) (default: True) + (Only Ubuntu/Debian currently) (default: False) --apache-handle-sites APACHE_HANDLE_SITES Let installer handle enabling sites for you (Only - Ubuntu/Debian currently) (default: True) + Ubuntu/Debian currently) (default: False) --apache-ctl APACHE_CTL Full path to Apache control script (default: - apache2ctl) + apachectl) certbot-route53:auth: Obtain certificates using a DNS TXT record (if you are using AWS Route53 @@ -602,7 +602,7 @@ dns-linode: --dns-linode-propagation-seconds DNS_LINODE_PROPAGATION_SECONDS The number of seconds to wait for DNS to propagate before asking the ACME server to verify the DNS - record. (default: 960) + record. (default: 1200) --dns-linode-credentials DNS_LINODE_CREDENTIALS Linode credentials INI file. (default: None) diff --git a/docs/contributing.rst b/docs/contributing.rst index 58db251d4..ead4d7e2b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -38,13 +38,13 @@ Certbot. cd certbot ./certbot-auto --debug --os-packages-only - tools/venv.sh + python tools/venv.py -If you have Python3 available and want to use it, run the ``venv3.sh`` script. +If you have Python3 available and want to use it, run the ``venv3.py`` script. .. code-block:: shell - tools/venv3.sh + python tools/venv3.py .. note:: You may need to repeat this when Certbot's dependencies change or when a new plugin is introduced. @@ -353,7 +353,7 @@ Steps: 1. Write your code! 2. Make sure your environment is set up properly and that you're in your - virtualenv. You can do this by running ``./tools/venv.sh``. + virtualenv. You can do this by running ``pip tools/venv.py``. (this is a **very important** step) 3. Run ``tox -e lint`` to check for pylint errors. Fix any errors. 4. Run ``tox --skip-missing-interpreters`` to run the entire test suite diff --git a/letsencrypt-auto b/letsencrypt-auto index 076c45e39..fe87317a7 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.27.1" +LE_AUTO_VERSION="0.28.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -195,7 +195,7 @@ if [ "$1" = "--cb-auto-has-root" ]; then else SetRootAuthMechanism if [ -n "$SUDO" ]; then - echo "Requesting to rerun $0 with root privileges..." + say "Requesting to rerun $0 with root privileges..." $SUDO "$0" --cb-auto-has-root "$@" exit 0 fi @@ -1197,18 +1197,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 747d98e2d..57745758b 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAluRtuUACgkQTRfJlc2X -dfIvhgf7BrKDo9wjHU8Yb2h1O63OJmoYSQMqM4Q44OVkTTjHQZgDYrOflbegq9g+ -nxxOcMakiPTxvefZOecczKGTZZ/S+A/w5kH/9vJbxW0277iNnYsj1G59m1UPNzgn -ECFL5AUKhl/RF3NWSpe2XhGA7ybls8LAidwxeS3b3nXNeuXIspKd84AIAqaWlpOa -I16NhJsU8VOq6I5RCgkx4WgmmUhCmzjLbYDH7rjj1dehCZa0Y63mlMdTKKs4BJSk -AtSVVV6nTupZdHPJtpQ1RxcT6iTy8Nr13cVuKnluui7KZ/uktOdB0H1o5kuWchvm -8/oqLVSfoqjhU6Fn/11Af+iCnpICUw== -=QRnC +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlvjV5wACgkQTRfJlc2X +dfKkRwf+MJ/Yo5ix7rxGMoliJl3GUUC2KvuYxObvbsAZW69Zl4aZVNeUP3Pe/EZj +zJlSMuiCPeTMmmr0+q78dk5Qk0vf+9D5qSQyy2U+RvPvX6z1PfaFXwjETwOEhE4i +7pABP4m/rIhlZbh336gou4XZK8sXsKHXBLQEyqmzPm6YFZ+5vowIoEinrN73PBuq +rgvoTFKi2NTjYNkQffYUeCIgO0pXlaOa8hkaupqoejHHEjjiXS2C9m0gAT2Wk2cO +zya5WQNcCCLWy/ChhPE2M7yRSpwqrszsHP0qo7QGL8vvsdXvNeJ7vwpAlq/9aipg +PpzSXy/ek8YAgApaj8+/w4OfdDhQ4Q== +=1hD2 -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 25c293898..3f5eec88d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.28.0.dev0" +LE_AUTO_VERSION="0.29.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -594,7 +594,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 @@ -1206,18 +1206,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1269,7 +1269,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -1281,7 +1281,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -1374,7 +1374,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -1387,7 +1387,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -1406,7 +1406,6 @@ def main(): if __name__ == '__main__': exit(main()) - UNLIKELY_EOF # ------------------------------------------------------------------------- # Set PATH so pipstrap upgrades the right (v)env: diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index b717e359b..33f5b2c00 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index 5759336c5..c55527590 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -8,7 +8,7 @@ BootstrapArchCommon() { # # "python-virtualenv" is Python3, but "python2-virtualenv" provides # only "virtualenv2" binary, not "virtualenv" necessary in - # ./tools/_venv_common.sh + # ./tools/_venv_common.py deps=" python2 diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index b9cd42694..401a8e25c 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.27.1 \ - --hash=sha256:89a8d8e44e272ee970259c93fa2ff2c9f063da8fd88a56d7ca30d7a2218791ea \ - --hash=sha256:3570bd14ed223c752f309dbd082044bd9f11a339d21671e70a2eeae4e51ed02a -acme==0.27.1 \ - --hash=sha256:0d42cfc9050a2e1d6d4e6b66334df8173778db0b3fe7a2b3bcb58f7034913597 \ - --hash=sha256:31a7b9023ce183616e6ebd5d783e842c3d68696ff70db59a06db9feea8f54f90 -certbot-apache==0.27.1 \ - --hash=sha256:1c73297e6a59cebcf5f5692025d4013ccd02c858bdc946fee3c6613f62bb9414 \ - --hash=sha256:61d6d706d49d726b53a831a2ea9099bd6c02657ff537a166dd197cd5f494d854 -certbot-nginx==0.27.1 \ - --hash=sha256:9772198bcfde9b68e448c15c3801b3cf9d20eb9ea9da1d9f4f9a7692b0fc2314 \ - --hash=sha256:ff5b849a9b4e3d1fd50ea351a1393738382fc9bd47bc5ac18c343d11a691349f +certbot==0.28.0 \ + --hash=sha256:f2f7c816acd695cbcda713a779b0db2b08e9de407146b46e1c4ef5561e0f5417 \ + --hash=sha256:31e3e2ee2a25c009a621c59ac9182f85d937a897c7bd1d47d0e01f3c712a090a +acme==0.28.0 \ + --hash=sha256:d3a564031155fece3f6edce8c5246d4cf34be0977b4c7c5ce84e86c68601c895 \ + --hash=sha256:bf7c2f1c24a26ab5b9fce3a6abca1d74a5914d46919649ae00ad5817db62bb85 +certbot-apache==0.28.0 \ + --hash=sha256:a57d7bac4f13ae5ecea4f4cbd479d381c02316c4832b25ab5a9d7c6826166370 \ + --hash=sha256:3f93f5de4a548e973c493a6cac5eeeb3dbbcae2988b61299ea0727d04a00f5bb +certbot-nginx==0.28.0 \ + --hash=sha256:1822a65910f0801087fa20a3af3fc94f878f93e0f11809483bb5387df861e296 \ + --hash=sha256:426fb403b0a7b203629f4e350a862cbc3bc1f69936fdab8ec7eafe0d8a3b5ddb diff --git a/letsencrypt-auto-source/pieces/pipstrap.py b/letsencrypt-auto-source/pieces/pipstrap.py index d55d5bceb..f21d36657 100755 --- a/letsencrypt-auto-source/pieces/pipstrap.py +++ b/letsencrypt-auto-source/pieces/pipstrap.py @@ -45,7 +45,7 @@ except ImportError: cmd = popenargs[0] raise CalledProcessError(retcode, cmd) return output -from sys import exit, version_info +from sys import exit, version_info, executable from tempfile import mkdtemp try: from urllib2 import build_opener, HTTPHandler, HTTPSHandler @@ -57,7 +57,7 @@ except ImportError: from urllib.parse import urlparse # 3.4 -__version__ = 1, 5, 1 +__version__ = 2, 0, 0 PIP_VERSION = '9.0.1' DEFAULT_INDEX_BASE = 'https://pypi.python.org' @@ -150,7 +150,7 @@ def get_index_base(): def main(): - pip_version = StrictVersion(check_output(['pip', '--version']) + pip_version = StrictVersion(check_output([executable, '-m', 'pip', '--version']) .decode('utf-8').split()[1]) min_pip_version = StrictVersion(PIP_VERSION) if pip_version >= min_pip_version: @@ -163,7 +163,7 @@ def main(): temp, digest) for path, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + + check_output('{0} -m pip install --no-index --no-deps -U '.format(quote(executable)) + # Disable cache since we're not using it and it otherwise # sometimes throws permission warnings: ('--no-cache-dir ' if has_pip_cache else '') + @@ -181,4 +181,4 @@ def main(): if __name__ == '__main__': - exit(main()) + exit(main()) \ No newline at end of file diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 6b5d63c80..4036e6efa 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -45,7 +45,7 @@ if [ $? -ne 0 ] ; then exit 1 fi -tools/_venv_common.sh -e acme[dev] -e .[dev,docs] -e certbot-apache +python tools/_venv_common.py -e acme[dev] -e .[dev,docs] -e certbot-apache sudo venv/bin/certbot -v --debug --text --agree-dev-preview --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL diff --git a/tests/letstest/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh index 84e4bcd22..bb9126673 100755 --- a/tests/letstest/scripts/test_tox.sh +++ b/tests/letstest/scripts/test_tox.sh @@ -14,5 +14,5 @@ VENV_BIN=${VENV_PATH}/bin "$LEA_PATH/letsencrypt-auto" --os-packages-only cd letsencrypt -./tools/venv.sh +python tools/venv.py venv/bin/tox -e py27 diff --git a/tests/lock_test.py b/tests/lock_test.py index b01cc5d58..0266cf029 100644 --- a/tests/lock_test.py +++ b/tests/lock_test.py @@ -1,4 +1,6 @@ """Tests to ensure the lock order is preserved.""" +from __future__ import print_function + import atexit import functools import logging @@ -235,4 +237,9 @@ def log_output(level, out, err): if __name__ == "__main__": - main() + if os.name != 'nt': + main() + else: + print( + 'Warning: lock_test cannot be executed on Windows, ' + 'as it relies on a Nginx distribution for Linux.') diff --git a/tests/modification-check.py b/tests/modification-check.py new file mode 100755 index 000000000..8abc0fbfe --- /dev/null +++ b/tests/modification-check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile +import shutil +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +def find_repo_path(): + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +# We do not use filecmp.cmp to take advantage of universal newlines +# handling in open() for Python 3.x and be insensitive to CRLF/LF when run on Windows. +# As a consequence, this function will not work correctly if executed by Python 2.x on Windows. +# But it will work correctly on Linux for any version, because every file tested will be LF. +def compare_files(path_1, path_2): + l1 = l2 = True + with open(path_1, 'r') as f1, open(path_2, 'r') as f2: + line = 1 + while l1 and l2: + line += 1 + l1 = f1.readline() + l2 = f2.readline() + if l1 != l2: + print('---') + print(( + 'While comparing {0} (1) and {1} (2), a difference was found at line {2}:' + .format(os.path.basename(path_1), os.path.basename(path_2), line))) + print('(1): {0}'.format(repr(l1))) + print('(2): {0}'.format(repr(l2))) + print('---') + return False + + return True + +def validate_scripts_content(repo_path, temp_cwd): + errors = False + + if not compare_files( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(repo_path, 'letsencrypt-auto')): + print('Root certbot-auto and letsencrypt-auto differ.') + errors = True + else: + shutil.copyfile( + os.path.join(repo_path, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')) + shutil.copy(os.path.normpath(os.path.join( + repo_path, + 'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd) + + # Compare file against current version in the target branch + branch = os.environ.get('TRAVIS_BRANCH', 'master') + url = ( + 'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto' + .format(branch)) + urlretrieve(url, os.path.join(temp_cwd, 'certbot-auto')) + + if compare_files( + os.path.join(temp_cwd, 'certbot-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were unchanged') + else: + # Compare file against the latest released version + latest_version = subprocess.check_output( + [sys.executable, 'fetch.py', '--latest-version'], cwd=temp_cwd) + subprocess.check_call( + [sys.executable, 'fetch.py', '--le-auto-script', + 'v{0}'.format(latest_version.decode().strip())], cwd=temp_cwd) + if compare_files( + os.path.join(temp_cwd, 'letsencrypt-auto'), + os.path.join(temp_cwd, 'local-auto')): + print('Root *-auto were updated to the latest version.') + else: + print('Root *-auto have unexpected changes.') + errors = True + + return errors + +def main(): + repo_path = find_repo_path() + temp_cwd = tempfile.mkdtemp() + errors = False + + try: + errors = validate_scripts_content(repo_path, temp_cwd) + + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'original-lea') + ) + subprocess.check_call([sys.executable, os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/build.py'))]) + shutil.copyfile( + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')), + os.path.join(temp_cwd, 'build-lea') + ) + shutil.copyfile( + os.path.join(temp_cwd, 'original-lea'), + os.path.normpath(os.path.join(repo_path, 'letsencrypt-auto-source/letsencrypt-auto')) + ) + + if not compare_files( + os.path.join(temp_cwd, 'original-lea'), + os.path.join(temp_cwd, 'build-lea')): + print('Script letsencrypt-auto-source/letsencrypt-auto ' + 'doesn\'t match output of build.py.') + errors = True + else: + print('Script letsencrypt-auto-source/letsencrypt-auto matches output of build.py.') + finally: + shutil.rmtree(temp_cwd) + + return errors + +if __name__ == '__main__': + if main(): + sys.exit(1) diff --git a/tests/modification-check.sh b/tests/modification-check.sh deleted file mode 100755 index 0145b0228..000000000 --- a/tests/modification-check.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -e - -temp_dir=`mktemp -d` -trap "rm -rf $temp_dir" EXIT - -# cd to repo root -cd $(dirname $(dirname $(readlink -f $0))) -FLAG=false - -if ! cmp -s certbot-auto letsencrypt-auto; then - echo "Root certbot-auto and letsencrypt-auto differ." - FLAG=true -else - cp certbot-auto "$temp_dir/local-auto" - cp letsencrypt-auto-source/pieces/fetch.py "$temp_dir/fetch.py" - cd $temp_dir - - # Compare file against current version in the target branch - BRANCH=${TRAVIS_BRANCH:-master} - URL="https://raw.githubusercontent.com/certbot/certbot/$BRANCH/certbot-auto" - curl -sS $URL > certbot-auto - if cmp -s certbot-auto local-auto; then - echo "Root *-auto were unchanged." - else - # Compare file against the latest released version - python fetch.py --le-auto-script "v$(python fetch.py --latest-version)" - if cmp -s letsencrypt-auto local-auto; then - echo "Root *-auto were updated to the latest version." - else - echo "Root *-auto have unexpected changes." - FLAG=true - fi - fi - cd ~- -fi - -# Compare letsencrypt-auto-source/letsencrypt-auto with output of build.py - -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/original-lea -python letsencrypt-auto-source/build.py -cp letsencrypt-auto-source/letsencrypt-auto ${temp_dir}/build-lea -cp ${temp_dir}/original-lea letsencrypt-auto-source/letsencrypt-auto - -cd $temp_dir - -if ! cmp -s original-lea build-lea; then - echo "letsencrypt-auto-source/letsencrypt-auto doesn't match output of \ -build.py." - FLAG=true -else - echo "letsencrypt-auto-source/letsencrypt-auto matches output of \ -build.py." -fi - -rm -rf $temp_dir - -if $FLAG ; then - exit 1 -fi diff --git a/tools/_venv_common.py b/tools/_venv_common.py new file mode 100755 index 000000000..c44d05bf7 --- /dev/null +++ b/tools/_venv_common.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import shutil +import glob +import time +import subprocess +import sys +import re + +VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') + + +class PythonExecutableNotFoundError(Exception): + pass + + +def find_python_executable(python_major): + # type: (int) -> str + """ + Find the relevant python executable that is of the given python major version. + Will test, in decreasing priority order: + * the current Python interpreter + * 'pythonX' executable in PATH (with X the given major version) if available + * 'python' executable in PATH if available + * Windows Python launcher 'py' executable in PATH if available + Incompatible python versions for Certbot will be evicted (eg. Python < 3.5 on Windows) + :param int python_major: the Python major version to target (2 or 3) + :rtype: str + :return: the relevant python executable path + :raise RuntimeError: if no relevant python executable path could be found + """ + python_executable_path = None + + # First try, current python executable + if _check_version('{0}.{1}.{2}'.format( + sys.version_info[0], sys.version_info[1], sys.version_info[2]), python_major): + return sys.executable + + # Second try, with python executables in path + versions_to_test = ['2.7', '2', ''] if python_major == 2 else ['3', ''] + for one_version in versions_to_test: + try: + one_python = 'python{0}'.format(one_version) + output = subprocess.check_output([one_python, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output.strip().split()[1], python_major): + return subprocess.check_output([one_python, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + # Last try, with Windows Python launcher + try: + env_arg = '-{0}'.format(python_major) + output_version = subprocess.check_output(['py', env_arg, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output_version.strip().split()[1], python_major): + return subprocess.check_output(['py', env_arg, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + if not python_executable_path: + raise RuntimeError('Error, no compatible Python {0} executable for Certbot could be found.' + .format(python_major)) + + +def _check_version(version_str, major_version): + search = VERSION_PATTERN.search(version_str) + + if not search: + return False + + version = (int(search.group(1)), int(search.group(2))) + + minimal_version_supported = (2, 7) + if major_version == 3 and os.name == 'nt': + minimal_version_supported = (3, 5) + elif major_version == 3: + minimal_version_supported = (3, 4) + + if version >= minimal_version_supported: + return True + + print('Incompatible python version for Certbot found: {0}'.format(version_str)) + return False + + +def subprocess_with_print(command): + print(command) + subprocess.check_call(command, shell=True) + + +def get_venv_python(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + if os.path.isfile(python_linux): + return python_linux + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_windows): + return python_windows + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + + +def main(venv_name, venv_args, args): + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' + .format(sys.executable, venv_name, venv_args)) + + python_executable = get_venv_python(venv_name) + + subprocess_with_print('"{0}" {1}'.format( + python_executable, + os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) + subprocess_with_print('"{0}" {1} {2}'.format( + python_executable, + os.path.normpath('./tools/pip_install.py'), + ' '.join(args))) + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + + +if __name__ == '__main__': + main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:]) diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh deleted file mode 100755 index 0f0ff7e28..000000000 --- a/tools/_venv_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -xe - -VENV_NAME=${VENV_NAME:-venv} - -# .egg-info directories tend to cause bizarre problems (e.g. `pip -e -# .` might unexpectedly install letshelp-certbot only, in case -# `python letshelp-certbot/setup.py build` has been called -# earlier) -rm -rf *.egg-info - -# virtualenv setup is NOT idempotent: shutil.Error: -# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and -# `venv/bin/python2` are the same file -mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true -virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS -. ./$VENV_NAME/bin/activate - -# Use pipstrap to update Python packaging tools to only update to a well tested -# version and to work around https://github.com/pypa/pip/issues/4817 on older -# systems. -python letsencrypt-auto-source/pieces/pipstrap.py -./tools/pip_install.sh "$@" - -set +x -echo "Please run the following command to activate developer environment:" -echo "source $VENV_NAME/bin/activate" diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 6fb8ed0d9..65182bcde 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.12.36 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.3 +dns-lexicon==2.7.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 diff --git a/tools/install_and_test.py b/tools/install_and_test.py new file mode 100755 index 000000000..1eb81770c --- /dev/null +++ b/tools/install_and_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# pip installs the requested packages in editable mode and runs unit tests on +# them. Each package is installed and tested in the order they are provided +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.py for more information on the versions pinned +# to. +from __future__ import print_function + +import os +import sys +import tempfile +import shutil +import subprocess +import re + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def call_with_print(command, cwd=None): + print(command) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + if os.environ.get('CERTBOT_NO_PIN') == '1': + command = [sys.executable, '-m', 'pip', '-q', '-e'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + + new_args = [] + for arg in args: + if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested.' + .format(arg))) + else: + new_args.append(arg) + + for requirement in new_args: + current_command = command[:] + current_command.append(requirement) + call_with_print(' '.join(current_command)) + pkg = re.sub(r'\[\w+\]', '', requirement) + + if pkg == '.': + pkg = 'certbot' + + temp_cwd = tempfile.mkdtemp() + shutil.copy2("pytest.ini", temp_cwd) + try: + call_with_print(' '.join([ + sys.executable, '-m', 'pytest', '--quiet', pkg.replace('-', '_')]), cwd=temp_cwd) + finally: + shutil.rmtree(temp_cwd) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh deleted file mode 100755 index e84fbc99e..000000000 --- a/tools/install_and_test.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -e -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. - -if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -q -e" -else - pip_install="$(dirname $0)/pip_install_editable.sh" -fi - -temp_cwd=$(mktemp -d) -trap "rm -rf $temp_cwd" EXIT -cp pytest.ini "$temp_cwd" - -set -x -for requirement in "$@" ; do - $pip_install $requirement - pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] - pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names - if [ $pkg = "." ]; then - pkg="certbot" - fi - cd "$temp_cwd" - pytest --quiet $pkg - cd - -done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index c8fb95351..ad44a55d0 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys - def read_file(file_path): """Reads in a Python requirements file. @@ -32,17 +31,17 @@ def read_file(file_path): return d -def print_requirements(requirements): - """Prints requirements to stdout. +def output_requirements(requirements): + """Prepare print requirements to stdout. :param dict requirements: mapping from a project to its pinned version """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) + return '\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items())) -def merge_requirements_files(*files): +def main(*files): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier @@ -54,8 +53,9 @@ def merge_requirements_files(*files): d = {} for f in files: d.update(read_file(f)) - print_requirements(d) + return output_requirements(d) if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) + merged_requirements = main(*sys.argv[1:]) + print(merged_requirements) diff --git a/tools/pip_install.py b/tools/pip_install.py new file mode 100755 index 000000000..8878674c9 --- /dev/null +++ b/tools/pip_install.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e ` and +# no other arguments. + +from __future__ import print_function, absolute_import + +import subprocess +import os +import sys +import re +import shutil +import tempfile + +import merge_requirements as merge_module +import readlink + + +def find_tools_path(): + return os.path.dirname(readlink.main(__file__)) + + +def certbot_oldest_processing(tools_path, args, test_constraints): + if args[0] != '-e' or len(args) != 2: + raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' + 'with a single -e argument.') + # remove any extras such as [dev] + pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + # packages like acme don't have any local oldest requirements + if not os.path.isfile(requirements): + requirements = None + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + + return requirements + + +def certbot_normal_processing(tools_path, test_constraints): + repo_path = os.path.dirname(tools_path) + certbot_requirements = os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + with open(certbot_requirements, 'r') as fd: + data = fd.readlines() + with open(test_constraints, 'w') as fd: + for line in data: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + fd.write('{0}{1}'.format(search.group(1), os.linesep)) + + +def merge_requirements(tools_path, test_constraints, all_constraints): + merged_requirements = merge_module.main( + os.path.join(tools_path, 'dev_constraints.txt'), + test_constraints + ) + with open(all_constraints, 'w') as fd: + fd.write(merged_requirements) + + +def call_with_print(command, cwd=None): + print(command) + subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + + +def main(args): + tools_path = find_tools_path() + working_dir = tempfile.mkdtemp() + + try: + test_constraints = os.path.join(working_dir, 'test_constraints.txt') + all_constraints = os.path.join(working_dir, 'all_constraints.txt') + + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) + + merge_requirements(tools_path, test_constraints, all_constraints) + if requirements: + call_with_print('"{0}" -m pip install -q --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) + + call_with_print('"{0}" -m pip install -q --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) + finally: + shutil.rmtree(working_dir) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index 78e2afa17..000000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the -# top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If -# CERTBOT_OLDEST is set, this script must be run with `-e ` and -# no other arguments. - -# get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -all_constraints=$(mktemp) -test_constraints=$(mktemp) -trap "rm -f $all_constraints $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - if [ "$1" != "-e" -o "$#" -ne "2" ]; then - echo "When CERTBOT_OLDEST is set, this script must be run with a single -e argument." - exit 1 - fi - pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] - requirements="$pkg_dir/local-oldest-requirements.txt" - # packages like acme don't have any local oldest requirements - if [ ! -f "$requirements" ]; then - unset requirements - fi - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi - -"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ - "$test_constraints" > "$all_constraints" - -set -x - -# install the requested packages using the pinned requirements as constraints -if [ -n "$requirements" ]; then - pip install -q --constraint "$all_constraints" --requirement "$requirements" -fi -pip install -q --constraint "$all_constraints" "$@" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py new file mode 100755 index 000000000..8eaf3a9fa --- /dev/null +++ b/tools/pip_install_editable.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# pip installs packages in editable mode using certbot-auto's requirements file +# as constraints + +from __future__ import absolute_import + +import sys + +import pip_install + +def main(args): + new_args = [] + for arg in args: + new_args.append('-e') + new_args.append(arg) + + pip_install.main(new_args) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh deleted file mode 100755 index 6130bf6e7..000000000 --- a/tools/pip_install_editable.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -# pip installs packages in editable mode using certbot-auto's requirements file -# as constraints - -args="" -for requirement in "$@" ; do - args="$args -e $requirement" -done - -"$(dirname $0)/pip_install.sh" $args diff --git a/tools/readlink.py b/tools/readlink.py index 02c74c48d..0199ce184 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -7,7 +7,12 @@ platforms. """ from __future__ import print_function + import os import sys -print(os.path.realpath(sys.argv[1])) +def main(link): + return os.path.realpath(link) + +if __name__ == '__main__': + print(main(sys.argv[1])) diff --git a/tools/venv.py b/tools/venv.py new file mode 100755 index 000000000..93b012e76 --- /dev/null +++ b/tools/venv.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client +import os + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + + +def main(): + if os.name == 'nt': + raise ValueError('Certbot for Windows is not supported on Python 2.x.') + + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(2)) + _venv_common.main('venv', venv_args, REQUIREMENTS) + + +if __name__ == '__main__': + main() diff --git a/tools/venv.sh b/tools/venv.sh deleted file mode 100755 index 5692f9ebf..000000000 --- a/tools/venv.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -xe -# Developer virtualenv setup for Certbot client - -if command -v python2; then - export VENV_ARGS="--python python2" -elif command -v python2.7; then - export VENV_ARGS="--python python2.7" -else - echo "Couldn't find python2 or python2.7 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tools/venv3.py b/tools/venv3.py new file mode 100755 index 000000000..c2374ba5a --- /dev/null +++ b/tools/venv3.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + + +def main(): + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(3)) + _venv_common.main('venv3', venv_args, REQUIREMENTS) + + +if __name__ == '__main__': + main() diff --git a/tools/venv3.sh b/tools/venv3.sh deleted file mode 100755 index 07512f370..000000000 --- a/tools/venv3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -xe -# Developer Python3 virtualenv setup for Certbot - -if command -v python3; then - export VENV_NAME="${VENV_NAME:-venv3}" - export VENV_ARGS="--python python3" -else - echo "Couldn't find python3 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tox-win.ini b/tox-win.ini deleted file mode 100644 index fe063c264..000000000 --- a/tox-win.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -skipsdist = True -envlist = py{34,35,36,37}-cover - -[testenv] -deps = -e acme[dev] - -e .[dev] -commands = pytest -n auto --pyargs acme - pytest -n auto --pyargs certbot - -[testenv:cover] -commands = pytest -n auto --cov acme --pyargs acme - pytest -n auto --cov certbot --cov-append --pyargs certbot diff --git a/tox.cover.py b/tox.cover.py new file mode 100755 index 000000000..c8a45d82d --- /dev/null +++ b/tox.cover.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +import argparse +import subprocess +import os +import sys + +DEFAULT_PACKAGES = [ + 'certbot', 'acme', 'certbot_apache', 'certbot_dns_cloudflare', 'certbot_dns_cloudxns', + 'certbot_dns_digitalocean', 'certbot_dns_dnsimple', 'certbot_dns_dnsmadeeasy', + 'certbot_dns_gehirn', 'certbot_dns_google', 'certbot_dns_linode', 'certbot_dns_luadns', + 'certbot_dns_nsone', 'certbot_dns_ovh', 'certbot_dns_rfc2136', 'certbot_dns_route53', + 'certbot_dns_sakuracloud', 'certbot_nginx', 'certbot_postfix', 'letshelp_certbot'] + +COVER_THRESHOLDS = { + 'certbot': {'linux': 98, 'windows': 94}, + 'acme': {'linux': 100, 'windows': 99}, + 'certbot_apache': {'linux': 100, 'windows': 100}, + 'certbot_dns_cloudflare': {'linux': 98, 'windows': 98}, + 'certbot_dns_cloudxns': {'linux': 99, 'windows': 99}, + 'certbot_dns_digitalocean': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsimple': {'linux': 98, 'windows': 98}, + 'certbot_dns_dnsmadeeasy': {'linux': 99, 'windows': 99}, + 'certbot_dns_gehirn': {'linux': 97, 'windows': 97}, + 'certbot_dns_google': {'linux': 99, 'windows': 99}, + 'certbot_dns_linode': {'linux': 98, 'windows': 98}, + 'certbot_dns_luadns': {'linux': 98, 'windows': 98}, + 'certbot_dns_nsone': {'linux': 99, 'windows': 99}, + 'certbot_dns_ovh': {'linux': 97, 'windows': 97}, + 'certbot_dns_rfc2136': {'linux': 99, 'windows': 99}, + 'certbot_dns_route53': {'linux': 92, 'windows': 92}, + 'certbot_dns_sakuracloud': {'linux': 97, 'windows': 97}, + 'certbot_nginx': {'linux': 97, 'windows': 97}, + 'certbot_postfix': {'linux': 100, 'windows': 100}, + 'letshelp_certbot': {'linux': 100, 'windows': 100} +} + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def cover(package): + threshold = COVER_THRESHOLDS.get(package)['windows' if os.name == 'nt' else 'linux'] + + pkg_dir = package.replace('_', '-') + + if os.name == 'nt' and pkg_dir in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested/covered.' + .format(pkg_dir))) + return + + subprocess.check_call([ + sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', package]) + subprocess.check_call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', + '{0}/*'.format(pkg_dir), '--show-missing']) + +def main(): + description = """ +This script is used by tox.ini (and thus by Travis CI and AppVeyor) in order +to generate separate stats for each package. It should be removed once those +packages are moved to a separate repo. + +Option -e makes sure we fail fast and don't submit to codecov.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--packages', nargs='+') + + args = parser.parse_args() + + packages = args.packages or DEFAULT_PACKAGES + + # --cov-append is on, make sure stats are correct + try: + os.remove('.coverage') + except OSError: + pass + + for package in packages: + cover(package) + +if __name__ == '__main__': + main() diff --git a/tox.cover.sh b/tox.cover.sh deleted file mode 100755 index a59b55239..000000000 --- a/tox.cover.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -xe - -# USAGE: ./tox.cover.sh [package] -# -# This script is used by tox.ini (and thus Travis CI) in order to -# generate separate stats for each package. It should be removed once -# those packages are moved to separate repo. -# -# -e makes sure we fail fast and don't submit to codecov - -if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_dnsmadeeasy certbot_dns_gehirn certbot_dns_google certbot_dns_linode certbot_dns_luadns certbot_dns_nsone certbot_dns_ovh certbot_dns_rfc2136 certbot_dns_route53 certbot_dns_sakuracloud certbot_nginx certbot_postfix letshelp_certbot" -else - pkgs="$@" -fi - -cover () { - if [ "$1" = "certbot" ]; then - min=98 - elif [ "$1" = "acme" ]; then - min=100 - elif [ "$1" = "certbot_apache" ]; then - min=100 - elif [ "$1" = "certbot_dns_cloudflare" ]; then - min=98 - elif [ "$1" = "certbot_dns_cloudxns" ]; then - min=99 - elif [ "$1" = "certbot_dns_digitalocean" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsimple" ]; then - min=98 - elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then - min=99 - elif [ "$1" = "certbot_dns_gehirn" ]; then - min=97 - elif [ "$1" = "certbot_dns_google" ]; then - min=99 - elif [ "$1" = "certbot_dns_linode" ]; then - min=98 - elif [ "$1" = "certbot_dns_luadns" ]; then - min=98 - elif [ "$1" = "certbot_dns_nsone" ]; then - min=99 - elif [ "$1" = "certbot_dns_ovh" ]; then - min=97 - elif [ "$1" = "certbot_dns_rfc2136" ]; then - min=99 - elif [ "$1" = "certbot_dns_route53" ]; then - min=92 - elif [ "$1" = "certbot_dns_sakuracloud" ]; then - min=97 - elif [ "$1" = "certbot_nginx" ]; then - min=97 - elif [ "$1" = "certbot_postfix" ]; then - min=100 - elif [ "$1" = "letshelp_certbot" ]; then - min=100 - else - echo "Unrecognized package: $1" - exit 1 - fi - - pkg_dir=$(echo "$1" | tr _ -) - pytest --cov "$pkg_dir" --cov-append --cov-report= "$1" - coverage report --fail-under="$min" --include="$pkg_dir/*" --show-missing -} - -rm -f .coverage # --cov-append is on, make sure stats are correct -for pkg in $pkgs -do - cover $pkg -done diff --git a/tox.ini b/tox.ini index 9db06f78c..e38f1311e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,16 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},cover,lint +envlist = modification,py{34,35,36},py27-cover,lint [base] # pip installs the requested packages in editable mode -pip_install = {toxinidir}/tools/pip_install_editable.sh +pip_install = python {toxinidir}/tools/pip_install_editable.py # pip installs the requested packages in editable mode and runs unit tests on # them. Each package is installed and tested in the order they are provided # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. -install_and_test = {toxinidir}/tools/install_and_test.sh +install_and_test = python {toxinidir}/tools/install_and_test.py dns_packages = certbot-dns-cloudflare \ certbot-dns-cloudxns \ @@ -38,7 +38,7 @@ all_packages = certbot-postfix \ letshelp-certbot install_packages = - {toxinidir}/tools/pip_install_editable.sh {[base]all_packages} + python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} source_paths = acme/acme certbot @@ -64,7 +64,9 @@ source_paths = tests/lock_test.py [testenv] -passenv = TRAVIS +passenv = + TRAVIS + APPVEYOR commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -120,11 +122,17 @@ basepython = python2.7 commands = {[base]install_packages} -[testenv:cover] +[testenv:py27-cover] basepython = python2.7 commands = {[base]install_packages} - ./tox.cover.sh + python tox.cover.py + +[testenv:py37-cover] +basepython = python3.7 +commands = + {[base]install_packages} + python tox.cover.py [testenv:lint] basepython = python2.7 @@ -133,7 +141,7 @@ basepython = python2.7 # continue, but tox return code will reflect previous error commands = {[base]install_packages} - pylint --reports=n --rcfile=.pylintrc {[base]source_paths} + python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} [testenv:mypy] basepython = python3 @@ -157,7 +165,7 @@ commands = # allow users to run the modification check by running `tox` [testenv:modification] commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py [testenv:apache_compat] commands = @@ -197,7 +205,7 @@ passenv = # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Trusty Docker image. commands = - {toxinidir}/tests/modification-check.sh + python {toxinidir}/tests/modification-check.py docker build -f letsencrypt-auto-source/Dockerfile.trusty -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals =