From 835f4b9271b7b5ee3cb3f54254a015e28f7bffdc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 17 Jan 2019 22:02:35 +0100 Subject: [PATCH] Allow to execute a tox target without pinned dependencies (#6590) This PR passes the CERTBOT_NO_PIN environment variable to the unit tests tox envs. By setting CERTBOT_NO_PIN to 1 before executing a given tox env, certbot dependencies will be installed at their latest version instead of the usual pinned version. I also moved the unpin logic one layer below to allow it to be used potentially more widely, and avoid unnecessary merging constraints operation in this case. As warnings are errors now, latest versions of Python will break now the tests, because collections launch a warning when some classes are imported from collections instead of collections.abc. Certbot code is patched, and warning is ignored for now, because a lot of third party libraries still depend on this behavior. * Allow to execute a tox target without pinned dependencies * Correct lint * Retrigger build. * Remove debug code * Added test against unpinned dependencies from test-everything-unpinned-dependencies branch * Remove duplicated assertion to pass TRAVIS and APPVEYOR in default tox environment. --- .travis.yml | 4 ++++ acme/acme/messages.py | 7 +++++-- pytest.ini | 10 +++++++--- tools/install_and_test.py | 9 ++++----- tools/pip_install.py | 27 +++++++++++++++++---------- tox.ini | 2 ++ 6 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38543845a..16cb6f23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,10 @@ matrix: env: TOXENV=nginxroundtrip # These environments are executed on cron events only + - python: "3.7" + dist: xenial + env: TOXENV=py37 CERTBOT_NO_PIN=1 + if: type = cron - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4400a6c31..7c82c8507 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,7 +1,10 @@ """ACME protocol messages.""" -import collections import six import json +try: + from collections.abc import Hashable # pylint: disable=no-name-in-module +except ImportError: + from collections import Hashable import josepy as jose @@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None).decode() -class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore +class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented diff --git a/pytest.ini b/pytest.ini index 9a5807f34..8f009045c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,10 +3,14 @@ # directly. [pytest] addopts = --numprocesses auto --pyargs -# ResourceWarnings are ignored as errors, since they're raised at close -# decodestring: https://github.com/rthalley/dnspython/issues/338 -# ignore our own TLS-SNI-01 warning +# In general, all warnings are treated as errors. Here are the exceptions: +# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- ignore our own TLS-SNI-01 warning +# 3- ignore warn for importing abstract classes from collections instead of collections.abc, +# too much third party dependencies are still relying on this behavior, +# but it should be corrected to allow Certbot compatiblity with Python >= 3.8 filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning + ignore:.*collections\.abc:DeprecationWarning diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 79a7c2264..0142eeea4 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -17,16 +17,15 @@ import re SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): - if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-e'] - else: - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] new_args = [] for arg in args: diff --git a/tools/pip_install.py b/tools/pip_install.py index 4466729e0..332422019 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -80,19 +80,26 @@ def main(args): test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') - requirements = None - if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + if os.environ.get('CERTBOT_NO_PIN') == '1': + # With unpinned dependencies, there is no constraint + call_with_print('"{0}" -m pip install --ignore-installed {1}' + .format(sys.executable, ' '.join(args))) else: - certbot_normal_processing(tools_path, test_constraints) + # Otherwise, we merge requirements to build the constraints and pin dependencies + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) + merge_requirements(tools_path, test_constraints, all_constraints) - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + if requirements: + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) + + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: if os.environ.get('TRAVIS'): print('travis_fold:end:install_certbot_deps') diff --git a/tox.ini b/tox.ini index 36057b5df..363f06bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,8 @@ source_paths = tests/lock_test.py [testenv] +passenv = + CERTBOT_NO_PIN commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py