From 08839758bd80015377e8aaf61361f1c091d4c98f Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 22 Jul 2021 12:00:30 -0700 Subject: [PATCH] Finish pinning system rewrite (#8934) * add oldest pyproject.toml file that works * make single oldest_constraints.txt file * remove unused merge_requirements.py * remove unused import * make conditional right * simplify pip_install.py * fix typo * bump min dns-lexicon dependency * fix zope import warning * pin back wheel * refactor pinning script * Add oldest script. * add pip comment * add pipstrap extra * simplify pinning scripts * remove pipstrap extra * update contributing * Add design doc * Update tools/pinning/DESIGN.md Co-authored-by: ohemorange * Update tools/pinning/DESIGN.md Co-authored-by: ohemorange * Update tools/pinning/DESIGN.md Co-authored-by: ohemorange * Update tools/pinning/DESIGN.md Co-authored-by: ohemorange * rename normal to current * no dummies * script improvements * mention need to update setup.py * try and clarify poetry behavior * tweak section title Co-authored-by: ohemorange --- acme/setup.py | 2 +- certbot-dns-cloudxns/setup.py | 2 +- certbot-dns-dnsimple/setup.py | 15 +- certbot-dns-dnsmadeeasy/setup.py | 2 +- certbot-dns-gehirn/setup.py | 2 +- certbot-dns-linode/setup.py | 2 +- certbot-dns-luadns/setup.py | 2 +- certbot-dns-nsone/setup.py | 2 +- certbot-dns-ovh/setup.py | 2 +- certbot-dns-sakuracloud/setup.py | 2 +- certbot/docs/contributing.rst | 49 +++++ certbot/setup.py | 9 +- pytest.ini | 4 + tools/dev_constraints.txt | 131 -------------- tools/merge_requirements.py | 92 ---------- tools/oldest_constraints.txt | 169 ++++++++++-------- tools/pinning/DESIGN.md | 125 +++++++++++++ .../common/export-pinned-dependencies.sh | 44 +++++ tools/pinning/{ => current}/pyproject.toml | 44 ++--- tools/pinning/current/repin.sh | 24 +++ tools/pinning/oldest/pyproject.toml | 145 +++++++++++++++ tools/pinning/oldest/repin.sh | 19 ++ tools/pinning/pin.sh | 48 ----- tools/pip_install.py | 48 ++--- tools/requirements.txt | 48 ++--- tox.ini | 3 + 26 files changed, 583 insertions(+), 452 deletions(-) delete mode 100644 tools/dev_constraints.txt delete mode 100755 tools/merge_requirements.py create mode 100644 tools/pinning/DESIGN.md create mode 100755 tools/pinning/common/export-pinned-dependencies.sh rename tools/pinning/{ => current}/pyproject.toml (71%) create mode 100755 tools/pinning/current/repin.sh create mode 100644 tools/pinning/oldest/pyproject.toml create mode 100755 tools/pinning/oldest/repin.sh delete mode 100755 tools/pinning/pin.sh diff --git a/acme/setup.py b/acme/setup.py index 9ae7d5bd9..563a523a5 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -14,7 +14,7 @@ install_requires = [ 'PyOpenSSL>=17.3.0', 'pyrfc3339', 'pytz', - 'requests>=2.6.0', + 'requests>=2.14.2', 'requests-toolbelt>=0.3.0', 'setuptools>=39.0.1', ] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 064b72be8..d200bd526 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 956a228bd..3d5e3736c 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -7,6 +7,9 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ + # This version of lexicon is required to address the problem described in + # https://github.com/AnalogJ/lexicon/issues/387. + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] @@ -25,18 +28,6 @@ elif 'bdist_wheel' in sys.argv[1:]: if os.environ.get('SNAP_BUILD'): install_requires.append('packaging') -# This package normally depends on dns-lexicon>=3.2.1 to address the -# problem described in https://github.com/AnalogJ/lexicon/issues/387, -# however, the fix there has been backported to older versions of -# lexicon found in various Linux distros. This conditional helps us test -# that we've maintained compatibility with these versions of lexicon -# which allows us to potentially upgrade our packages in these distros -# as necessary. -if os.environ.get('CERTBOT_OLDEST') == '1': - install_requires.append('dns-lexicon>=3.1.0') # Changed parameter name -else: - install_requires.append('dns-lexicon>=3.2.1') - docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 0cccf9a14..cde8e90d8 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 1bb2afb61..c0bd3eb25 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index cf7996b05..f54f98d22 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index c3791a375..0b8d7276b 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e59eaeb6f..3b8034ee7 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index b3eca861d..5ef947008 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 133a910aa..070a15591 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -7,7 +7,7 @@ from setuptools import setup version = '1.18.0.dev0' install_requires = [ - 'dns-lexicon>=3.1.0', # Changed `rtype` parameter name + 'dns-lexicon>=3.2.1', 'setuptools>=39.0.1', 'zope.interface', ] diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 94e35f1bf..0436ce995 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -566,6 +566,55 @@ and run the command: This would generate the HTML documentation in ``_build/html`` in your current ``docs/`` directory. +Certbot's dependencies +====================== + +We attempt to pin all of Certbot's dependencies whenever we can for reliability +and consistency. Some of the places we have Certbot's dependencies pinned +include our snaps, Docker images, Windows installer, CI, and our development +environments. + +In most cases, the file where dependency versions are specified is +``tools/requirements.txt``. There are two exceptions to this. The first is our +"oldest" tests where ``tools/oldest_constraints.txt`` is used instead. The +purpose of the "oldest" tests is to ensure Certbot continues to work with the +oldest versions of our dependencies which we claim to support. The oldest +versions of the dependencies we support should also be declared in our setup.py +files to communicate this information to our users. + +The second exception to using ``tools/requirements.txt`` is in our unpinned +tests. As of writing this, there is one test we run nightly in CI where we +leave Certbot's dependencies unpinned. The thinking behind this test is to help +us learn about breaking changes in our dependencies so that we can respond +accordingly. + +The choices of whether Certbot's dependencies are pinned and what file is used +if they are should be automatically handled for you most of the time by +Certbot's tooling. The way it works though is ``tools/pip_install.py`` (which +many of our other tools build on) checks for the presence of environment +variables. If ``CERTBOT_NO_PIN`` is set to 1, Certbot's dependencies will not +be pinned. If that variable is not set and ``CERTBOT_OLDEST`` is set to 1, +``tools/oldest_constraints.txt`` will be used as constraints for ``pip``. +Otherwise, ``tools/requirements.txt`` is used as constraints. + +Updating dependency versions +---------------------------- + +``tools/requirements.txt`` and ``tools/oldest_constraints.txt`` can be updated +using ``tools/pinning/current/repin.sh`` and ``tools/pinning/oldest/repin.sh`` +respectively. This works by using ``poetry`` to generate pinnings based on a +Poetry project defined by the ``pyproject.toml`` file in the same directory as +the script. In many cases, you can just run the script to generate updated +dependencies, however, if you need to pin back packages or unpin packages that +were previously restricted to an older version, you will need to modify the +``pyproject.toml`` file. The syntax used by this file is described at +https://python-poetry.org/docs/pyproject/ and how dependencies are specified in +this file is further described at +https://python-poetry.org/docs/dependency-specification/. + +If you want to learn more about the design used here, see +``tools/pinning/DESIGN.md`` in the Certbot repo. + .. _docker-dev: Running the client with Docker diff --git a/certbot/setup.py b/certbot/setup.py index 99c92a0b8..0e5a69ce8 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -70,13 +70,10 @@ dev_extras = [ 'azure-devops', 'ipdb', 'PyGithub', - 'pip', # poetry 1.2.0+ is required for it to pin pip, setuptools, and wheel. See # https://github.com/python-poetry/poetry/issues/1584. 'poetry>=1.2.0a1', - 'tox', 'twine', - 'wheel', ] docs_extras = [ @@ -87,16 +84,22 @@ docs_extras = [ 'sphinx_rtd_theme', ] +# Tools like pip, wheel, and tox are listed here to ensure they are properly +# pinned and installed during automated testing. test_extras = [ 'coverage', 'mypy', + 'pip', 'pylint', 'pytest', 'pytest-cov', 'pytest-xdist', + 'setuptools', + 'tox', # typing-extensions is required to import typing.Protocol and make the mypy checks # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7 'typing-extensions', + 'wheel', ] diff --git a/pytest.ini b/pytest.ini index 94122db30..1eebe5b65 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,10 @@ # The current warnings being ignored are: # 1) The warning raised when importing certbot.tests.util and the external mock # library is installed. +# 2) An ImportWarning is raised with older versions of setuptools and +# zope.interface. See +# https://github.com/zopefoundation/zope.interface/issues/68 for more info. filterwarnings = error ignore:The external mock module:PendingDeprecationWarning + ignore:.*zope. missing __init__:ImportWarning diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt deleted file mode 100644 index 50b52222d..000000000 --- a/tools/dev_constraints.txt +++ /dev/null @@ -1,131 +0,0 @@ -# Specifies extra Python package versions during our tests with the oldest -# supported versions of our dependencies. Some dev package versions specified -# here may be overridden by higher level constraints files during tests (eg. -# tools/oldest_constraints.txt). -alabaster==0.7.10 -apacheconfig==0.3.2 -apipkg==1.4 -appnope==0.1.0 -asn1crypto==0.22.0 -astroid==2.3.3 -attrs==17.3.0 -azure-devops==6.0.0b2 -Babel==2.5.1 -backcall==0.2.0 -backports.functools-lru-cache==1.5 -backports.shutil-get-terminal-size==1.0.0 -backports.ssl-match-hostname==3.7.0.1 -bcrypt==3.1.6 -boto3==1.17.4 -botocore==1.20.4 -cached-property==1.5.1 -cloudflare==2.3.1 -configparser==3.7.4 -contextlib2==0.6.0.post1 -coverage==4.5.4 -decorator==4.4.1 -deprecated==1.2.10 -dns-lexicon==3.3.17 -dnspython==2.1.0 -docker==4.3.1 -docker-compose==1.26.2 -docker-pycreds==0.4.0 -dockerpty==0.4.1 -docopt==0.6.2 -docutils==0.15.2 -execnet==1.5.0 -functools32==3.2.3.post2 -future==0.16.0 -futures==3.3.0 -filelock==3.0.12 -google-api-python-client==1.5.5 -httplib2==0.18.1 -imagesize==0.7.1 -importlib-metadata==0.23 -ipdb==0.12.3 -ipython==7.9.0 -ipython-genutils==0.2.0 -isodate==0.6.0 -isort==4.3.21 -jedi==0.17.1 -Jinja2==2.9.6 -jmespath==0.9.4 -josepy==1.1.0 -jsonpickle==2.0.0 -jsonschema==2.6.0 -lazy-object-proxy==1.4.3 -logger==1.4 -logilab-common==1.4.1 -MarkupSafe==1.1.1 -mccabe==0.6.1 -more-itertools==5.0.0 -msrest==0.6.18 -mypy==0.812 -mypy-extensions==0.4.3 -ndg-httpsclient==0.3.2 -oauth2client==4.0.0 -oauthlib==3.1.0 -packaging==19.2 -paramiko==2.4.2 -parso==0.7.0 -pathlib2==2.3.5 -pexpect==4.7.0 -pickleshare==0.7.5 -pip==20.2.4 -pkginfo==1.4.2 -pluggy==0.13.0 -ply==3.4 -prompt-toolkit==2.0.10 -ptyprocess==0.6.0 -py==1.8.0 -pyasn1==0.1.9 -pyasn1-modules==0.0.10 -PyGithub==1.52 -Pygments==2.2.0 -pyjwt==1.7.1 -pylint==2.4.3 -pynacl==1.3.0 -# If pynsist version is upgraded, our NSIS template windows-installer/template.nsi -# must be upgraded if necessary using the new built-in one from pynsist. -pynsist==2.7 -pytest==3.2.5 -pytest-cov==2.5.1 -pytest-forked==0.2 -pytest-xdist==1.22.5 -pytest-sugar==0.9.2 -pytest-rerunfailures==4.2 -python-dateutil==2.8.1 -python-digitalocean==1.15.0 -python-dotenv==0.14.0 -pywin32==300 -PyYAML==5.3.1 -repoze.sphinx.autointerface==0.8 -requests-file==1.4.2 -requests-oauthlib==1.3.0 -requests-toolbelt==0.8.0 -rsa==3.4.2 -s3transfer==0.3.1 -scandir==1.10.0 -simplegeneric==0.8.1 -singledispatch==3.4.0.3 -snowballstemmer==1.2.1 -Sphinx==1.7.5 -sphinx-rtd-theme==0.2.4 -sphinxcontrib-websupport==1.0.1 -texttable==0.9.1 -tldextract==2.2.0 -toml==0.10.0 -tox==3.14.0 -tqdm==4.19.4 -traitlets==4.3.3 -twine==1.11.0 -typed-ast==1.4.1 -typing==3.6.4 -typing-extensions==3.7.4.3 -uritemplate==3.0.0 -virtualenv==16.6.2 -wcwidth==0.1.8 -websocket-client==0.56.0 -wheel==0.35.1 -wrapt==1.11.2 -zipp==0.6.0 diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py deleted file mode 100755 index 44a61249b..000000000 --- a/tools/merge_requirements.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -"""Merges multiple Python requirements files into one file. - -Requirements files specified later take precedence over earlier ones. -Only the simple formats SomeProject==1.2.3 or SomeProject<=1.2.3 are -currently supported. - -""" -import sys - - -def process_entries(entries): - """ - Ignore empty lines, comments and editable requirements - - :param list entries: List of entries - - :returns: mapping from a project to its version specifier - :rtype: dict - """ - data = {} - for e in entries: - e = e.strip() - if e and not e.startswith('#') and not e.startswith('-e'): - # Support for <= was added as part of - # https://github.com/certbot/certbot/pull/8460 because we weren't - # able to pin a package to an exact version. Normally, this - # functionality shouldn't be needed so we could remove it in the - # future. If you do so, make sure to update other places in this - # file related to this behavior such as this file's docstring. - for comparison in ('==', '<=',): - parts = e.split(comparison) - if len(parts) == 2: - project_name = parts[0] - version = parts[1] - data[project_name] = comparison + version - break - else: - raise ValueError("Unexpected syntax '{0}'".format(e)) - return data - -def read_file(file_path): - """Reads in a Python requirements file. - - :param str file_path: path to requirements file - - :returns: list of entries in the file - :rtype: list - - """ - with open(file_path) as file_h: - return file_h.readlines() - -def output_requirements(requirements): - """Prepare print requirements to stdout. - - :param dict requirements: mapping from a project to its version - specifier - - """ - return '\n'.join('{0}{1}'.format(key, value) - for key, value in sorted(requirements.items())) - - -def main(*paths): - """Merges multiple requirements files together and prints the result. - - Requirement files specified later in the list take precedence over earlier - files. Files are read from file paths passed from the command line arguments. - - If no command line arguments are defined, data is read from stdin instead. - - :param tuple paths: paths to requirements files provided on command line - - """ - data = {} - if paths: - for path in paths: - data.update(process_entries(read_file(path))) - else: - # Need to check if interactive to avoid blocking if nothing is piped - if not sys.stdin.isatty(): - stdin_data = [] - for line in sys.stdin: - stdin_data.append(line) - data.update(process_entries(stdin_data)) - - return output_requirements(data) - - -if __name__ == '__main__': - print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index c31c5d4f8..d9ecf8c30 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -1,77 +1,100 @@ -# This file contains the oldest versions of our dependencies we're trying to -# support. Usually these version numbers are taken from the packages of our -# dependencies available in popular LTS Linux distros. Keeping compatibility -# with those versions makes it much easier for OS maintainers to update their -# Certbot packages. -# -# When updating these dependencies, we should try to only update them to the -# oldest version of the package that is found in a non-EOL'd version of -# CentOS, Debian, or Ubuntu that has Certbot packages in their OS repositories -# using a version of Python we support. If the distro is EOL'd or using a -# version of Python we don't support, it can be ignored. - -# CentOS/RHEL 7 EPEL constraints -# Some of these constraints may be stricter than necessary because they -# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL. -cffi==1.9.1 -chardet==2.2.1 -ipaddress==1.0.16 -mock==1.0.1 -ndg-httpsclient==0.3.2 -ply==3.4 -pyOpenSSL==17.3.0 -pyasn1==0.1.9 -pycparser==2.14 -pyRFC3339==1.0 -python-augeas==0.5.0 -oauth2client==4.0.0 -urllib3==1.10.2 -zope.component==4.1.0 -zope.event==4.0.3 -zope.interface==4.0.5 - -# Debian Jessie Backports constraints -# Debian Jessie has reached end of life so these dependencies can probably be -# updated as needed or desired. -colorama==0.3.2 -enum34==1.0.3 -html5lib==0.999 -pbr==1.8.0 -pytz==2012rc0 - -# Debian Buster constraints -google-api-python-client==1.5.5 -pyparsing==2.2.0 - -# Our setup.py constraints +# This file was generated by tools/pinning/oldest/repin.sh and can be updated using +# that script. apacheconfig==0.3.2 -cloudflare==1.5.1 -python-digitalocean==1.11 -requests==2.6.0 - -# Ubuntu Xenial constraints -# Ubuntu Xenial only has versions of Python which we do not support available -# so these dependencies can probably be updated as needed or desired. -ConfigArgParse==0.10.0 -funcsigs==0.4 -zope.hookable==4.0.4 - -# Ubuntu Bionic constraints. -cryptography==2.1.4 -distro==1.0.1 -httplib2==0.9.2 -idna==2.6 -setuptools==39.0.1 -six==1.11.0 - -# Ubuntu Focal constraints asn1crypto==0.24.0 -configobj==5.0.6 -parsedatetime==2.4 - -# Plugin constraints -# These aren't necessarily the oldest versions we need to support -# Tracking at https://github.com/certbot/certbot/issues/6473 +astroid==2.5.6; python_version >= "3.6" and python_version < "4.0" +atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" +attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" boto3==1.4.7 botocore==1.7.41 -dns-lexicon==3.1.0 +cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cffi==1.9.1 +chardet==2.2.1 +cloudflare==1.5.1 +colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or python_full_version >= "3.5.0" and python_version >= "3.6" and sys_platform == "win32" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.5.0" +configargparse==0.10.0 +configobj==5.0.6 +coverage==5.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" +cryptography==2.1.4 +cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +distro==1.0.1 +dns-lexicon==3.2.1 +dnspython==2.1.0; python_version >= "3.6" +docker-compose==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker-pycreds==0.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker==3.7.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docutils==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +filelock==3.0.12; python_version >= "3.6" +funcsigs==0.4 +future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" +google-api-python-client==1.5.5 +httplib2==0.9.2 +idna==2.6 +importlib-metadata==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.4.0" and python_version >= "3.6" and python_version < "3.8" or python_version < "3.8" and python_version >= "3.6" +iniconfig==1.1.1; python_version >= "3.6" +ipaddress==1.0.16 +isort==5.8.0; python_version >= "3.6" and python_version < "4.0" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" +josepy==1.8.0; python_version >= "3.6" +jsonschema==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +lazy-object-proxy==1.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0" +logger==1.4; python_version >= "3.6" +mccabe==0.6.1; python_version >= "3.6" and python_version < "4.0" +mock==1.0.1 +mypy-extensions==0.4.3; python_version >= "3.6" +mypy==0.910; python_version >= "3.6" +ndg-httpsclient==0.3.2 +oauth2client==4.0.0 +packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +paramiko==2.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +parsedatetime==2.4 +pbr==1.8.0 +pip==20.2.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +ply==3.4 +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyasn1-modules==0.0.10; python_version >= "3.6" +pyasn1==0.1.9 +pycparser==2.14 +pylint==2.8.3; python_version >= "3.6" and python_version < "4.0" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pyopenssl==17.3.0 +pyparsing==2.2.0 +pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +pyrfc3339==1.0 +pytest-cov==2.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pytest-xdist==2.3.0; python_version >= "3.6" or python_version >= "3.6" +pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +python-augeas==0.5.0 +python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" +python-digitalocean==1.11 +pytz==2012c +pywin32==301; sys_platform == "win32" and python_version >= "3.6" or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +pyyaml==3.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6" +requests-file==1.5.1; python_version >= "3.6" +requests-toolbelt==0.9.1; python_version >= "3.6" +requests==2.14.2 +rsa==4.7.2; python_version >= "3.6" and python_version < "4" +s3transfer==0.1.13; python_version >= "3.6" +setuptools==39.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +six==1.11.0 +texttable==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +tldextract==3.1.0; python_version >= "3.6" +toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" +typed-ast==1.4.3; python_version >= "3.6" and python_version < "3.8" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" +typing-extensions==3.10.0.0; python_version >= "3.6" or python_version < "3.8" and python_version >= "3.6" +uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +urllib3==1.10.2 +websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +wheel==0.33.6; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") +wrapt==1.12.1; python_version >= "3.6" and python_version < "4.0" +zipp==3.4.1; python_version < "3.8" and python_version >= "3.6" +zope.component==4.1.0 +zope.event==4.0.3 +zope.hookable==4.0.4 +zope.interface==4.0.5 diff --git a/tools/pinning/DESIGN.md b/tools/pinning/DESIGN.md new file mode 100644 index 000000000..946eb927e --- /dev/null +++ b/tools/pinning/DESIGN.md @@ -0,0 +1,125 @@ +# Certbot dependency pinning + +As described in the developer guide, we try to pin Certbot's dependencies to +well tested versions in almost all cases. Pinning Python dependencies across +different environments like this is actually quite tricky though and the files +under this directory make a somewhat reasonable, best effort approach to solve +this problem. + +## Python packaging background + +### Sdists and wheels + +Python projects are most commonly distributed on [PyPI](https://pypi.org/) as +either a [source +distribution](https://packaging.python.org/glossary/#term-Source-Distribution-or-sdist) +or a [wheel](https://packaging.python.org/glossary/#term-Wheel). Wheels don't +present a problem for us pinning dependencies because they offer a well defined +format where the dependencies of a package can be easily parsed. + +Source distributions or "sdists" are a problem though because they can contain +arbitrary Python code that must be run to determine the dependencies of the +package. This code could theoretically do anything, but it most commonly +inspects the environment it is running in and changes the project's +dependencies based the environment. As of writing this, we even do this in some +of Certbot's packages as you can see +[here](https://github.com/certbot/certbot/blob/8b610239bfcf7aac06f6e36d09a5abba3ba87047/certbot-dns-cloudflare/setup.py#L15-L27). +This is a problem because it means the environment an sdist was run in affects +the dependencies it declares making it difficult for us to determine Certbot's +dependencies for an arbitrary environment. Luckily, this is becoming less and +less of an issue with the increasing use of wheels, however, as of writing +this, some of Certbot's dependencies are still only available as sdists on some +platforms. + +### Environment markers and pyproject.toml + +Two other things have helped reduce the problems caused by sdists and are +relevant here. The first is the usage of [environment +markers](https://www.python.org/dev/peps/pep-0496/) which allows a package to +consistently declare its conditional dependencies with a static string +specifying the conditions where a dependency is required instead of dynamically +generating the list of required dependencies at runtime. This static string +keeps the package's declaration of its dependencies consistent across +environments. + +The other relatively recent change in Python packaging is the adoption of +[pyproject.toml files](https://www.python.org/dev/peps/pep-0518/) which allows +sdists to define their packages using a static file instead of a setup.py +file, which has historically been the norm. +Using a static file instead of arbitrary Python code makes it +much easier for package declarations to be reliably interpreted. The +introduction of pyproject.toml also allows for the use of build systems other +than setuptools which becomes relevant in the next section of this doc. + +## Our pinning system + +### Overview + +The files inside `tools/pinning` are used to generate Certbot's pinning files. +The files under `oldest` are used to generate the constraints file used for our +"oldest" tests while `current` is used to generate the constraints used +everywhere else. `common` includes shared files that are used for both sets of +pinnings. + +Under `current` and `oldest`, there are two files as of writing this. One is a +pyproject.toml file for use with [Poetry](https://python-poetry.org/) while +the other is a script that can be run to regenerate pinnings. The +pyproject.toml file defines a Python package that depends on everything we want +to pin. This file largely just depends on our own local packages, however, +extra dependencies can be declared to further constrain package versions or to +declare additional dependencies. + +The reason we use Poetry is that it is somewhat unique among Python packaging +tools in that when locking dependencies, it makes a best effort approach to do +this for all environments rather than just the current environment. This +includes recursively resolving dependencies declared through environment +markers that are not relevant for the current platform. It also includes +checking all wheels and sdists of a package for dependencies when picking a +specific version of a package from PyPI. You can see this in action through +the inclusion of dependencies like pywin32 which we only have a dependency on +for Windows. + +### Potential problems + +As of writing this, I'm aware of two potential problems with this pinning +system. The first is largely described earlier in the doc which is the problem +of sdists that use code to dynamically declare its dependencies. It's simply +not feasible to ensure this arbitrary Python code declares its dependencies in +the same way across all environments. Luckily, this is a largely a theoretical +problem and I'm aware of no issues with our current dependencies. + +The second problem with this approach is that build dependencies are not +tracked and pinned. Unfortunately, [this seems to be a largely unsolved problem +in Python right +now](https://discuss.python.org/t/pinning-build-dependencies/8363). Our tooling +ensures that when installing build dependencies when using our pinning files +that versions from the pinning files are used (see +https://github.com/certbot/certbot/pull/8443 for more info about that), but I'm +not aware of any tool that automates the process of tracking and pinning down +build dependencies. For now, if we find any unpinned build dependencies, we can +declare a dependency on them in pyproject.toml. If a build dependency isn't +included in the constraints file, pip will use the latest version available on +PyPI. + +## Theoretical future work + +I think the system described above should work pretty well and I think it's +much better than the system we had before where how to update things like our +"oldest" pinnings was an open question. If we wanted to improve on this in the +future though, I think things to consider would be: + +1. We could require that wheels are used for all of our dependencies. If a + wheel is not available for one of our dependencies, we could try to work + with upstream to change that or build it and host it locally for ourselves. + (If we do the latter, how to pin build dependencies when building the wheel + remains an open question.) +2. We could only really try to pin our dependencies for certain environments. + This would be done by doing something like installing our packages in each + environment we care about and saving the output of a command like `pip + freeze`. With our use of snaps and Docker, this may be somewhat reasonable + because we could base them all on a common system like Ubuntu LTS, however, + it's not entirely trivial because we still have problems such as supporting + multiple CPU architectures and pinning dependencies for Windows. Alternative + development and test environments also wouldn't be fully supported. +3. We could help build better tooling that solves some of the problems with + this approach or adopts it when it becomes available. diff --git a/tools/pinning/common/export-pinned-dependencies.sh b/tools/pinning/common/export-pinned-dependencies.sh new file mode 100755 index 000000000..69c55fbbf --- /dev/null +++ b/tools/pinning/common/export-pinned-dependencies.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# This script accepts a directory containing a pyproject.toml file configured +# for use with poetry and generates and prints the pinned dependencies of that +# file. Any dependencies on acme or those referencing certbot will be removed +# from the output. The exported requirements are printed to stdout. +# +# For example, if a directory containing a pyproject.toml file for poetry is at +# ../current, you could activate Certbot's developer environment and then run a +# command like the following to generate requirements.txt for that environment: +# ./export-pinned-dependencies.sh ../current > requirements.txt +set -euo pipefail + +# If this script wasn't given a command line argument, print usage and exit. +if [ -z ${1+x} ]; then + echo "Usage:" >&2 + echo "$0 PYPROJECT_TOML_DIRECTORY" >&2 + exit 1 +fi + +REPO_ROOT="$(git rev-parse --show-toplevel)" +WORK_DIR="$1" + +if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then + echo "Please install poetry 1.2+." >&2 + echo "You may need to recreate Certbot's virtual environment and activate it." >&2 + exit 1 +fi + +# Old eggs can cause outdated dependency information to be used by poetry so we +# delete them before generating the lock file. See +# https://github.com/python-poetry/poetry/issues/4103 for more info. +rm -rf ${REPO_ROOT}/*.egg-info + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock >&2 +trap 'rm poetry.lock' EXIT + +# We need to remove local packages from the output. +poetry export --without-hashes | sed '/^acme @/d; /certbot/d;' diff --git a/tools/pinning/pyproject.toml b/tools/pinning/current/pyproject.toml similarity index 71% rename from tools/pinning/pyproject.toml rename to tools/pinning/current/pyproject.toml index 580922c4e..c29eeb539 100644 --- a/tools/pinning/pyproject.toml +++ b/tools/pinning/current/pyproject.toml @@ -12,28 +12,28 @@ python = "^3.6" # Any local packages that have dependencies on other local packages must be # listed below before the package it depends on. For instance, certbot depends # on acme so certbot must be listed before acme. -certbot-ci = {path = "../../certbot-ci"} -certbot-compatibility-test = {path = "../../certbot-compatibility-test"} -certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]} -certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]} -certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]} -certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]} -certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]} -certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]} -certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]} -certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]} -certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]} -certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]} -certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]} -certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]} -certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]} -certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]} -certbot-nginx = {path = "../../certbot-nginx"} -certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} -certbot = {path = "../../certbot", extras = ["all"]} -acme = {path = "../../acme", extras = ["docs", "test"]} -letstest = {path = "../../letstest"} -windows-installer = {path = "../../windows-installer"} +certbot-ci = {path = "../../../certbot-ci"} +certbot-compatibility-test = {path = "../../../certbot-compatibility-test"} +certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare", extras = ["docs"]} +certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns", extras = ["docs"]} +certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean", extras = ["docs"]} +certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple", extras = ["docs"]} +certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy", extras = ["docs"]} +certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn", extras = ["docs"]} +certbot-dns-google = {path = "../../../certbot-dns-google", extras = ["docs"]} +certbot-dns-linode = {path = "../../../certbot-dns-linode", extras = ["docs"]} +certbot-dns-luadns = {path = "../../../certbot-dns-luadns", extras = ["docs"]} +certbot-dns-nsone = {path = "../../../certbot-dns-nsone", extras = ["docs"]} +certbot-dns-ovh = {path = "../../../certbot-dns-ovh", extras = ["docs"]} +certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136", extras = ["docs"]} +certbot-dns-route53 = {path = "../../../certbot-dns-route53", extras = ["docs"]} +certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud", extras = ["docs"]} +certbot-nginx = {path = "../../../certbot-nginx"} +certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../../certbot", extras = ["all"]} +acme = {path = "../../../acme", extras = ["docs", "test"]} +letstest = {path = "../../../letstest"} +windows-installer = {path = "../../../windows-installer"} # Extra dependencies # awscli is just listed here as a performance optimization. As of writing this, diff --git a/tools/pinning/current/repin.sh b/tools/pinning/current/repin.sh new file mode 100755 index 000000000..c8ec8e87f --- /dev/null +++ b/tools/pinning/current/repin.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies including automatically updating the correct file. +# Dependencies can be pinned to older versions by modifying pyproject.toml in +# the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +COMMON_DIR="$(dirname "${WORK_DIR}")/common" +REPO_ROOT="$(git rev-parse --show-toplevel)" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" + +PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}") +cat << EOF > "$REQUIREMENTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +EOF +echo "${PINNINGS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pinning/oldest/pyproject.toml b/tools/pinning/oldest/pyproject.toml new file mode 100644 index 000000000..ba9cf1972 --- /dev/null +++ b/tools/pinning/oldest/pyproject.toml @@ -0,0 +1,145 @@ +[tool.poetry] +name = "certbot-pinner" +version = "0.1.0" +description = "A simple project for pinning Certbot's dependencies using Poetry." +authors = ["Certbot Project"] +license = "Apache License 2.0" + +[tool.poetry.dependencies] +# The Python version here should be kept in sync with the one used in our +# oldest tests in tox.ini. +python = "3.6" + +# Local dependencies +# Any local packages that have dependencies on other local packages must be +# listed below before the package it depends on. For instance, certbot depends +# on acme so certbot must be listed before acme. +certbot-ci = {path = "../../../certbot-ci"} +certbot-dns-cloudflare = {path = "../../../certbot-dns-cloudflare"} +certbot-dns-cloudxns = {path = "../../../certbot-dns-cloudxns"} +certbot-dns-digitalocean = {path = "../../../certbot-dns-digitalocean"} +certbot-dns-dnsimple = {path = "../../../certbot-dns-dnsimple"} +certbot-dns-dnsmadeeasy = {path = "../../../certbot-dns-dnsmadeeasy"} +certbot-dns-gehirn = {path = "../../../certbot-dns-gehirn"} +certbot-dns-google = {path = "../../../certbot-dns-google"} +certbot-dns-linode = {path = "../../../certbot-dns-linode"} +certbot-dns-luadns = {path = "../../../certbot-dns-luadns"} +certbot-dns-nsone = {path = "../../../certbot-dns-nsone"} +certbot-dns-ovh = {path = "../../../certbot-dns-ovh"} +certbot-dns-rfc2136 = {path = "../../../certbot-dns-rfc2136"} +certbot-dns-route53 = {path = "../../../certbot-dns-route53"} +certbot-dns-sakuracloud = {path = "../../../certbot-dns-sakuracloud"} +certbot-nginx = {path = "../../../certbot-nginx"} +certbot-apache = {path = "../../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../../certbot", extras = ["test"]} +acme = {path = "../../../acme", extras = ["test"]} + +# Oldest dependencies +# We specify the oldest versions our dependencies that we're trying to keep +# support for below. Usually these version numbers are taken from the packages +# of our dependencies available in popular LTS Linux distros. Keeping +# compatibility with those versions makes it much easier for OS maintainers to +# update their Certbot packages. +# +# When updating these dependencies, we should ideally try to only update them +# to the oldest version of the dependency that is found in a non-EOL'd version +# of CentOS, Debian, or Ubuntu that has Certbot packages in their OS +# repositories using a version of Python we support. If the distro is EOL'd or +# using a version of Python we don't support, it can be ignored. If the +# dependency being updated is a direct dependency of one of our own packages, +# the minimum required version of that dependency should be updated in our +# setup.py files as well to communicate this information to our users. + +# CentOS/RHEL 7 EPEL dependencies +# Some of these dependencies may be stricter than necessary because they +# initially referred to the Python 2 packages in CentOS/RHEL 7 with EPEL. +cffi = "1.9.1" +chardet = "2.2.1" +ipaddress = "1.0.16" +mock = "1.0.1" +ndg-httpsclient = "0.3.2" +ply = "3.4" +pyOpenSSL = "17.3.0" +pyasn1 = "0.1.9" +pycparser = "2.14" +pyRFC3339 = "1.0" +python-augeas = "0.5.0" +oauth2client = "4.0.0" +requests = "2.14.2" +urllib3 = "1.10.2" +# Package names containing "." need to be quoted. +"zope.component" = "4.1.0" +"zope.event" = "4.0.3" +"zope.interface" = "4.0.5" + +# Debian Jessie Backports dependencies +# Debian Jessie has reached end of life so these dependencies can probably be +# updated as needed or desired. +pbr = "1.8.0" +pytz = "2012rc0" + +# Debian Buster dependencies +google-api-python-client = "1.5.5" +pyparsing = "2.2.0" + +# Our setup.py dependencies +apacheconfig = "0.3.2" +cloudflare = "1.5.1" +python-digitalocean = "1.11" + +# Ubuntu Xenial dependencies +# Ubuntu Xenial only has versions of Python which we do not support available +# so these dependencies can probably be updated as needed or desired. +ConfigArgParse = "0.10.0" +funcsigs = "0.4" +# Package names containing "." need to be quoted. +"zope.hookable" = "4.0.4" + +# Ubuntu Bionic dependencies. +cryptography = "2.1.4" +distro = "1.0.1" +httplib2 = "0.9.2" +idna = "2.6" +setuptools = "39.0.1" +six = "1.11.0" + +# Ubuntu Focal dependencies +asn1crypto = "0.24.0" +configobj = "5.0.6" +parsedatetime = "2.4" + +# Plugin dependencies +# These aren't necessarily the oldest versions we need to support +# Tracking at https://github.com/certbot/certbot/issues/6473 +boto3 = "1.4.7" +botocore = "1.7.41" +dns-lexicon = "3.2.1" + +# Build dependencies +# Since there doesn't appear to +# doesn't appear to be a good way to automatically track down and pin build +# dependencies in Python (see +# https://discuss.python.org/t/how-to-pin-build-dependencies/8238), we list any +# build dependencies here to ensure they're pinned for extra stability. + +# cython is a build dependency of pyyaml +cython = "*" + +# Other dependencies +# We add any dependencies that must be specified in this file for any another +# reason below. + +# pip's new dependency resolver fails on local packages that depend on each +# other when those packages are requested with extras such as 'certbot[dev]' so +# let's pin it back for now. See https://github.com/pypa/pip/issues/9204. +pip = "20.2.4" + +# wheel 0.34.1+ does not support the version of setuptools pinned above (and +# wheel 0.34.0 is buggy). +wheel = "<0.34.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/pinning/oldest/repin.sh b/tools/pinning/oldest/repin.sh new file mode 100755 index 000000000..e2b685a22 --- /dev/null +++ b/tools/pinning/oldest/repin.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies including automatically updating the correct file. +# Dependencies can be pinned to older versions by modifying pyproject.toml in +# the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +COMMON_DIR="$(dirname "${WORK_DIR}")/common" +REPO_ROOT="$(git rev-parse --show-toplevel)" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +CONSTRAINTS_FILE="$REPO_ROOT/tools/oldest_constraints.txt" + +PINNINGS=$("${COMMON_DIR}/export-pinned-dependencies.sh" "${WORK_DIR}") +cat << EOF > "$CONSTRAINTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +EOF +echo "${PINNINGS}" >> "${CONSTRAINTS_FILE}" diff --git a/tools/pinning/pin.sh b/tools/pinning/pin.sh deleted file mode 100755 index 1c8111e1b..000000000 --- a/tools/pinning/pin.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# This script accepts no arguments and automates the process of updating -# Certbot's dependencies. Dependencies can be pinned to older versions by -# modifying pyproject.toml in the same directory as this file. -set -euo pipefail - -WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" -RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" -REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" - -if ! command -v poetry >/dev/null || [ $(poetry --version | grep -oE '[0-9]+\.[0-9]+' | sed 's/\.//') -lt 12 ]; then - echo "Please install poetry 1.2+." - echo "You may need to recreate Certbot's virtual environment and activate it." - exit 1 -fi - -# Old eggs can cause outdated dependency information to be used by poetry so we -# delete them before generating the lock file. See -# https://github.com/python-poetry/poetry/issues/4103 for more info. -cd "${REPO_ROOT}" -rm -rf */*.egg-info - -cd "${WORK_DIR}" - -if [ -f poetry.lock ]; then - rm poetry.lock -fi - -poetry lock - -TEMP_REQUIREMENTS=$(mktemp) -trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT - -poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes -# We need to remove local packages from the requirements file. -sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" - -cat << EOF > "$REQUIREMENTS_FILE" -# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using -# that script. -# -# It is normally used as constraints to pip, however, it has the name -# requirements.txt so that is scanned by GitHub. See -# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems -# for more info. -EOF -cat "${TEMP_REQUIREMENTS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pip_install.py b/tools/pip_install.py index aa1d06a80..6cd7af680 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -1,54 +1,23 @@ #!/usr/bin/env python # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt and -# tools/dev_constraints.txt is used, otherwise, tools/requirements.txt is used. +# to 1, tools/oldest_constraints.txt is used, otherwise, tools/requirements.txt +# is used. from __future__ import absolute_import from __future__ import print_function -import contextlib import os -import re -import shutil import subprocess import sys import tempfile -import merge_requirements as merge_module import readlink -# Once this code doesn't need to support Python 2, we can simply use -# tempfile.TemporaryDirectory. -@contextlib.contextmanager -def temporary_directory(): - dirpath = tempfile.mkdtemp() - try: - yield dirpath - finally: - shutil.rmtree(dirpath) - - def find_tools_path(): return os.path.dirname(readlink.main(__file__)) -def certbot_oldest_processing(tools_path, constraints_path): - # The order of the files in this list matters as files specified later can - # override the pinnings found in earlier files. - pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'), - os.path.join(tools_path, 'oldest_constraints.txt')] - with open(constraints_path, 'w') as fd: - fd.write(merge_module.main(*pinning_files)) - - -def certbot_normal_processing(tools_path, constraints_path): - repo_path = os.path.dirname(tools_path) - requirements = os.path.normpath(os.path.join( - repo_path, 'tools/requirements.txt')) - shutil.copy(requirements, constraints_path) - - def call_with_print(command, env=None): if not env: env = os.environ @@ -66,17 +35,20 @@ def pip_install_with_print(args_str, env=None): def main(args): tools_path = find_tools_path() - with temporary_directory() as working_dir: + with tempfile.TemporaryDirectory() as working_dir: if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint pip_install_with_print(' '.join(args)) else: - # Otherwise, we merge requirements to build the constraints and pin dependencies - constraints_path = os.path.join(working_dir, 'constraints.txt') + # Otherwise, we pick the constraints file based on the environment + # variable CERTBOT_OLDEST. + repo_path = os.path.dirname(tools_path) if os.environ.get('CERTBOT_OLDEST') == '1': - certbot_oldest_processing(tools_path, constraints_path) + constraints_path = os.path.normpath(os.path.join( + repo_path, 'tools', 'oldest_constraints.txt')) else: - certbot_normal_processing(tools_path, constraints_path) + constraints_path = os.path.normpath(os.path.join( + repo_path, 'tools', 'requirements.txt')) env = os.environ.copy() env["PIP_CONSTRAINT"] = constraints_path diff --git a/tools/requirements.txt b/tools/requirements.txt index 8370ea0a6..e12072652 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,4 +1,4 @@ -# This file was generated by tools/pinning/pin.sh and can be updated using +# This file was generated by tools/pinning/current/repin.sh and can be updated using # that script. # # It is normally used as constraints to pip, however, it has the name @@ -12,31 +12,31 @@ appnope==0.1.2; python_version == "3.6" and sys_platform == "darwin" or python_v astroid==2.6.2; python_version >= "3.6" and python_version < "4.0" atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -awscli==1.19.109; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +awscli==1.20.1; python_version >= "3.6" azure-devops==6.0.0b4; python_version >= "3.6" babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" backcall==0.2.0; python_version == "3.6" or python_version >= "3.7" bcrypt==3.2.0; python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" -bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.17.109; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -botocore==1.20.109; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +bleach==3.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +boto3==1.18.1; python_version >= "3.6" +botocore==1.21.1; python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" -certifi==2021.5.30; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" +certifi==2021.5.30; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" cffi==1.14.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" -chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +charset-normalizer==2.0.3; python_full_version >= "3.6.0" and python_version >= "3.6" cleo==1.0.0a3; python_version >= "3.6" and python_version < "4.0" cloudflare==2.8.15; python_version >= "3.6" -colorama==0.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and platform_system == "Windows" or python_version >= "3.6" and python_full_version >= "3.5.0" and platform_system == "Windows" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version == "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version == "3.6" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version >= "3.7" and sys_platform == "win32" and python_full_version >= "3.5.0" +colorama==0.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and platform_system == "Windows" or python_version >= "3.6" and python_full_version >= "3.5.0" and platform_system == "Windows" or python_version == "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version == "3.6" and sys_platform == "win32" and python_full_version >= "3.5.0" or python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or python_version >= "3.7" and sys_platform == "win32" and python_full_version >= "3.5.0" configargparse==1.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" configobj==5.0.6; python_version >= "3.6" coverage==5.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux" -cython==0.29.23; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +cython==0.29.24; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" decorator==5.0.9; python_version == "3.6" or python_version > "3.6" or python_version >= "3.5" or python_version >= "3.7" deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" @@ -48,19 +48,19 @@ docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0 docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -docutils==0.15.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.3.0" +docutils==0.15.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" entrypoints==0.3; python_version >= "3.6" and python_version < "4.0" execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" fabric==2.6.0; python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0" google-api-core==1.31.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.12.0; python_version >= "3.6" +google-api-python-client==2.13.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==1.32.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-auth==1.33.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" httplib2==0.19.1; python_version >= "3.6" -idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" +idna==3.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_version >= "3.6" and python_version < "3.8" and python_full_version >= "3.5.0" importlib-resources==5.2.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.4.0" @@ -73,9 +73,9 @@ ipython==7.25.0; python_version >= "3.7" isodate==0.6.0; python_version >= "3.6" isort==5.8.0; python_version >= "3.6" and python_version < "4.0" jedi==0.18.0; python_version == "3.6" or python_version >= "3.7" -jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux" +jeepney==0.7.0; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux" jinja2==3.0.1; python_version >= "3.6" or python_version >= "3.6" -jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" josepy==1.8.0; python_version >= "3.6" jsonlines==2.0.0; python_version >= "3.6" jsonpickle==2.0.0; python_version >= "3.6" @@ -111,7 +111,7 @@ protobuf==3.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or p ptyprocess==0.7.0; python_version >= "3.6" and python_version < "4.0" py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.5.0" pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" -pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") or python_version >= "3.6" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" or python_version >= "3.6" pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pygithub==1.55; python_version >= "3.6" pygments==2.9.0; python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" @@ -130,7 +130,7 @@ pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" pytest-xdist==2.3.0; python_version >= "3.6" or python_version >= "3.6" pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" python-augeas==0.5.0 -python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" +python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" python-digitalocean==1.16.0; python_version >= "3.6" python-dotenv==0.18.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" @@ -143,12 +143,12 @@ requests-download==0.1.2; python_version >= "3.6" requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" or python_version >= "3.6" -requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +requests==2.26.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0" rfc3986==1.5.0; python_version >= "3.6" -rsa==4.7.2; python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") or python_version >= "3.6" and python_version < "4" -s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +rsa==4.7.2; python_version >= "3.6" and python_version < "4" or python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +s3transfer==0.5.0; python_version >= "3.6" secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux" -setuptools==57.1.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" +setuptools==57.2.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" shellingham==1.4.0; python_version >= "3.6" and python_version < "4.0" six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" or python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0" snowballstemmer==2.1.0; python_version >= "3.6" @@ -161,18 +161,18 @@ sphinxcontrib-htmlhelp==2.0.0; python_version >= "3.6" sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" sphinxcontrib-serializinghtml==1.1.5; python_version >= "3.6" -texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +texttable==1.6.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0" or python_version > "3.6" and python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" tomlkit==0.7.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" -tox==3.23.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.24.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" tqdm==4.61.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" traitlets==4.3.3 twine==3.3.0; python_version >= "3.6" typed-ast==1.4.3; python_version >= "3.6" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6" typing-extensions==3.10.0.0; python_version >= "3.6" or python_version >= "3.6" and python_version < "3.8" uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -urllib3==1.26.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +urllib3==1.26.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" wcwidth==0.2.5; python_version == "3.6" webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" diff --git a/tox.ini b/tox.ini index 7bdb57cff..a0b0dfba6 100644 --- a/tox.ini +++ b/tox.ini @@ -43,6 +43,9 @@ setenv = # Setting basepython allows the tests to fail fast if that version of Python # isn't available instead of potentially trying to use a newer version of # Python which is unlikely to work. +# +# This version should be kept in sync with the one declared in +# tools/pinning/oldest/pyproject.toml. basepython = python3.6 commands = {[testenv]commands}