From 8133d3e70a6487c819d69c660620d7f2b754c952 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 29 Jul 2021 20:25:25 +0200 Subject: [PATCH] Fix python-augeas failure on Windows (v2) (#8951) This PR is a new approach for fixing #8732 based on the discussions occurred in the first PR #8877. This PR upgrades python-augeas to the latest version, and avoids tests failure of Windows because of this upgrade. To do so it leverages the [tox multi-platform feature](https://tox.readthedocs.io/en/latest/example/platform.html) and modifications to `tools/venv.py` in order to not install and not test `certbot-apache` on Windows. * Unpin python-augeas and upgrade current pinnings * Do not install certbot-apache in Windows dev environments * Introduce tox specific win packages + remove certbot compatibility on windows * Add libaugeas to sphinx build * Redefine lint and mypy targets * Keep the lint and mypy environments --- .../templates/jobs/standard-tests-jobs.yml | 8 +++--- .../templates/steps/sphinx-steps.yml | 3 +++ tools/install_and_test.py | 14 +--------- tools/pinning/current/pyproject.toml | 6 ----- tools/requirements.txt | 20 +++++++------- tools/venv.py | 2 ++ tox.ini | 27 ++++++++++++------- 7 files changed, 37 insertions(+), 43 deletions(-) diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml index fda33a71d..da914dd47 100644 --- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml +++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml @@ -15,11 +15,11 @@ jobs: windows-py36: IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.6 - TOXENV: py36 + TOXENV: py36-win windows-py38-cover: IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.8 - TOXENV: py38-cover + TOXENV: py38-cover-win windows-integration-certbot: IMAGE_NAME: vs2017-win2016 PYTHON_VERSION: 3.8 @@ -43,11 +43,11 @@ jobs: linux-py39-lint: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.9 - TOXENV: lint + TOXENV: lint-posix linux-py39-mypy: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.9 - TOXENV: mypy + TOXENV: mypy-posix linux-integration: IMAGE_NAME: ubuntu-18.04 PYTHON_VERSION: 3.8 diff --git a/.azure-pipelines/templates/steps/sphinx-steps.yml b/.azure-pipelines/templates/steps/sphinx-steps.yml index fa2952b05..e6a267e80 100644 --- a/.azure-pipelines/templates/steps/sphinx-steps.yml +++ b/.azure-pipelines/templates/steps/sphinx-steps.yml @@ -1,5 +1,8 @@ steps: - bash: | + set -e + sudo apt-get update + sudo apt-get install -y --no-install-recommends libaugeas0 FINAL_STATUS=0 declare -a FAILED_BUILDS tools/venv.py diff --git a/tools/install_and_test.py b/tools/install_and_test.py index e7a34286c..1c84de96f 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -10,9 +10,6 @@ import re import subprocess import sys -SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache'] - - def call_with_print(command): print(command) subprocess.check_call(command, shell=True) @@ -22,16 +19,7 @@ def main(args): script_dir = os.path.dirname(os.path.abspath(__file__)) command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] - new_args = [] - for arg in args: - if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: - print(( - 'Info: currently {0} is not supported on Windows and will not be tested.' - .format(arg))) - else: - new_args.append(arg) - - for requirement in new_args: + for requirement in args: current_command = command[:] current_command.append(requirement) call_with_print(' '.join(current_command)) diff --git a/tools/pinning/current/pyproject.toml b/tools/pinning/current/pyproject.toml index c29eeb539..0625c1a9a 100644 --- a/tools/pinning/current/pyproject.toml +++ b/tools/pinning/current/pyproject.toml @@ -73,12 +73,6 @@ pip = "20.2.4" # here in addition to certbot/setup.py because otherwise the pre-release # version of poetry will not be installed. poetry = ">=1.2.0a1" -# We were originally pinning back python-augeas for certbot-auto because we -# found the way older versions of the library linked to Augeas were more -# reliable. That's no longer a concern, however, we continue to pin back the -# library for now because it causes Certbot tests on Windows to fail. See -# https://github.com/certbot/certbot/issues/8732. -python-augeas = "0.5.0" # Because some parts of Certbot documentation are generated from zope.interfaces, # we need the Sphinx plugin repoze.sphinx.autointerface. This plugin is not # currently compatible with Sphinx>=4 though so we're pinning an older version diff --git a/tools/requirements.txt b/tools/requirements.txt index e12072652..21e615c1a 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -9,18 +9,18 @@ alabaster==0.7.12; python_version >= "3.6" apacheconfig==0.3.2; python_version >= "3.6" appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" appnope==0.1.2; python_version == "3.6" and sys_platform == "darwin" or python_version >= "3.7" and sys_platform == "darwin" -astroid==2.6.2; python_version >= "3.6" and python_version < "4.0" +astroid==2.6.5; python_version >= "3.6" and python_version < "4.0" atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -awscli==1.20.1; python_version >= "3.6" +awscli==1.20.5; python_version >= "3.6" azure-devops==6.0.0b4; python_version >= "3.6" babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" backcall==0.2.0; python_version == "3.6" or python_version >= "3.7" bcrypt==3.2.0; python_version >= "3.6" beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6" bleach==3.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" -boto3==1.18.1; python_version >= "3.6" -botocore==1.21.1; python_version >= "3.6" +boto3==1.18.5; python_version >= "3.6" +botocore==1.21.5; python_version >= "3.6" cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") @@ -54,9 +54,9 @@ execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or pyt fabric==2.6.0; python_version >= "3.6" filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0" google-api-core==1.31.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -google-api-python-client==2.13.0; python_version >= "3.6" +google-api-python-client==2.14.0; python_version >= "3.6" google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==1.33.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-auth==1.33.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" httplib2==0.19.1; python_version >= "3.6" @@ -117,7 +117,7 @@ pygithub==1.55; python_version >= "3.6" pygments==2.9.0; python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" pyjwt==2.1.0; python_version >= "3.6" pylev==1.4.0; python_version >= "3.6" and python_version < "4.0" -pylint==2.9.3; python_version >= "3.6" and python_version < "4.0" +pylint==2.9.5; python_version >= "3.6" and python_version < "4.0" pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" pynsist==2.7; python_version >= "3.6" pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" @@ -129,7 +129,7 @@ pytest-cov==2.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" pytest-xdist==2.3.0; python_version >= "3.6" or python_version >= "3.6" pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -python-augeas==0.5.0 +python-augeas==1.1.0; python_version >= "3.6" python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" python-digitalocean==1.16.0; python_version >= "3.6" python-dotenv==0.18.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -148,7 +148,7 @@ rfc3986==1.5.0; python_version >= "3.6" rsa==4.7.2; python_version >= "3.6" and python_version < "4" or python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") s3transfer==0.5.0; python_version >= "3.6" secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux" -setuptools==57.2.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" +setuptools==57.4.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" or python_version >= "3.6" and python_version < "4.0" shellingham==1.4.0; python_version >= "3.6" and python_version < "4.0" six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" or python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0" snowballstemmer==2.1.0; python_version >= "3.6" @@ -183,5 +183,5 @@ yarg==0.1.9; python_version >= "3.6" zipp==3.5.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_version >= "3.6" and python_version < "3.8" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.4.0" zope.component==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.hookable==5.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" diff --git a/tools/venv.py b/tools/venv.py index 8d8b01018..4be597bd1 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -49,6 +49,8 @@ REQUIREMENTS = [ if sys.platform == 'win32': REQUIREMENTS.append('-e windows-installer') + REQUIREMENTS.remove('-e certbot-apache') + REQUIREMENTS.remove('-e certbot-compatibility-test') VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') diff --git a/tox.ini b/tox.ini index a0b0dfba6..cfadbd73a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = py3-cover,lint,mypy +envlist = {py3-cover,lint,mypy}-{win,posix} [base] # pip installs the requested packages in editable mode @@ -14,23 +14,28 @@ pip_install = python {toxinidir}/tools/pip_install_editable.py # before the script moves on to the next package. All dependencies are pinned # to a specific version for increased stability for developers. install_and_test = python {toxinidir}/tools/install_and_test.py -install_packages = python {toxinidir}/tools/pip_install_editable.py {[base]all_packages} # Packages are listed on one line because tox seems to have inconsistent # behavior with substitutions that contain line continuations, see # https://github.com/tox-dev/tox/issues/2069 for more info. dns_packages = certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud -all_packages = acme[test] certbot[test] certbot-apache {[base]dns_packages} certbot-nginx +win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx +all_packages = {[base]win_all_packages} certbot-apache source_paths = acme/acme certbot/certbot certbot-ci/certbot_integration_tests certbot-apache/certbot_apache certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-cloudxns/certbot_dns_cloudxns certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx tests/lock_test.py [testenv] passenv = CERTBOT_NO_PIN +platform = + win: win32 + posix: ^(?!.*win32).*$ commands_pre = python {toxinidir}/tools/pipstrap.py commands = - !cover: {[base]install_and_test} {[base]all_packages} + !cover-win: {[base]install_and_test} {[base]win_all_packages} + !cover-!win: {[base]install_and_test} {[base]all_packages} !cover: python tests/lock_test.py - cover: {[base]install_packages} - cover: {[base]pip_install} certbot-apache[dev] + + cover-win: {[base]pip_install} {[base]win_all_packages} + cover-!win: {[base]pip_install} {[base]all_packages} certbot-apache[dev] cover: python tox.cover.py # We always recreate the virtual environment to avoid problems like # https://github.com/certbot/certbot/issues/7745. @@ -112,20 +117,22 @@ commands = python {toxinidir}/tools/pip_install.py mock {[base]install_and_test} {[base]all_packages} -[testenv:lint] +[testenv:lint{,-win,-posix}] basepython = python3 # separating into multiple invocations disables cross package # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - {[base]install_packages} + win: {[base]pip_install} {[base]win_all_packages} + !win: {[base]pip_install} {[base]all_packages} python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths} // TODO: Re-enable strict checks for optionals with appropriate type corrections or code redesign. -[testenv:mypy] +[testenv:mypy{,-win,-posix}] basepython = python3 commands = - {[base]install_packages} + win: {[base]pip_install} {[base]win_all_packages} + !win: {[base]pip_install} {[base]all_packages} mypy {[base]source_paths} [testenv:apacheconftest]