Fix python-augeas failure on Windows (v2) (#8951)

This PR is a new approach for fixing #8732 based on the discussions occurred in the first PR #8877.

This PR upgrades python-augeas to the latest version, and avoids tests failure of Windows because of this upgrade. To do so it leverages the [tox multi-platform feature](https://tox.readthedocs.io/en/latest/example/platform.html) and modifications to `tools/venv.py` in order to not install and not test `certbot-apache` on Windows.

* Unpin python-augeas and upgrade current pinnings

* Do not install certbot-apache in Windows dev environments

* Introduce tox specific win packages + remove certbot compatibility on windows

* Add libaugeas to sphinx build

* Redefine lint and mypy targets

* Keep the lint and mypy environments

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -15,11 +15,11 @@ jobs:
         windows-py36:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.6
-          TOXENV: py36
+          TOXENV: py36-win
         windows-py38-cover:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.8
-          TOXENV: py38-cover
+          TOXENV: py38-cover-win
         windows-integration-certbot:
           IMAGE_NAME: vs2017-win2016
           PYTHON_VERSION: 3.8
@@ -43,11 +43,11 @@ jobs:
         linux-py39-lint:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.9
-          TOXENV: lint
+          TOXENV: lint-posix
         linux-py39-mypy:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.9
-          TOXENV: mypy
+          TOXENV: mypy-posix
         linux-integration:
           IMAGE_NAME: ubuntu-18.04
           PYTHON_VERSION: 3.8

.azure-pipelines/templates/steps/sphinx-steps.yml

@@ -1,5 +1,8 @@
 steps:
   - bash: |
+      set -e
+      sudo apt-get update
+      sudo apt-get install -y --no-install-recommends libaugeas0
       FINAL_STATUS=0
       declare -a FAILED_BUILDS
       tools/venv.py

tools/install_and_test.py

@@ -10,9 +10,6 @@ import re
 import subprocess
 import sys
 
-SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache']
-
-
 def call_with_print(command):
     print(command)
     subprocess.check_call(command, shell=True)
@@ -22,16 +19,7 @@ def main(args):
     script_dir = os.path.dirname(os.path.abspath(__file__))
     command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
 
-    new_args = []
+    for requirement in args:
-    for arg in args:
-        if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS:
-            print((
-                'Info: currently {0} is not supported on Windows and will not be tested.'
-                .format(arg)))
-        else:
-            new_args.append(arg)
-
-    for requirement in new_args:
         current_command = command[:]
         current_command.append(requirement)
         call_with_print(' '.join(current_command))

tools/pinning/current/pyproject.toml

@@ -73,12 +73,6 @@ pip = "20.2.4"
 # here in addition to certbot/setup.py because otherwise the pre-release
 # version of poetry will not be installed.
 poetry = ">=1.2.0a1"
-# We were originally pinning back python-augeas for certbot-auto because we
-# found the way older versions of the library linked to Augeas were more
-# reliable. That's no longer a concern, however, we continue to pin back the
-# library for now because it causes Certbot tests on Windows to fail. See
-# https://github.com/certbot/certbot/issues/8732.
-python-augeas = "0.5.0"
 # Because some parts of Certbot documentation are generated from zope.interfaces,
 # we need the Sphinx plugin repoze.sphinx.autointerface. This plugin is not
 # currently compatible with Sphinx>=4 though so we're pinning an older version

tools/requirements.txt

@@ -9,18 +9,18 @@ alabaster==0.7.12; python_version >= "3.6"
 apacheconfig==0.3.2; python_version >= "3.6"
 appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 appnope==0.1.2; python_version == "3.6" and sys_platform == "darwin" or python_version >= "3.7" and sys_platform == "darwin"
-astroid==2.6.2; python_version >= "3.6" and python_version < "4.0"
+astroid==2.6.5; python_version >= "3.6" and python_version < "4.0"
 atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0"
 attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-awscli==1.20.1; python_version >= "3.6"
+awscli==1.20.5; python_version >= "3.6"
 azure-devops==6.0.0b4; python_version >= "3.6"
 babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 backcall==0.2.0; python_version == "3.6" or python_version >= "3.7"
 bcrypt==3.2.0; python_version >= "3.6"
 beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" or python_version >= "3.6"
 bleach==3.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-boto3==1.18.1; python_version >= "3.6"
+boto3==1.18.5; python_version >= "3.6"
-botocore==1.21.1; python_version >= "3.6"
+botocore==1.21.5; python_version >= "3.6"
 cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
@@ -54,9 +54,9 @@ execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or pyt
 fabric==2.6.0; python_version >= "3.6"
 filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0"
 google-api-core==1.31.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-google-api-python-client==2.13.0; python_version >= "3.6"
+google-api-python-client==2.14.0; python_version >= "3.6"
 google-auth-httplib2==0.1.0; python_version >= "3.6"
-google-auth==1.33.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-auth==1.33.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
 httplib2==0.19.1; python_version >= "3.6"
@@ -117,7 +117,7 @@ pygithub==1.55; python_version >= "3.6"
 pygments==2.9.0; python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7"
 pyjwt==2.1.0; python_version >= "3.6"
 pylev==1.4.0; python_version >= "3.6" and python_version < "4.0"
-pylint==2.9.3; python_version >= "3.6" and python_version < "4.0"
+pylint==2.9.5; python_version >= "3.6" and python_version < "4.0"
 pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 pynsist==2.7; python_version >= "3.6"
 pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
@@ -129,7 +129,7 @@ pytest-cov==2.12.1; python_version >= "3.6" and python_full_version < "3.0.0" or
 pytest-forked==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pytest-xdist==2.3.0; python_version >= "3.6" or python_version >= "3.6"
 pytest==6.2.4; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-python-augeas==0.5.0
+python-augeas==1.1.0; python_version >= "3.6"
 python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
 python-digitalocean==1.16.0; python_version >= "3.6"
 python-dotenv==0.18.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -148,7 +148,7 @@ rfc3986==1.5.0; python_version >= "3.6"
 rsa==4.7.2; python_version >= "3.6" and python_version < "4" or python_version >= "3.5" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 s3transfer==0.5.0; python_version >= "3.6"
 secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and sys_platform == "linux"
-setuptools==57.2.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7"
+setuptools==57.4.0; python_version >= "3.6" or python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version == "3.6" or python_version >= "3.7" or python_version >= "3.6" and python_version < "4.0"
 shellingham==1.4.0; python_version >= "3.6" and python_version < "4.0"
 six==1.16.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6" or python_full_version >= "3.3.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" or python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.3.0"
 snowballstemmer==2.1.0; python_version >= "3.6"
@@ -183,5 +183,5 @@ yarg==0.1.9; python_version >= "3.6"
 zipp==3.5.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_version >= "3.6" and python_version < "3.8" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.4.0"
 zope.component==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+zope.hookable==5.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 zope.interface==5.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"

tools/venv.py

@@ -49,6 +49,8 @@ REQUIREMENTS = [
 
 if sys.platform == 'win32':
     REQUIREMENTS.append('-e windows-installer')
+    REQUIREMENTS.remove('-e certbot-apache')
+    REQUIREMENTS.remove('-e certbot-compatibility-test')
 
 VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$')
 

tox.ini

@@ -4,7 +4,7 @@
 
 [tox]
 skipsdist = true
-envlist = py3-cover,lint,mypy
+envlist = {py3-cover,lint,mypy}-{win,posix}
 
 [base]
 # pip installs the requested packages in editable mode
@@ -14,23 +14,28 @@ pip_install = python {toxinidir}/tools/pip_install_editable.py
 # before the script moves on to the next package. All dependencies are pinned
 # to a specific version for increased stability for developers.
 install_and_test = python {toxinidir}/tools/install_and_test.py
-install_packages = python {toxinidir}/tools/pip_install_editable.py {[base]all_packages}
 # Packages are listed on one line because tox seems to have inconsistent
 # behavior with substitutions that contain line continuations, see
 # https://github.com/tox-dev/tox/issues/2069 for more info.
 dns_packages = certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-digitalocean certbot-dns-dnsimple certbot-dns-dnsmadeeasy certbot-dns-gehirn certbot-dns-google certbot-dns-linode certbot-dns-luadns certbot-dns-nsone certbot-dns-ovh certbot-dns-rfc2136 certbot-dns-route53 certbot-dns-sakuracloud
-all_packages = acme[test] certbot[test] certbot-apache {[base]dns_packages} certbot-nginx
+win_all_packages = acme[test] certbot[test] {[base]dns_packages} certbot-nginx
+all_packages = {[base]win_all_packages} certbot-apache
 source_paths = acme/acme certbot/certbot certbot-ci/certbot_integration_tests certbot-apache/certbot_apache certbot-compatibility-test/certbot_compatibility_test certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-cloudxns/certbot_dns_cloudxns certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-dnsimple/certbot_dns_dnsimple certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy certbot-dns-gehirn/certbot_dns_gehirn certbot-dns-google/certbot_dns_google certbot-dns-linode/certbot_dns_linode certbot-dns-luadns/certbot_dns_luadns certbot-dns-nsone/certbot_dns_nsone certbot-dns-ovh/certbot_dns_ovh certbot-dns-rfc2136/certbot_dns_rfc2136 certbot-dns-route53/certbot_dns_route53 certbot-dns-sakuracloud/certbot_dns_sakuracloud certbot-nginx/certbot_nginx tests/lock_test.py
 
 [testenv]
 passenv =
     CERTBOT_NO_PIN
+platform =
+    win: win32
+    posix: ^(?!.*win32).*$
 commands_pre = python {toxinidir}/tools/pipstrap.py
 commands =
-    !cover: {[base]install_and_test} {[base]all_packages}
+    !cover-win: {[base]install_and_test} {[base]win_all_packages}
+    !cover-!win: {[base]install_and_test} {[base]all_packages}
     !cover: python tests/lock_test.py
-    cover: {[base]install_packages}
+
-    cover: {[base]pip_install} certbot-apache[dev]
+    cover-win: {[base]pip_install} {[base]win_all_packages}
+    cover-!win: {[base]pip_install} {[base]all_packages} certbot-apache[dev]
     cover: python tox.cover.py
 # We always recreate the virtual environment to avoid problems like
 # https://github.com/certbot/certbot/issues/7745.
@@ -112,20 +117,22 @@ commands =
     python {toxinidir}/tools/pip_install.py mock
     {[base]install_and_test} {[base]all_packages}
 
-[testenv:lint]
+[testenv:lint{,-win,-posix}]
 basepython = python3
 # separating into multiple invocations disables cross package
 # duplicate code checking; if one of the commands fails, others will
 # continue, but tox return code will reflect previous error
 commands =
-    {[base]install_packages}
+    win: {[base]pip_install} {[base]win_all_packages}
+    !win: {[base]pip_install} {[base]all_packages}
     python -m pylint --reports=n --rcfile=.pylintrc {[base]source_paths}
 
 // TODO: Re-enable strict checks for optionals with appropriate type corrections or code redesign.
-[testenv:mypy]
+[testenv:mypy{,-win,-posix}]
 basepython = python3
 commands =
-    {[base]install_packages}
+    win: {[base]pip_install} {[base]win_all_packages}
+    !win: {[base]pip_install} {[base]all_packages}
     mypy {[base]source_paths}
 
 [testenv:apacheconftest]