[Windows|Unix] Rewrite bash scripts for tests into python (#6435)

Certbot relies heavily on bash scripts to deploy a development environment and to execute tests. This is fine for Linux systems, including Travis, but problematic for Windows machines.

This PR converts all theses scripts into Python, to make them platform independant.

As a consequence, tox-win.ini is not needed anymore, and tox can be run indifferently on Windows or on Linux using a common tox.ini. AppVeyor is updated accordingly to execute tests for acme, certbot and all dns plugins. Other tests are not executed as they are for Docker, unsupported Apache/Nginx/Postfix plugins (for now) or not relevant for Windows (explicit Linux distribution tests or pylint).

Another PR will be done on certbot website to update how a dev environment can be set up.

* Replace several shell scripts by python equivalent.

* Correction on tox coverage

* Extend usage of new python scripts

* Various corrections

* Replace venv construction bash scripts by python equivalents

* Update tox.ini

* Unicode lines to compare files

* Put modifications on letsencrypt-auto-source instead of generated scripts

* Add executable permissions for Linux.

* Merge tox win tests into main tox

* Skip lock_test on Windows

* Correct appveyor config

* Update appveyor.yml

* Explicit coverage py27 or py37

* Avoid to cover non supported certbot plugins on Windows

* Update tox.ini

* Remove specific warnings during CI

* No cover on a debug code for tests only.

* Update documentation and help script on venv/venv3.py

* Customize help message for Windows

* Quote correctly executable path with potential spaces in it.

* Copy pipstrap from upstream

tools/_venv_common.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import shutil
+import glob
+import time
+import subprocess
+import sys
+
+def subprocess_with_print(command):
+    print(command)
+    subprocess.call(command, shell=True)
+
+def get_venv_python(venv_path):
+    python_linux = os.path.join(venv_path, 'bin/python')
+    python_windows = os.path.join(venv_path, 'Scripts\\python.exe')
+    if os.path.isfile(python_linux):
+        return python_linux
+    if os.path.isfile(python_windows):
+        return python_windows
+
+    raise ValueError((
+        'Error, could not find python executable in venv path {0}: is it a valid venv ?'
+        .format(venv_path)))
+
+def main(venv_name, venv_args, args):
+    for path in glob.glob('*.egg-info'):
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+    if os.path.isdir(venv_name):
+        os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time())))
+
+    subprocess_with_print(' '.join([
+        sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools',
+        venv_name, venv_args]))
+
+    python_executable = get_venv_python(venv_name)
+
+    subprocess_with_print(' '.join([
+        python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')]))
+    command = [python_executable, os.path.normpath('./tools/pip_install.py')]
+    command.extend(args)
+    subprocess_with_print(' '.join(command))
+
+    if os.path.isdir(os.path.join(venv_name, 'bin')):
+        # Linux/OSX specific
+        print('-------------------------------------------------------------------')
+        print('Please run the following command to activate developer environment:')
+        print('source {0}/bin/activate'.format(venv_name))
+        print('-------------------------------------------------------------------')
+    elif os.path.isdir(os.path.join(venv_args, 'Scripts')):
+        # Windows specific
+        print('---------------------------------------------------------------------------')
+        print('Please run one of the following commands to activate developer environment:')
+        print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name))
+        print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name))
+        print('---------------------------------------------------------------------------')
+    else:
+        raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name))
+
+if __name__ == '__main__':
+    main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), sys.argv[1:])

tools/_venv_common.sh

@@ -1,26 +0,0 @@
-#!/bin/sh -xe
-
-VENV_NAME=${VENV_NAME:-venv}
-
-# .egg-info directories tend to cause bizarre problems (e.g. `pip -e
-# .` might unexpectedly install letshelp-certbot only, in case
-# `python letshelp-certbot/setup.py build` has been called
-# earlier)
-rm -rf *.egg-info
-
-# virtualenv setup is NOT idempotent: shutil.Error:
-# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and
-# `venv/bin/python2` are the same file
-mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
-virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
-. ./$VENV_NAME/bin/activate
-
-# Use pipstrap to update Python packaging tools to only update to a well tested
-# version and to work around https://github.com/pypa/pip/issues/4817 on older
-# systems.
-python letsencrypt-auto-source/pieces/pipstrap.py
-./tools/pip_install.sh "$@"
-
-set +x
-echo "Please run the following command to activate developer environment:"
-echo "source $VENV_NAME/bin/activate"

tools/install_and_test.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# pip installs the requested packages in editable mode and runs unit tests on
+# them. Each package is installed and tested in the order they are provided
+# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.py for more information on the versions pinned
+# to.
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+import shutil
+import subprocess
+import re
+
+SKIP_PROJECTS_ON_WINDOWS = [
+    'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot']
+
+def call_with_print(command, cwd=None):
+    print(command)
+    subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    if os.environ.get('CERTBOT_NO_PIN') == '1':
+        command = [sys.executable, '-m', 'pip', '-q', '-e']
+    else:
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
+
+    new_args = []
+    for arg in args:
+        if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS:
+            print((
+                'Info: currently {0} is not supported on Windows and will not be tested.'
+                .format(arg)))
+        else:
+            new_args.append(arg)
+
+    for requirement in new_args:
+        current_command = command[:]
+        current_command.append(requirement)
+        call_with_print(' '.join(current_command))
+        pkg = re.sub(r'\[\w+\]', '', requirement)
+
+        if pkg == '.':
+            pkg = 'certbot'
+
+        temp_cwd = tempfile.mkdtemp()
+        try:
+            call_with_print(' '.join([
+                sys.executable, '-m', 'pytest', '--numprocesses', 'auto',
+                '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd)
+        finally:
+            shutil.rmtree(temp_cwd)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

tools/install_and_test.sh

@@ -1,29 +0,0 @@
-#!/bin/sh -e
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using pinned versions of all of our
-# dependencies. See pip_install.sh for more information on the versions pinned
-# to.
-
-if [ "$CERTBOT_NO_PIN" = 1 ]; then
-  pip_install="pip install -q -e"
-else
-  pip_install="$(dirname $0)/pip_install_editable.sh"
-fi
-
-temp_cwd=$(mktemp -d)
-trap "rm -rf $temp_cwd" EXIT
-
-set -x
-for requirement in "$@" ; do
-  $pip_install $requirement
-  pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
-  pkg=$(echo "$pkg" | tr - _ )  # convert package names to Python import names
-  if [ $pkg = "." ]; then
-    pkg="certbot"
-  fi
-  cd "$temp_cwd"
-  pytest --numprocesses auto --quiet --pyargs $pkg
-  cd -
-done

tools/merge_requirements.py

@@ -10,7 +10,6 @@ from __future__ import print_function
 
 import sys
 
-
 def read_file(file_path):
     """Reads in a Python requirements file.
 
@@ -32,17 +31,17 @@ def read_file(file_path):
     return d
 
 
-def print_requirements(requirements):
-    """Prints requirements to stdout.
+def output_requirements(requirements):
+    """Prepare print requirements to stdout.
 
     :param dict requirements: mapping from a project to its pinned version
 
     """
-    print('\n'.join('{0}=={1}'.format(k, v)
-          for k, v in sorted(requirements.items())))
+    return '\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items()))
 
 
-def merge_requirements_files(*files):
+def main(*files):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
@@ -54,8 +53,9 @@ def merge_requirements_files(*files):
     d = {}
     for f in files:
         d.update(read_file(f))
-    print_requirements(d)
+    return output_requirements(d)
 
 
 if __name__ == '__main__':
-    merge_requirements_files(*sys.argv[1:])
+    merged_requirements = main(*sys.argv[1:])
+    print(merged_requirements)

tools/pip_install.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt,
+# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# top level of the package's directory is used, otherwise, a combination of
+# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
+# other file always takes precedence over tools/dev_constraints.txt. If
+# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
+# no other arguments.
+
+from __future__ import print_function, absolute_import
+
+import subprocess
+import os
+import sys
+import re
+import shutil
+import tempfile
+
+import merge_requirements as merge_module
+import readlink
+
+def find_tools_path():
+    return os.path.dirname(readlink.main(__file__))
+
+def certbot_oldest_processing(tools_path, args, test_constraints):
+    if args[0] != '-e' or len(args) != 2:
+        raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
+                         'with a single -e <path> argument.')
+    # remove any extras such as [dev]
+    pkg_dir = re.sub(r'\[\w+\]', '', args[1])
+    requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
+    # packages like acme don't have any local oldest requirements
+    if not os.path.isfile(requirements):
+        requirements = None
+    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
+
+    return requirements
+
+def certbot_normal_processing(tools_path, test_constraints):
+    repo_path = os.path.dirname(tools_path)
+    certbot_requirements = os.path.normpath(os.path.join(
+        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
+    with open(certbot_requirements, 'r') as fd:
+        data = fd.readlines()
+    with open(test_constraints, 'w') as fd:
+        for line in data:
+            search = re.search(r'^(\S*==\S*).*$', line)
+            if search:
+                fd.write('{0}{1}'.format(search.group(1), os.linesep))
+
+def merge_requirements(tools_path, test_constraints, all_constraints):
+    merged_requirements = merge_module.main(
+        os.path.join(tools_path, 'dev_constraints.txt'),
+        test_constraints
+    )
+    with open(all_constraints, 'w') as fd:
+        fd.write(merged_requirements)
+
+def call_with_print(command, cwd=None):
+    print(command)
+    subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    tools_path = find_tools_path()
+    working_dir = tempfile.mkdtemp()
+    try:
+        test_constraints = os.path.join(working_dir, 'test_constraints.txt')
+        all_constraints = os.path.join(working_dir, 'all_constraints.txt')
+
+        requirements = None
+        if os.environ.get('CERTBOT_OLDEST') == '1':
+            requirements = certbot_oldest_processing(tools_path, args, test_constraints)
+        else:
+            certbot_normal_processing(tools_path, test_constraints)
+
+        merge_requirements(tools_path, test_constraints, all_constraints)
+        if requirements:
+            call_with_print(' '.join([
+                sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints,
+                '--requirement', requirements]))
+
+        command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints]
+        command.extend(args)
+        call_with_print(' '.join(command))
+    finally:
+        shutil.rmtree(working_dir)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

tools/pip_install.sh

@@ -1,44 +0,0 @@
-#!/bin/sh -e
-# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
-# to 1, a combination of tools/oldest_constraints.txt,
-# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
-# top level of the package's directory is used, otherwise, a combination of
-# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
-# other file always takes precedence over tools/dev_constraints.txt. If
-# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
-# no other arguments.
-
-# get the root of the Certbot repo
-tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
-all_constraints=$(mktemp)
-test_constraints=$(mktemp)
-trap "rm -f $all_constraints $test_constraints" EXIT
-
-if [ "$CERTBOT_OLDEST" = 1 ]; then
-    if [ "$1" != "-e" -o "$#" -ne "2" ]; then
-        echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument."
-        exit 1
-    fi
-    pkg_dir=$(echo $2 | cut -f1 -d\[)  # remove any extras such as [dev]
-    requirements="$pkg_dir/local-oldest-requirements.txt"
-    # packages like acme don't have any local oldest requirements
-    if [ ! -f "$requirements" ]; then
-        unset requirements
-    fi
-    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
-else
-    repo_root=$(dirname "$tools_dir")
-    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
-    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
-fi
-
-"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \
-                                   "$test_constraints" > "$all_constraints"
-
-set -x
-
-# install the requested packages using the pinned requirements as constraints
-if [ -n "$requirements" ]; then
-    pip install -q --constraint "$all_constraints" --requirement "$requirements"
-fi
-pip install -q --constraint "$all_constraints" "$@"

tools/pip_install_editable.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# pip installs packages in editable mode using certbot-auto's requirements file
+# as constraints
+
+from __future__ import absolute_import
+
+import sys
+
+import pip_install
+
+def main(args):
+    new_args = []
+    for arg in args:
+        new_args.append('-e')
+        new_args.append(arg)
+    pip_install.main(new_args)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

tools/pip_install_editable.sh

@@ -1,10 +0,0 @@
-#!/bin/sh -e
-# pip installs packages in editable mode using certbot-auto's requirements file
-# as constraints
-
-args=""
-for requirement in "$@" ; do
-  args="$args -e $requirement"
-done
-
-"$(dirname $0)/pip_install.sh" $args

tools/readlink.py

@@ -7,7 +7,12 @@ platforms.
 
 """
 from __future__ import print_function
+
 import os
 import sys
 
-print(os.path.realpath(sys.argv[1]))
+def main(link):
+    return os.path.realpath(link)
+
+if __name__ == '__main__':
+    print(main(sys.argv[1]))

tools/venv.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        command_python2_st_code = subprocess.call(
+            'command -v python2', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python2_st_code:
+            return '--python python2'
+
+        command_python27_st_code = subprocess.call(
+            'command -v python2.7', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python27_st_code:
+            return '--python python2.7'
+
+    raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    if os.name == 'nt':
+        raise ValueError('Certbot for Windows is not supported on Python 2.x.')
+
+    venv_args = get_venv_args()
+
+    _venv_common.main('venv', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    main()

tools/venv.sh

@@ -1,34 +0,0 @@
-#!/bin/sh -xe
-# Developer virtualenv setup for Certbot client
-
-if command -v python2; then
-    export VENV_ARGS="--python python2"
-elif command -v python2.7; then
-    export VENV_ARGS="--python python2.7"
-else
-    echo "Couldn't find python2 or python2.7 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test

tools/venv3.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        where_python3_st_code = subprocess.call(
+            'where python3', shell=True, stdout=fnull, stderr=fnull)
+        command_python3_st_code = subprocess.call(
+            'command -v python3', shell=True, stdout=fnull, stderr=fnull)
+
+    if not where_python3_st_code or not command_python3_st_code:
+        return '--python python3'
+
+    raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    venv_args = get_venv_args()
+
+    _venv_common.main('venv3', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    main()

tools/venv3.sh

@@ -1,33 +0,0 @@
-#!/bin/sh -xe
-# Developer Python3 virtualenv setup for Certbot
-
-if command -v python3; then
-    export VENV_NAME="${VENV_NAME:-venv3}"
-    export VENV_ARGS="--python python3"
-else
-    echo "Couldn't find python3 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test