Make a test farm tests package (#8821)

Fixes https://github.com/certbot/certbot/issues/8781.

This PR makes our test farm tests into a normal package so it and its dependencies can be tracked and installed like our other packages.

Other noteworthy changes in this PR:

* Rather than continuing to place logs in your CWD, they're placed in a temporary directory that is printed to the terminal.
*  `tests/letstest/auto_targets.yaml` was deleted rather than renamed because the file is no longer used.

* make a letstest package

* remove deleted deps

* fix letstest install

* add __init__.py

* call main

* Explicitly mention activating venv

* rerename file

* fix version.py path

* clarify "this"

* Use >= instead of caret requirement

.gitignore

@@ -30,12 +30,6 @@ tags
 # auth --cert-path --chain-path
 /*.pem
 
-# letstest
-tests/letstest/letest-*/
-tests/letstest/*.pem
-tests/letstest/venv/
-tests/letstest/venv3/
-
 .venv
 
 # pytest cache

letstest/README.md

@@ -14,17 +14,13 @@ Simple AWS testfarm scripts for certbot client testing
     are needed, they need to be requested via online webform.
 
 ## Installation and configuration
-These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you're
-on a Debian based system, make sure you also have the python3-venv package
-installed. If you have Python 3 installed, you can use requirements.txt to
-create a virtual environment with a known set of dependencies by running:
-```
-python3 -m venv venv3
-. ./venv3/bin/activate
-pip install --requirement requirements.txt
-```
 
-You can then configure AWS credentials and create a key by running:
+This package is installed in the Certbot development environment that is
+created by following the instructions at
+https://certbot.eff.org/docs/contributing.html#running-a-local-copy-of-the-client.
+
+After activating that virtual environment, you can then configure AWS
+credentials and create a key by running:
 ```
 >aws configure --profile <profile name>
 [interactive: enter secrets for IAM role]
@@ -35,9 +31,9 @@ Note: whatever you pick for `<key name>` will be shown to other users with AWS a
 When prompted for a default region name, enter: `us-east-1`.
 
 ## Usage
-To run tests, activate the virtual environment you created above and run:
+To run tests, activate the virtual environment you created above and from this directory run:
 ```
->python multitester.py targets.yaml /path/to/your/key.pem <profile name> scripts/<test to run>
+>letstest targets/targets.yaml /path/to/your/key.pem <profile name> scripts/<test to run>
 ```
 
 You can only run up to two tests at once. The following error is often indicative of there being too many AWS instances running on our account:
@@ -52,7 +48,7 @@ aws ec2 terminate-instances --profile <profile name> --instance-ids $(aws ec2 de
 
 It will take a minute for these instances to shut down and become available again. Running this will invalidate any in progress tests.
 
-A folder named `letest-<timestamp>` is also created with a log file from each instance of the test and a file named "results" containing the output above.
+A temporary directory whose name is output by the tests is also created with a log file from each instance of the test and a file named "results" containing the output above.
 The tests take quite a while to run.
 
 ## Scripts

letstest/letstest/multitester.py

@@ -22,7 +22,7 @@ Usage:
 >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
  --query 'KeyMaterial' --output text > MyKeyPair.pem
 then:
->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh
+>letstest targets/targets.yaml MyKeyPair.pem HappyHacker scripts/test_sdists.sh
 see:
   https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
   https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
@@ -33,6 +33,7 @@ from multiprocessing import Manager
 import os
 import socket
 import sys
+import tempfile
 import time
 import traceback
 import urllib.error as urllib_error
@@ -374,9 +375,8 @@ def main():
 
     # Set up local copy of git repo
     #-------------------------------------------------------------------------------
-    log_dir = "letest-%d"%int(time.time()) #points to logging / working directory
+    log_dir = tempfile.mkdtemp()  # points to logging / working directory
-    print("Making local dir for test repo and logs: %s"%log_dir)
+    print("Local dir for test repo and logs: %s"%log_dir)
-    local_cxn.local('mkdir %s'%log_dir)
 
     try:
         # figure out what git object to test and locally create it in log_dir

letstest/scripts/test_apache2.sh

@@ -59,7 +59,7 @@ fi
 cd letsencrypt
 
 echo "Bootstrapping dependencies..."
-sudo tests/letstest/scripts/bootstrap_os_packages.sh
+sudo letstest/scripts/bootstrap_os_packages.sh
 if [ $? -ne 0 ] ; then
     exit 1
 fi
@@ -113,7 +113,7 @@ elif [ "$OS_TYPE" = "centos" ]; then
 fi
 OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9)
 APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8)
-"venv/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION"
+"venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION"
 if [ $? -ne 0 ] ; then
     FAIL=1
 fi

letstest/scripts/test_sdists.sh

@@ -2,7 +2,7 @@
 
 cd letsencrypt
 
-BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh"
+BOOTSTRAP_SCRIPT="letstest/scripts/bootstrap_os_packages.sh"
 VENV_PATH=venv
 
 # install OS packages
@@ -41,7 +41,7 @@ for pkg_dir in acme certbot $PLUGINS; do
     cd -
 done
 
-VERSION=$(python tests/letstest/scripts/version.py)
+VERSION=$(python letstest/scripts/version.py)
 # test sdists
 cd $TEMP_DIR
 for pkg in acme certbot $PLUGINS; do

letstest/scripts/version.py

@@ -9,10 +9,10 @@ from os.path import abspath, dirname, join
 import re
 
 
-def certbot_version(caller_dir):
+def certbot_version(letstest_scripts_dir):
     """Return the version number stamped in certbot/__init__.py."""
     return re.search('''^__version__ = ['"](.+)['"].*''',
-                     file_contents(join(dirname(dirname(dirname(caller_dir))),
+                     file_contents(join(dirname(dirname(letstest_scripts_dir)),
                                         'certbot',
                                         'certbot',
                                         '__init__.py')),

letstest/setup.py

@@ -0,0 +1,45 @@
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name='letstest',
+    version='1.0',
+    description='Test Certbot on different AWS images',
+    url='https://github.com/certbot/certbot',
+    author='Certbot Project',
+    author_email='certbot-dev@eff.org',
+    license='Apache License 2.0',
+    python_requires='>=3.6',
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+    ],
+
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        # awscli isn't required by the tests themselves, but it is a useful
+        # tool to have when using these tests to generate keys and control
+        # running instances so the dependency is declared here for convenience.
+        'awscli',
+        'boto3',
+        'botocore',
+        # The API from Fabric 2.0+ is used instead of the 1.0 API.
+        'fabric>=2',
+        'pyyaml',
+    ],
+    entry_points={
+        'console_scripts': [
+            'letstest=letstest.multitester:main',
+        ],
+    }
+)

tests/letstest/auto_targets.yaml

@@ -1,59 +0,0 @@
-# These images are located in us-east-1.
-
-targets:
-  #-----------------------------------------------------------------------------
-  #Ubuntu
-  - ami: ami-095192256fe1477ad
-    name: ubuntu18.04LTS
-    type: ubuntu
-    virt: hvm
-    user: ubuntu
-  - ami: ami-09677e0a6b14905b0
-    name: ubuntu16.04LTS
-    type: ubuntu
-    virt: hvm
-    user: ubuntu
-  #-----------------------------------------------------------------------------
-  # Debian
-  - ami: ami-01db78123b2b99496
-    name: debian10
-    type: ubuntu
-    virt: hvm
-    user: admin
-  - ami: ami-003f19e0e687de1cd
-    name: debian9
-    type: ubuntu
-    virt: hvm
-    user: admin
-  - ami: ami-0ed54dd1b25657636
-    name: debian9_arm64
-    type: ubuntu
-    virt: hvm
-    user: admin
-    machine_type: a1.medium
-  #-----------------------------------------------------------------------------
-  # Other Redhat Distros
-  - ami: ami-0916c408cb02e310b
-    name: RHEL7
-    type: centos
-    virt: hvm
-    user: ec2-user
-  - ami: ami-0c322300a1dd5dc79
-    name: RHEL8
-    type: centos
-    virt: hvm
-    user: ec2-user
-  #-----------------------------------------------------------------------------
-  # CentOS
-  # These Marketplace AMIs must, irritatingly, have their terms manually
-  # agreed to on the AWS marketplace site for any new AWS account using them...
-  - ami: ami-9887c6e7
-    name: centos7
-    type: centos
-    virt: hvm
-    user: centos
-  - ami: ami-01ca03df4a6012157
-    name: centos8
-    type: centos
-    virt: hvm
-    user: centos

tests/letstest/requirements.txt

@@ -1,22 +0,0 @@
-awscli==1.19.36
-bcrypt==3.2.0
-boto3==1.17.36
-botocore==1.20.36
-cffi==1.14.5
-colorama==0.4.3
-cryptography==3.4.6
-docutils==0.15.2
-fabric==2.6.0
-invoke==1.5.0
-jmespath==0.10.0
-paramiko==2.7.2
-pathlib2==2.3.5
-pyasn1==0.4.8
-pycparser==2.20
-PyNaCl==1.4.0
-python-dateutil==2.8.1
-PyYAML==5.4.1
-rsa==4.5
-s3transfer==0.3.6
-six==1.15.0
-urllib3==1.26.4

tools/_release.sh

@@ -211,8 +211,8 @@ git add certbot/CHANGELOG.md
 git commit -m "Add contents to certbot/CHANGELOG.md for next version"
 
 echo "New root: $root"
-echo "Test commands (in the letstest repo):"
+echo "Test commands (in the letstest directory):"
-echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh'
+echo 'letstest --saveinstances targets/targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh'
 echo "In order to upload packages run the following command:"
 echo twine upload "$root/dist.$version/*/*"
 

tools/pinning/pyproject.toml

@@ -32,9 +32,18 @@ certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]}
 certbot-apache = {path = "../../certbot-apache", extras = ["dev"]}
 certbot = {path = "../../certbot", extras = ["dev", "docs"]}
 acme = {path = "../../acme", extras = ["dev", "docs"]}
+letstest = {path = "../../letstest"}
 windows-installer = {path = "../../windows-installer"}
 
 # Extra dependencies
+# awscli is just listed here as a performance optimization. As of writing this,
+# there are some conflicts in shared dependencies between it and other packages
+# we depend on. To try and resolve them, poetry searches through older versions
+# of awscli to see if that resolves the conflict, but there are over 1000
+# versions of awscli on PyPI so this process takes too long. Providing a high
+# minimum version here prevents poetry from searching this path and it fairly
+# quickly resolves the dependency conflict in another way.
+awscli = ">=1.19.62"
 # As of writing this, cython is a build dependency of pyyaml. Since there
 # doesn't appear to be a good way to automatically track down and pin build
 # dependencies in Python (see

tools/requirements.txt

@@ -12,25 +12,26 @@ appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or pyt
 appnope==0.1.2
 astroid==2.3.3; python_version >= "3.6"
 attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+awscli==1.19.62; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
 azure-devops==6.0.0b4; python_version >= "3.6"
-babel==2.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
+babel==2.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 backcall==0.2.0
-bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+bcrypt==3.2.0; python_version >= "3.6"
 beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0"
 bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-boto3==1.17.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+boto3==1.17.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-botocore==1.20.53; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+botocore==1.20.62; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
+cachetools==4.2.2; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 cloudflare==2.8.15; python_version >= "3.6"
-colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+colorama==0.4.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 configobj==5.0.6; python_version >= "3.6"
 coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4")
@@ -47,14 +48,15 @@ docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0
 docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
+docutils==0.15.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0")
 execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+fabric==2.6.0; python_version >= "3.6"
 filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0"
 future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0"
 google-api-core==1.26.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-google-api-python-client==2.2.0; python_version >= "3.6"
+google-api-python-client==2.3.0; python_version >= "3.6"
 google-auth-httplib2==0.1.0; python_version >= "3.6"
-google-auth==1.29.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-auth==1.30.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 httplib2==0.19.1; python_version >= "3.6"
@@ -62,6 +64,7 @@ idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python
 imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0")
 importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7"
+invoke==1.5.0; python_version >= "3.6"
 ipdb==0.13.7; python_version >= "3.6"
 ipython-genutils==0.2.0; python_version == "3.6"
 ipython==7.16.1; python_version == "3.6"
@@ -93,6 +96,7 @@ paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or py
 parsedatetime==2.6; python_version >= "3.6"
 parso==0.8.2; python_version == "3.6"
 pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+pathlib2==2.3.5; python_version >= "3.6"
 pexpect==4.8.0
 pickleshare==0.7.5
 pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
@@ -105,14 +109,14 @@ protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or p
 ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-pyasn1==0.4.8; python_version >= "3.6" and python_version < "4"
+pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pygithub==1.54.1; python_version >= "3.6"
+pygithub==1.55; python_version >= "3.6"
 pygments==2.8.1
-pyjwt==1.7.1; python_version >= "3.6"
+pyjwt==2.1.0; python_version >= "3.6"
 pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 pylint==2.4.3; python_version >= "3.5"
-pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 pynsist==2.7; python_version >= "3.6"
 pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -126,7 +130,7 @@ pytest==3.2.5
 python-augeas==0.5.0
 python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 python-digitalocean==1.16.0; python_version >= "3.6"
-python-dotenv==0.17.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
+python-dotenv==0.17.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0"
 pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32"
 pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6")
@@ -140,10 +144,10 @@ requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0
 requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0"
 rfc3986==1.4.0; python_version >= "3.6"
 rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
-s3transfer==0.3.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+s3transfer==0.4.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux"
 shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
+six==1.15.0; python_version == "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version == "3.6"
 snowballstemmer==2.1.0; python_version >= "3.6"
 soupsieve==2.2.1; python_version >= "3.6"
 sphinx-rtd-theme==0.5.2; python_version >= "3.6"
@@ -166,7 +170,7 @@ typed-ast==1.4.3; implementation_name == "cpython" and python_version < "3.8" an
 typing-extensions==3.7.4.3; python_version >= "3.6"
 uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
-virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
+virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 wcwidth==0.2.5; python_version == "3.6"
 webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
 websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"

tools/venv.py

@@ -44,6 +44,7 @@ REQUIREMENTS = [
     '-e certbot-nginx',
     '-e certbot-compatibility-test',
     '-e certbot-ci',
+    '-e letstest',
 ]
 
 if sys.platform == 'win32':

tox.ini

@@ -277,22 +277,26 @@ passenv = DOCKER_*
 setenv = {[testenv:oldest]setenv}
 
 [testenv:test-farm-tests-base]
-changedir = tests/letstest
+changedir = letstest
-deps = -rtests/letstest/requirements.txt
+# The package to install is in the current working directory because of the
+# value of changedir.
+commands = {[base]pip_install} .
 passenv =
     AWS_*
 setenv = AWS_DEFAULT_REGION=us-east-1
 
 [testenv:test-farm-apache2]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = {toxinidir}/tools/retry.sh python multitester.py apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
+commands =
-deps = {[testenv:test-farm-tests-base]deps}
+    {[testenv:test-farm-tests-base]commands}
+    {toxinidir}/tools/retry.sh letstest targets/apache2_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_apache2.sh --repo {toxinidir}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}
 
 [testenv:test-farm-sdists]
 changedir = {[testenv:test-farm-tests-base]changedir}
-commands = {toxinidir}/tools/retry.sh python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
+commands =
-deps = {[testenv:test-farm-tests-base]deps}
+    {[testenv:test-farm-tests-base]commands}
+    {toxinidir}/tools/retry.sh letstest targets/targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_sdists.sh --repo {toxinidir}
 passenv = {[testenv:test-farm-tests-base]passenv}
 setenv = {[testenv:test-farm-tests-base]setenv}