From acb6d34c5f2658653687fade6a6f5b81a0ca0caf Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 18 Aug 2020 10:07:27 -0700 Subject: [PATCH] Update test farm tests to stop using certbot-auto (#8207) * Create bootstrap script * Delete a whole bunch of the bootstrap script * modify test_tests to use new script * put python version checking in back in * add x * call the venv creation from inside the bootstrap * add targets back * modify test_apache2 to use new format * shouldn't need virtualenv on rhel * readd targets * Update test_sdists to use new script * move setting up venv back out of script so it's not run with sudo * take venv3.py call out of bootstrap in all scripts * add additional python3-devel pkg name * fix test_sdists * enable additional rhel7 repos * clean up code and comments * Update tests and instructions to use auto_targets.yaml with test_leauto_upgrades.sh and test_letsencrypt_auto_certonly_standalone.sh * only install python3-devel.x86_64 for rhel7 * Upgrade python version for debian in test_apache2.sh * don't run test_tests or test_sdists on debian 9 or ubuntu 16.04 * Add 20.04 and 20.04 arm images to targets.yaml * use pyenv to upgrade to python3.5 * remove arm64 instance because it's having auth trouble * correct pyenv usage on ubuntu * add arm64 target to targets.yaml * replace debian 9 arm64 with ubuntu 20 * don't try to upgrade a perfectly good python version * let's just add ubuntu20 to apache2_targets while we're here * uncomment test_apache2 * move adding python3-devel.x86_64 to bootstrap_os_packages to avoid potential race condition * no need to specify the arch once extra rhel7 repos enabled * explicitly specify python3 * don't fail if we can't enable rhel7 extras * capture python36-devel as well --- tests/letstest/apache2_targets.yaml | 17 +- tests/letstest/auto_targets.yaml | 89 +++++++++ .../letstest/scripts/bootstrap_os_packages.sh | 170 ++++++++++++++++++ tests/letstest/scripts/test_apache2.sh | 39 +++- tests/letstest/scripts/test_sdists.sh | 39 ++-- tests/letstest/scripts/test_tests.sh | 20 ++- tests/letstest/targets.yaml | 31 ++-- tools/_release.sh | 4 +- tox.ini | 4 +- 9 files changed, 341 insertions(+), 72 deletions(-) create mode 100644 tests/letstest/auto_targets.yaml create mode 100755 tests/letstest/scripts/bootstrap_os_packages.sh diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 19411435c..d584705ca 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -3,6 +3,17 @@ targets: #----------------------------------------------------------------------------- #Ubuntu + - ami: ami-0758470213bdd23b1 + name: ubuntu20.04 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-008680ee60f23c94b + name: ubuntu20.04_arm64 + type: ubuntu + virt: hvm + user: ubuntu + machine_type: a1.medium - ami: ami-0545f7036167eb3aa name: ubuntu19.10 type: ubuntu @@ -30,12 +41,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-0ed54dd1b25657636 - name: debian9_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium #----------------------------------------------------------------------------- # Fedora - ami: ami-0fcbe88944a53b4c8 diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml new file mode 100644 index 000000000..9daba3910 --- /dev/null +++ b/tests/letstest/auto_targets.yaml @@ -0,0 +1,89 @@ +# These images are located in us-east-1. + +targets: + #----------------------------------------------------------------------------- + #Ubuntu + - ami: ami-0545f7036167eb3aa + name: ubuntu19.10 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-095192256fe1477ad + name: ubuntu18.04LTS + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-09677e0a6b14905b0 + name: ubuntu16.04LTS + type: ubuntu + virt: hvm + user: ubuntu + #----------------------------------------------------------------------------- + # Debian + - ami: ami-01db78123b2b99496 + name: debian10 + type: ubuntu + virt: hvm + user: admin + - ami: ami-003f19e0e687de1cd + name: debian9 + type: ubuntu + virt: hvm + user: admin + - ami: ami-0ed54dd1b25657636 + name: debian9_arm64 + type: ubuntu + virt: hvm + user: admin + machine_type: a1.medium + # userdata: | + # #cloud-init + # runcmd: + # - [ apt-get, install, -y, curl ] + #----------------------------------------------------------------------------- + # Other Redhat Distros + - ami: ami-0916c408cb02e310b + name: RHEL7 + type: centos + virt: hvm + user: ec2-user + - ami: ami-0c322300a1dd5dc79 + name: RHEL8 + type: centos + virt: hvm + user: ec2-user + - ami: ami-0fcbe88944a53b4c8 + name: fedora31 + type: centos + virt: hvm + user: fedora + - ami: ami-00bbc6858140f19ed + name: fedora30 + type: centos + virt: hvm + user: fedora + #----------------------------------------------------------------------------- + # CentOS + # These Marketplace AMIs must, irritatingly, have their terms manually + # agreed to on the AWS marketplace site for any new AWS account using them... + - ami: ami-9887c6e7 + name: centos7 + type: centos + virt: hvm + user: centos + # centos6 requires EPEL repo added + - ami: ami-1585c46a + name: centos6 + type: centos + virt: hvm + user: centos + userdata: | + #cloud-config + runcmd: + - yum install -y epel-release + - iptables -F + - ami: ami-01ca03df4a6012157 + name: centos8 + type: centos + virt: hvm + user: centos diff --git a/tests/letstest/scripts/bootstrap_os_packages.sh b/tests/letstest/scripts/bootstrap_os_packages.sh new file mode 100755 index 000000000..96506282b --- /dev/null +++ b/tests/letstest/scripts/bootstrap_os_packages.sh @@ -0,0 +1,170 @@ +#!/bin/sh +# +# Install OS dependencies for test farm tests. + +set -ex # Work even if somebody does "sh thisscript.sh". + +error() { + echo "$@" +} + +if command -v command > /dev/null 2>&1 ; then + export EXISTS="command -v" +elif which which > /dev/null 2>&1 ; then + export EXISTS="which" +else + error "Cannot find command nor which... please install one!" + exit 1 +fi + +# Sets LE_PYTHON to Python version string and PYVER to the first two +# digits of the python version. +DeterminePythonVersion() { + # If no Python is found, PYVER is set to 0. + for LE_PYTHON in python3 python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + $EXISTS "$LE_PYTHON" > /dev/null && break + done + if [ "$?" != "0" ]; then + PYVER=0 + return 0 + fi + + PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') +} + +BootstrapDebCommon() { + apt-get update || error apt-get update hit problems but continuing anyway... + + apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + gcc \ + libaugeas0 \ + libssl-dev \ + openssl \ + libffi-dev \ + ca-certificates \ + make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel + +} + +# Sets TOOL to the name of the package manager +InitializeRPMCommonBase() { + if type dnf 2>/dev/null + then + TOOL=dnf + elif type yum 2>/dev/null + then + TOOL=yum + + else + error "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + +} + +BootstrapRpmCommonBase() { + # Arguments: whitespace-delimited python packages to install + + InitializeRPMCommonBase + + pkgs=" + gcc + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + + # Add the python packages + pkgs="$pkgs + $1 + " + + if $TOOL list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if ! $TOOL install -y $pkgs; then + error "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +# This bootstrap concerns old RedHat-based distributions that do not ship by default +# with Python 2.7, but only Python 2.6. We bootstrap them by enabling SCL and installing +# Python 3.6. Some of these distributions are: CentOS/RHEL/OL/SL 6. +BootstrapRpmPython3Legacy() { + # Tested with: + # - CentOS 6 + + InitializeRPMCommonBase + + if ! "${TOOL}" list rh-python36 >/dev/null 2>&1; then + echo "To use Certbot on this operating system, packages from the SCL repository need to be installed." + if ! "${TOOL}" list centos-release-scl >/dev/null 2>&1; then + error "Enable the SCL repository and try running Certbot again." + exit 1 + fi + if ! "${TOOL}" install -y centos-release-scl; then + error "Could not enable SCL. Aborting bootstrap!" + exit 1 + fi + fi + + # CentOS 6 must use rh-python36 from SCL + if "${TOOL}" list rh-python36 >/dev/null 2>&1; then + python_pkgs="rh-python36-python + rh-python36-python-virtualenv + rh-python36-python-devel + " + else + error "No supported Python package available to install. Aborting bootstrap!" + exit 1 + fi + + BootstrapRpmCommonBase "${python_pkgs}" +} + +BootstrapRpmPython3() { + InitializeRPMCommonBase + + python_pkgs="python3 + python3-devel + " + + if ! $TOOL list 'python3*-devel' >/dev/null 2>&1; then + yum-config-manager --enable rhui-REGION-rhel-server-extras rhui-REGION-rhel-server-optional + fi + + BootstrapRpmCommonBase "$python_pkgs" +} + +# Set Bootstrap to the function that installs OS dependencies on this system. +if [ -f /etc/debian_version ]; then + Bootstrap() { + BootstrapDebCommon + } +elif [ -f /etc/redhat-release ]; then + DeterminePythonVersion + # Handle legacy RPM distributions + if [ "$PYVER" -eq 26 ]; then + Bootstrap() { + BootstrapRpmPython3Legacy + } + else + Bootstrap() { + BootstrapRpmPython3 + } + fi + +fi + +Bootstrap diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 9c3b95a31..e39cee9ef 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -12,6 +12,25 @@ then # For apache 2.4, set up ServerName sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE + if [ $(python3 -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -ne 38 ] + then + # Upgrade python version using pyenv because py3.5 is deprecated + # Don't upgrade if it's already 3.8 because pyenv doesn't work great on arm, and + # our arm representative happens to be ubuntu20, which already has a perfectly + # good version of python. + sudo apt-get install -y make gcc build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev python-openssl git # pyenv deps + curl https://pyenv.run | bash + export PATH="~/.pyenv/bin:$PATH" + pyenv init - + pyenv virtualenv-init - + pyenv install 3.8.5 + pyenv global 3.8.5 + # you do, in fact need to run these again, exactly like this. + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + fi elif [ "$OS_TYPE" = "centos" ] then CONFFILE=/etc/httpd/conf/httpd.conf @@ -40,18 +59,20 @@ fi cd letsencrypt echo "Bootstrapping dependencies..." -letsencrypt-auto-source/letsencrypt-auto --os-packages-only +sudo tests/letstest/scripts/bootstrap_os_packages.sh if [ $? -ne 0 ] ; then exit 1 fi -# This script sets the environment variables PYTHON_NAME, VENV_PATH, and -# VENV_SCRIPT based on the version of Python available on the system. For -# instance, Fedora uses Python 3 and Python 2 is not installed. -. tests/letstest/scripts/set_python_envvars.sh +if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment + # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap + PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" +fi -"$VENV_SCRIPT" -e acme[dev] -e certbot[dev,docs] -e certbot-apache -sudo "$VENV_PATH/bin/certbot" -v --debug --text --agree-tos \ +tools/venv3.py -e acme[dev] -e certbot[dev,docs] -e certbot-apache + +sudo "venv3/bin/certbot" -v --debug --text --agree-tos \ --renew-by-default --redirect --register-unsafely-without-email \ --domain $PUBLIC_HOSTNAME --server $BOULDER_URL if [ $? -ne 0 ] ; then @@ -68,7 +89,7 @@ elif [ "$OS_TYPE" = "centos" ]; then fi OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) -"$PYTHON_NAME" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +"venv3/bin/python" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" if [ $? -ne 0 ] ; then FAIL=1 fi @@ -76,7 +97,7 @@ fi if [ "$OS_TYPE" = "ubuntu" ] ; then export SERVER="$BOULDER_URL" - "$VENV_PATH/bin/tox" -e apacheconftest + "venv3/bin/tox" -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE fi diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index 338569e6d..05afb63dc 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -2,43 +2,27 @@ cd letsencrypt -# If we're on a RHEL 6 based system, we can be confident Python is already -# installed because the package manager is written in Python. +BOOTSTRAP_SCRIPT="tests/letstest/scripts/bootstrap_os_packages.sh" +VENV_PATH=venv3 + +# install OS packages +sudo $BOOTSTRAP_SCRIPT + if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then - # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment - RUN_RHEL6_TESTS=1 -fi - -letsencrypt-auto-source/letsencrypt-auto --install-only -n --debug - -if [ "$RUN_RHEL6_TESTS" = 1 ]; then + # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" fi -PLUGINS="certbot-apache certbot-nginx" -PYTHON_MAJOR_VERSION=$(/opt/eff.org/certbot/venv/bin/python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1) -TEMP_DIR=$(mktemp -d) - -if [ "$PYTHON_MAJOR_VERSION" = "3" ]; then - # Some distros like Fedora may only have an executable named python3 installed. - PYTHON_NAME="python3" - VENV_PATH="venv3" - VENV_SCRIPT="tools/venv3.py" -else - PYTHON_NAME="python" - VENV_SCRIPT="tools/venv.py" - VENV_PATH="venv" -fi - -VERSION=$("$PYTHON_NAME" letsencrypt-auto-source/version.py) - # setup venv -CERTBOT_PIP_NO_BINARY=":all:" "$VENV_SCRIPT" --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt +CERTBOT_PIP_NO_BINARY=:all: tools/venv3.py --requirement letsencrypt-auto-source/pieces/dependency-requirements.txt . "$VENV_PATH/bin/activate" # pytest is needed to run tests on some of our packages so we install a pinned version here. tools/pip_install.py pytest +PLUGINS="certbot-apache certbot-nginx" +TEMP_DIR=$(mktemp -d) + # build sdists for pkg_dir in acme certbot $PLUGINS; do cd $pkg_dir @@ -49,6 +33,7 @@ for pkg_dir in acme certbot $PLUGINS; do cd - done +VERSION=$(python letsencrypt-auto-source/version.py) # test sdists cd $TEMP_DIR for pkg in acme certbot $PLUGINS; do diff --git a/tests/letstest/scripts/test_tests.sh b/tests/letstest/scripts/test_tests.sh index fb86ce4cd..f62584709 100755 --- a/tests/letstest/scripts/test_tests.sh +++ b/tests/letstest/scripts/test_tests.sh @@ -8,12 +8,21 @@ REPO_ROOT="letsencrypt" LE_AUTO="$REPO_ROOT/letsencrypt-auto-source/letsencrypt-auto" LE_AUTO="$LE_AUTO --debug --no-self-upgrade --non-interactive" MODULES="acme certbot certbot-apache certbot-nginx" -PIP_INSTALL="$REPO_ROOT/tools/pip_install.py" -VENV_NAME=venv +PIP_INSTALL="tools/pip_install.py" +VENV_NAME=venv3 +BOOTSTRAP_SCRIPT="$REPO_ROOT/tests/letstest/scripts/bootstrap_os_packages.sh" +VENV_SCRIPT="tools/venv3.py" -# *-auto respects VENV_PATH -$LE_AUTO --os-packages-only -LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version +sudo $BOOTSTRAP_SCRIPT + +if command -v python && [ $(python -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//') -eq 26 ]; then + # RHEL/CentOS 6 will need a special treatment, so we need to detect that environment + # Enable the SCL Python 3.6 installed by letsencrypt-auto bootstrap + PATH="/opt/rh/rh-python36/root/usr/bin:$PATH" +fi + +cd $REPO_ROOT +$VENV_SCRIPT . $VENV_NAME/bin/activate "$PIP_INSTALL" pytest @@ -21,7 +30,6 @@ LE_AUTO_SUDO="" VENV_PATH="$VENV_NAME" $LE_AUTO --no-bootstrap --version # from the repo root. The directory structure should still # cause the installed packages to be tested while using # the tests available in the subdirectories. -cd $REPO_ROOT for module in $MODULES ; do echo testing $module diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 9daba3910..711053469 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -3,6 +3,17 @@ targets: #----------------------------------------------------------------------------- #Ubuntu + - ami: ami-0758470213bdd23b1 + name: ubuntu20.04 + type: ubuntu + virt: hvm + user: ubuntu + - ami: ami-008680ee60f23c94b + name: ubuntu20.04_arm64 + type: ubuntu + virt: hvm + user: ubuntu + machine_type: a1.medium - ami: ami-0545f7036167eb3aa name: ubuntu19.10 type: ubuntu @@ -13,11 +24,6 @@ targets: type: ubuntu virt: hvm user: ubuntu - - ami: ami-09677e0a6b14905b0 - name: ubuntu16.04LTS - type: ubuntu - virt: hvm - user: ubuntu #----------------------------------------------------------------------------- # Debian - ami: ami-01db78123b2b99496 @@ -25,21 +31,6 @@ targets: type: ubuntu virt: hvm user: admin - - ami: ami-003f19e0e687de1cd - name: debian9 - type: ubuntu - virt: hvm - user: admin - - ami: ami-0ed54dd1b25657636 - name: debian9_arm64 - type: ubuntu - virt: hvm - user: admin - machine_type: a1.medium - # userdata: | - # #cloud-init - # runcmd: - # - [ apt-get, install, -y, curl ] #----------------------------------------------------------------------------- # Other Redhat Distros - ami: ami-0916c408cb02e310b diff --git a/tools/_release.sh b/tools/_release.sh index 7e483905e..e17332f30 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -278,8 +278,8 @@ git commit -m "Add contents to certbot/CHANGELOG.md for next version" echo "New root: $root" echo "Test commands (in the letstest repo):" -echo 'python multitester.py targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' -echo 'python multitester.py targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' +echo 'python multitester.py auto_targets.yaml $AWS_KEY $USERNAME scripts/test_leauto_upgrades.sh --alt_pip $YOUR_PIP_REPO --branch public-beta' +echo 'python multitester.py auto_targets.yaml $AWK_KEY $USERNAME scripts/test_letsencrypt_auto_certonly_standalone.sh --branch candidate-0.1.1' echo 'python multitester.py --saveinstances targets.yaml $AWS_KEY $USERNAME scripts/test_apache2.sh' echo "In order to upload packages run the following command:" echo twine upload "$root/dist.$version/*/*" diff --git a/tox.ini b/tox.ini index c61379b94..0336a57df 100644 --- a/tox.ini +++ b/tox.ini @@ -290,14 +290,14 @@ setenv = {[testenv:test-farm-tests-base]setenv} [testenv:test-farm-leauto-upgrades] changedir = {[testenv:test-farm-tests-base]changedir} -commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir} +commands = python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_leauto_upgrades.sh --repo {toxinidir} deps = {[testenv:test-farm-tests-base]deps} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv} [testenv:test-farm-certonly-standalone] changedir = {[testenv:test-farm-tests-base]changedir} -commands = python multitester.py targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir} +commands = python multitester.py auto_targets.yaml {env:AWS_EC2_PEM_FILE} SET_BY_ENV scripts/test_letsencrypt_auto_certonly_standalone.sh --repo {toxinidir} deps = {[testenv:test-farm-tests-base]deps} passenv = {[testenv:test-farm-tests-base]passenv} setenv = {[testenv:test-farm-tests-base]setenv}