Cleanup test farm tests and add test_sdists (#4089)

* add get_certbot_version.sh

* Use get_certbot_version.sh in build.py

* make test_leauto_upgrades.sh more robust

* auto upgrades break tests

* OCSP experimental is not needed anymore

* Add test_sdists.sh

* Use LE_AUTO_VERSION, not repo version

* install OS deps earlier

* use readlink not realpath

* undo changes to build.py

* Factor out version code from build.py

* Use version.py in test_sdists

* Remove get_certbot_version

* workaround setuptools breakage

letsencrypt-auto-source/build.py

@@ -8,26 +8,13 @@ other, special definitions.
 """
 from os.path import abspath, dirname, join
 import re
-from sys import argv
+
+from version import certbot_version, file_contents
 
 
 DIR = dirname(abspath(__file__))
 
 
-def certbot_version(build_script_dir):
-    """Return the version number stamped in certbot/__init__.py."""
-    return re.search('''^__version__ = ['"](.+)['"].*''',
-                     file_contents(join(dirname(build_script_dir),
-                                        'certbot',
-                                        '__init__.py')),
-                     re.M).group(1)
-
-
-def file_contents(path):
-    with open(path) as file:
-        return file.read()
-
-
 def build(version=None, requirements=None):
     """Return the built contents of the letsencrypt-auto script.
 

letsencrypt-auto-source/version.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""Get the current Certbot version number.
+
+Provides simple utilities for determining the Certbot version number and
+building letsencrypt-auto.
+
+"""
+from __future__ import print_function
+from os.path import abspath, dirname, join
+import re
+
+
+def certbot_version(build_script_dir):
+    """Return the version number stamped in certbot/__init__.py."""
+    return re.search('''^__version__ = ['"](.+)['"].*''',
+                     file_contents(join(dirname(build_script_dir),
+                                        'certbot',
+                                        '__init__.py')),
+                     re.M).group(1)
+
+
+def file_contents(path):
+    with open(path) as file:
+        return file.read()
+
+
+if __name__ == '__main__':
+    print(certbot_version(dirname(abspath(__file__))))

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -29,7 +29,8 @@ unset PIP_INDEX_URL
 export PIP_EXTRA_INDEX_URL="$SAVE"
 
 git checkout -f "$BRANCH"
-if ! ./letsencrypt-auto -v --debug --version | grep 0.9.0 ; then
+EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2)
+if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade | grep $EXPECTED_VERSION ; then
     echo upgrade appeared to fail
     exit 1
 fi

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -23,7 +23,7 @@ cd tests/letstest/testdata/
 
 # 2. refer to the config with the same level of relativity that it itself
 # contains :/
-OUT=`letsencrypt-auto certificates --config-dir sample-config -v`
+OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade`
 TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
 REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
 

tests/letstest/scripts/test_ocsp_experimental.sh

@@ -1,39 +0,0 @@
-#!/bin/bash -x
-
-# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
-
-# with curl, instance metadata available from EC2 metadata service:
-#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
-
-cd letsencrypt
-export PATH="$PWD/letsencrypt-auto-source:$PATH"
-letsencrypt-auto-source/letsencrypt-auto --os-packages-only --debug --version
-tools/venv.sh
-sudo venv/bin/certbot certonly --no-self-upgrade -v --standalone --debug \
-                   --text --agree-dev-preview --agree-tos \
-                   --renew-by-default --redirect \
-                   --register-unsafely-without-email \
-                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
-
-# we have to jump through some hoops to cope with relative paths in renewal
-# conf files ...
-# 1. be in the right directory
-cd tests/letstest/testdata/
-
-# 2. refer to the config with the same level of relativity that it itself
-# contains :/
-OUT=`sudo ../../../venv/bin/certbot certificates  -v --config-dir sample-config`
-TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
-REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
-
-if [ "$TEST_CERTS" != 2 ] ; then
-    echo "Did not find two test certs as expected ($TEST_CERTS)"
-    exit 1
-fi
-
-if [ "$REVOKED" != 1 ] ; then
-    echo "Did not find one revoked cert as expected ($REVOKED)"
-    exit 1
-fi

tests/letstest/scripts/test_sdists.sh

@@ -0,0 +1,36 @@
+#!/bin/sh -xe
+
+cd letsencrypt
+./certbot-auto --os-packages-only -n --debug
+
+PLUGINS="certbot-apache certbot-nginx"
+PYTHON=$(command -v python2.7 || command -v python27 || command -v python2 || command -v python)
+TEMP_DIR=$(mktemp -d)
+VERSION=$(letsencrypt-auto-source/version.py)
+
+# setup venv
+virtualenv --no-site-packages -p $PYTHON --setuptools venv
+. ./venv/bin/activate
+pip install -U pip
+pip install -U setuptools
+
+# build sdists
+for pkg_dir in acme . $PLUGINS; do
+    cd $pkg_dir
+    python setup.py clean
+    rm -rf build dist
+    python setup.py sdist
+    mv dist/* $TEMP_DIR
+    cd -
+done
+
+# test sdists
+cd $TEMP_DIR
+for pkg in acme certbot $PLUGINS; do
+    tar -xvf "$pkg-$VERSION.tar.gz"
+    cd "$pkg-$VERSION"
+    python setup.py build
+    python setup.py test
+    python setup.py install
+    cd -
+done