Fix and speed up compatibility-tests

* Fix nginx-compatibility tests

* sleep is overrated

* Reduce verbosity of nginx tests

certbot-compatibility-test/certbot_compatibility_test/test_driver.py

@@ -1,7 +1,6 @@
 """Tests Certbot plugins against different server configurations."""
 import argparse
 import filecmp
-import functools
 import logging
 import os
 import shutil
@@ -64,12 +63,12 @@ def test_authenticator(plugin, config, temp_dir):
                 type(achalls[i]), achalls[i].domain, config)
             success = False
         elif isinstance(responses[i], challenges.TLSSNI01Response):
-            verify = functools.partial(responses[i].simple_verify, achalls[i].chall,
+            verified = responses[i].simple_verify(achalls[i].chall,
-                                       achalls[i].domain,
+                                                  achalls[i].domain,
-                                       util.JWK.public_key(),
+                                                  util.JWK.public_key(),
-                                       host="127.0.0.1",
+                                                  host="127.0.0.1",
-                                       port=plugin.https_port)
+                                                  port=plugin.https_port)
-            if _try_until_true(verify):
+            if verified:
                 logger.info(
                     "tls-sni-01 verification for %s succeeded", achalls[i].domain)
             else:
@@ -155,10 +154,11 @@ def test_deploy_cert(plugin, temp_dir, domains):
         return False
 
     success = True
+    time.sleep(3)
     for domain in domains:
-        verify = functools.partial(validator.Validator().certificate, cert,
+        verified = validator.Validator().certificate(
-                                   domain, "127.0.0.1", plugin.https_port)
+            cert, domain, "127.0.0.1", plugin.https_port)
-        if not _try_until_true(verify):
+        if not verified:
             logger.error("**** Could not verify certificate for domain %s", domain)
             success = False
 
@@ -181,10 +181,8 @@ def test_enhancements(plugin, domains):
 
     for domain, info in domains_and_info:
         try:
-            verify = functools.partial(validator.Validator().any_redirect,
+            previous_redirect = validator.Validator().any_redirect(
-                                       "localhost", plugin.http_port,
+                "localhost", plugin.http_port, headers={"Host": domain})
-                                       headers={"Host": domain})
-            previous_redirect = _try_until_true(verify)
             info.append(previous_redirect)
             plugin.enhance(domain, "redirect")
             plugin.save()  # Needed by the Apache plugin
@@ -204,9 +202,9 @@ def test_enhancements(plugin, domains):
     for domain, info in domains_and_info:
         previous_redirect = info[0]
         if not previous_redirect:
-            verify = functools.partial(validator.Validator().redirect, "localhost",
+            verified = validator.Validator().redirect(
-                                       plugin.http_port, headers={"Host": domain})
+                "localhost", plugin.http_port, headers={"Host": domain})
-            if not _try_until_true(verify):
+            if not verified:
                 logger.error("*** Improper redirect for domain %s", domain)
                 success = False
 
@@ -216,17 +214,6 @@ def test_enhancements(plugin, domains):
     return success
 
 
-def _try_until_true(func, max_tries=5, sleep_time=0.5):
-    """Calls func up to max_tries times until it returns True"""
-    for _ in xrange(0, max_tries):
-        if func():
-            return True
-        else:
-            time.sleep(sleep_time)
-
-    return False
-
-
 def _save_and_restart(plugin, title=None):
     """Saves and restart the plugin, returning True if no errors occurred"""
     try:

tox.ini

@@ -151,7 +151,7 @@ passenv = DOCKER_*
 commands =
     docker build -t certbot-compatibility-test -f certbot-compatibility-test/Dockerfile .
     docker build -t nginx-compat -f certbot-compatibility-test/Dockerfile-nginx .
-    docker run --rm -it nginx-compat -c nginx.tar.gz -vvvv -aie
+    docker run --rm -it nginx-compat -c nginx.tar.gz -vv -aie
 whitelist_externals =
     docker
 passenv = DOCKER_*