Retry failures to start boulder (#5176)

Occasionally a network error prevents Docker from starting boulder causing
Travis tests to fail like it did at
https://travis-ci.org/certbot/certbot/jobs/282923098. This works around the
problem by using travis_retry to try to start boulder again if it fails.

This also moves the logic of waiting for boulder to start into
tests/boulder-fetch.sh so people running integration tests locally can benefit.

.travis.yml

@@ -161,7 +161,9 @@ addons:
     - libapache2-mod-macro
 
 install: "travis_retry pip install tox coveralls"
-script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)'
+script:
+    - travis_retry tox
+    - '[ -z "${BOULDER_INTEGRATION+x}" ] || (travis_retry tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
 
 after_success: '[ "$TOXENV" == "cover" ] && coveralls'
 

tests/boulder-fetch.sh

@@ -17,3 +17,9 @@ FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}
 [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1
 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml
 docker-compose up -d
+
+set +x  # reduce verbosity while waiting for boulder
+until curl http://localhost:4000/directory 2>/dev/null; do
+  echo waiting for boulder
+  sleep 1
+done

tests/tox-boulder-integration.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+# A simple wrapper around tests/boulder-integration.sh that activates the tox
+# virtual environment defined by the environment variable TOXENV before running
+# integration tests.
+
+if [ -z "${TOXENV+x}" ]; then
+    echo "The environment variable TOXENV must be set to use this script!" >&2
+    exit 1
+fi
+
+source .tox/$TOXENV/bin/activate
+tests/boulder-integration.sh

tests/travis-integration.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-
-./tests/boulder-fetch.sh
-
-source .tox/$TOXENV/bin/activate
-
-until curl http://boulder:4000/directory 2>/dev/null; do
-  echo waiting for boulder
-  sleep 1
-done
-
-./tests/boulder-integration.sh