[Unix] Create a framework for certbot integration tests: PART 5-FINAL (#6989)

* Connect certbot-ci to travis. Remove old bash files.

* Configure test-everything

* Protect against import error

* Remove unused ignore

* Better handling of urllib3

* Correct path

* Remove a warning

* Correct call

* Protect atexit register execution

* Update docs/contributing.rst

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

* Update docs/contributing.rst

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

* Add again some bash scripts to avoid breaking to much retro-compatiblity on third party scripts

* Move boulder-v1 and boulder-v2 in nightly tests

* Separate oldest unit tests and oldest integration tests

* Remove try/except

* Test integration included in toxenv

* Add a wait to avoid a transient issue on OCSP status in oldest tests

* Clean travis.yml, split other tests

* Remove useless config

* Update .travis.yml

Co-Authored-By: Brad Warren <bmw@users.noreply.github.com>

* Update tox.ini

* Update tox.ini

* Remove pytest-sugar

* Remove empty pytest.ini, tests are working without it

.travis.yml

@@ -5,7 +5,9 @@ cache:
         - $HOME/.cache/pip
 
 before_script:
-  - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi'
+  - 'if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ulimit -n 1024 ; fi'
+  # On Travis, the fastest parallelization for integration tests has proved to be 4.
+  - 'if [[ "$TOXENV" == *"integration"* ]]; then export PYTEST_ADDOPTS="--numprocesses 4"; fi'
   - export TOX_TESTENV_PASSENV=TRAVIS
 
 # Only build pushes to the master branch, PRs, and branches beginning with
@@ -30,12 +32,7 @@ matrix:
   include:
     # Main test suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install
-      sudo: required
-      services: docker
-      <<: *not-on-master
-    - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=all TOXENV=py27_install
+      env: ACME_SERVER=pebble TOXENV=integration
       sudo: required
       services: docker
       <<: *not-on-master
@@ -102,64 +99,93 @@ matrix:
       env: TOXENV=py37 CERTBOT_NO_PIN=1
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest
+      env: TOXENV=py27-certbot-oldest
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: TOXENV=py27-nginx-oldest
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-certbot-oldest
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "2.7"
-      env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-certbot-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration-nginx-oldest
+      sudo: required
+      services: docker
+      <<: *extended-test-suite
+    - python: "2.7"
+      env: ACME_SERVER=boulder-v2 TOXENV=integration-nginx-oldest
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.4"
-      env: TOXENV=py34 BOULDER_INTEGRATION=v1
+      env: TOXENV=py34
+      <<: *extended-test-suite
+    - python: "3.5"
+      env: TOXENV=py35
+      <<: *extended-test-suite
+    - python: "3.6"
+      env: TOXENV=py36
+      <<: *extended-test-suite
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37
+      <<: *extended-test-suite
+    - python: "3.4"
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.4"
-      env: TOXENV=py34 BOULDER_INTEGRATION=v2
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.5"
-      env: TOXENV=py35 BOULDER_INTEGRATION=v1
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.5"
-      env: TOXENV=py35 BOULDER_INTEGRATION=v2
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.6"
-      env: TOXENV=py36 BOULDER_INTEGRATION=v1
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.6"
-      env: TOXENV=py36 BOULDER_INTEGRATION=v2
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.7"
       dist: xenial
-      env: TOXENV=py37 BOULDER_INTEGRATION=v1
+      env: ACME_SERVER=boulder-v1 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
     - python: "3.7"
       dist: xenial
-      env: TOXENV=py37 BOULDER_INTEGRATION=v2
+      env: ACME_SERVER=boulder-v2 TOXENV=integration
       sudo: required
       services: docker
       <<: *extended-test-suite
@@ -220,9 +246,7 @@ addons:
     - openssl
 
 install: "$(command -v pip || command -v pip3) install codecov tox"
-script:
-    - tox
-    - '[ -z "${BOULDER_INTEGRATION+x}" ] || (tests/boulder-fetch.sh && tests/tox-boulder-integration.sh)'
+script: tox
 
 after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux'