Switch from nose to pytest (#5282)

* Use pipstrap to install a good version of pip

* Use pytest in cb-auto tests

* Remove nose usage in auto_test.py

* remove nose dev dep

* use pytest in test_tests

* Use pytest in tox

* Update dev dependency pinnings

* remove nose multiprocess lines

* Use pytest for coverage

* Use older py and pytest for old python versions

* Add test for Error.__str__

* pin pytest in oldest test

* Fix tests for DNS-DO plugin on py26

* Work around bug for Python 3.3

* Clarify dockerfile comments

.coveragerc

@@ -1,3 +1,2 @@
 [report]
-# show lines missing coverage in output
+omit = */setup.py
-show_missing = True

acme/acme/crypto_util_test.py

@@ -18,7 +18,6 @@ from acme import test_util
 class SSLSocketAndProbeSNITest(unittest.TestCase):
     """Tests for acme.crypto_util.SSLSocket/probe_sni."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.cert = test_util.load_comparable_cert('rsa2048_cert.pem')
@@ -69,7 +68,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
 class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
     """Test for acme.crypto_util._pyopenssl_cert_or_req_san."""
 
-    _multiprocess_can_split_ = True
 
     @classmethod
     def _call(cls, loader, name):
@@ -140,7 +138,6 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
 class RandomSnTest(unittest.TestCase):
     """Test for random certificate serial numbers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.cert_count = 5

acme/acme/messages_test.py

@@ -71,6 +71,12 @@ class ErrorTest(unittest.TestCase):
         self.assertTrue(is_acme_error(Error.with_code('badCSR')))
         self.assertRaises(ValueError, Error.with_code, 'not an ACME error code')
 
+    def test_str(self):
+        self.assertEqual(
+            str(self.error),
+            u"{0.typ} :: {0.description} :: {0.detail} :: {0.title}"
+            .format(self.error))
+
 
 class ConstantTest(unittest.TestCase):
     """Tests for acme.messages._Constant."""

acme/acme/standalone_test.py

@@ -23,7 +23,6 @@ from acme import test_util
 class TLSServerTest(unittest.TestCase):
     """Tests for acme.standalone.TLSServer."""
 
-    _multiprocess_can_split_ = True
 
     def test_bind(self):  # pylint: disable=no-self-use
         from acme.standalone import TLSServer
@@ -42,7 +41,6 @@ class TLSServerTest(unittest.TestCase):
 class TLSSNI01ServerTest(unittest.TestCase):
     """Test for acme.standalone.TLSSNI01Server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.certs = {b'localhost': (
@@ -70,7 +68,6 @@ class TLSSNI01ServerTest(unittest.TestCase):
 class HTTP01ServerTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01Server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.account_key = jose.JWK.load(
@@ -124,7 +121,6 @@ class HTTP01ServerTest(unittest.TestCase):
 class BaseDualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.BaseDualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     class SingleProtocolServer(socketserver.TCPServer):
         """Server that only serves on a single protocol. FreeBSD has this behavior for AF_INET6."""
@@ -174,7 +170,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase):
 class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
     """Test for acme.standalone.TLSSNI01DualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.certs = {b'localhost': (
@@ -202,7 +197,6 @@ class TLSSNI01DualNetworkedServersTest(unittest.TestCase):
 class HTTP01DualNetworkedServersTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01DualNetworkedServers."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         self.account_key = jose.JWK.load(
@@ -254,7 +248,6 @@ class HTTP01DualNetworkedServersTest(unittest.TestCase):
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         # mirror ../examples/standalone

acme/setup.py

@@ -31,7 +31,8 @@ if sys.version_info < (2, 7):
     ])
 
 dev_extras = [
-    'nose',
+    'pytest',
+    'pytest-xdist',
     'tox',
 ]
 

certbot-apache/certbot_apache/tests/augeas_configurator_test.py

@@ -13,7 +13,6 @@ from certbot_apache.tests import util
 class AugeasConfiguratorTest(util.ApacheTest):
     """Test for Augeas Configurator base class."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(AugeasConfiguratorTest, self).setUp()

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -30,7 +30,6 @@ from certbot_apache.tests import util
 class MultipleVhostsTest(util.ApacheTest):
     """Test two standard well-configured HTTP vhosts."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTest, self).setUp()
@@ -1369,7 +1368,6 @@ class MultipleVhostsTest(util.ApacheTest):
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependent on augeas version."""
     # pylint: disable=protected-access
-    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         td = "debian_apache_2_4/augeas_vhosts"

certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py

@@ -5,7 +5,6 @@ import unittest
 
 import digitalocean
 import mock
-import six
 
 from certbot import errors
 from certbot.plugins import dns_test_common
@@ -134,8 +133,8 @@ class DigitalOceanClientTest(unittest.TestCase):
 
         correct_record_mock.destroy.assert_called()
 
-        six.assertCountEqual(self, first_record_mock.destroy.call_args_list, [])
+        self.assertFalse(first_record_mock.destroy.call_args_list)
-        six.assertCountEqual(self, last_record_mock.destroy.call_args_list, [])
+        self.assertFalse(last_record_mock.destroy.call_args_list)
 
     def test_del_txt_record_error_finding_domain(self):
         self.manager.get_all_domains.side_effect = API_ERROR

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -24,7 +24,6 @@ from certbot_nginx.tests import util
 class NginxConfiguratorTest(util.NginxTest):
     """Test a semi complex vhost configuration."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         super(NginxConfiguratorTest, self).setUp()

certbot/tests/cli_test.py

@@ -26,7 +26,6 @@ PLUGINS = disco.PluginsRegistry.find_all()
 class TestReadFile(TempDirTestCase):
     '''Test cli.read_file'''
 
-    _multiprocess_can_split_ = True
 
     def test_read_file(self):
         rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo'))
@@ -46,7 +45,6 @@ class TestReadFile(TempDirTestCase):
 class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
     '''Test the cli args entrypoint'''
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         reload_module(cli)
@@ -418,7 +416,6 @@ class ParseTest(unittest.TestCase):  # pylint: disable=too-many-public-methods
 class DefaultTest(unittest.TestCase):
     """Tests for certbot.cli._Default."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         # pylint: disable=protected-access
@@ -439,7 +436,6 @@ class DefaultTest(unittest.TestCase):
 class SetByCliTest(unittest.TestCase):
     """Tests for certbot.set_by_cli and related functions."""
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         reload_module(cli)

certbot/tests/ocsp_test.py

@@ -13,7 +13,6 @@ ocsp: Use -help for summary.
 
 class OCSPTest(unittest.TestCase):
 
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         from certbot import ocsp

certbot/tests/storage_test.py

@@ -43,7 +43,6 @@ class BaseRenewableCertTest(test_util.ConfigTestCase):
     your test.  Check :class:`.cli_test.DuplicateCertTest` for an example.
 
     """
-    _multiprocess_can_split_ = True
 
     def setUp(self):
         from certbot import storage

letsencrypt-auto-source/Dockerfile.centos6

@@ -5,9 +5,13 @@ FROM centos:6
 
 RUN yum install -y epel-release
 
-# Install pip, sudo and nose:
+# Install pip and sudo:
 RUN yum install -y python-pip sudo
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea
@@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.precise

@@ -6,13 +6,16 @@ FROM ubuntu:precise
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea
 
-# Install pip, sudo, openssl, and nose:
+# Install pip, sudo, and openssl:
 RUN apt-get update && \
     apt-get -q -y install python-pip sudo openssl && \
     apt-get clean
 
-ENV PIP_INDEX_URL https://pypi.python.org/simple
+# Use pipstrap to update to a stable and tested version of pip
-RUN pip install nose
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Let that user sudo:
 RUN sed -i.bkp -e \
@@ -30,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.trusty

@@ -11,11 +11,15 @@ RUN sed -i.bkp -e \
       's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
       /etc/sudoers
 
-# Install pip and nose:
+# Install pip:
 RUN apt-get update && \
     apt-get -q -y install python-pip && \
     apt-get clean
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 RUN mkdir -p /home/lea/certbot
 
@@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.wheezy

@@ -6,13 +6,15 @@ FROM debian:wheezy
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea
 
-# Install pip, sudo, openssl, and nose:
+# Install pip, sudo, and openssl:
 RUN apt-get update && \
     apt-get -q -y install python-pip sudo openssl && \
     apt-get clean
-
+# Use pipstrap to update to a stable and tested version of pip
-ENV PIP_INDEX_URL https://pypi.python.org/simple
+COPY ./pieces/pipstrap.py /opt
-RUN pip install nose
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Let that user sudo:
 RUN sed -i.bkp -e \
@@ -30,4 +32,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/tests/auto_test.py

@@ -17,10 +17,10 @@ from tempfile import mkdtemp
 from threading import Thread
 from unittest import TestCase
 
-from nose.tools import eq_, nottest, ok_
+from pytest import mark
 
 
-@nottest
+@mark.skip
 def tests_dir():
     """Return a path to the "tests" directory."""
     return dirname(abspath(__file__))
@@ -279,8 +279,8 @@ class AutoTests(TestCase):
                 # installed, and pip hashes verify:
                 install_le_auto(build_le_auto(version='50.0.0'), le_auto_path)
                 out, err = run_letsencrypt_auto()
-                ok_(re.match(r'letsencrypt \d+\.\d+\.\d+',
+                self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+',
-                             err.strip().splitlines()[-1]))
+                                err.strip().splitlines()[-1]))
                 # Make a few assertions to test the validity of the next tests:
                 self.assertTrue('Upgrading certbot-auto ' in out)
                 self.assertTrue('Creating virtual environment...' in out)
@@ -327,7 +327,7 @@ class AutoTests(TestCase):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("Couldn't verify signature of downloaded "
                                     "certbot-auto." in exc.output)
                 else:
@@ -348,10 +348,11 @@ class AutoTests(TestCase):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES "
                                     "FROM THE REQUIREMENTS FILE" in exc.output)
-                    ok_(not exists(venv_dir),
+                    self.assertFalse(
+                        exists(venv_dir),
                         msg="The virtualenv was left around, even though "
                             "installation didn't succeed. We shouldn't do "
                             "this, as it foils our detection of whether we "

setup.cfg

@@ -3,9 +3,3 @@ universal = 1
 
 [easy_install]
 zip_ok = false
-
-[nosetests]
-nocapture=1
-cover-package=certbot,acme,certbot_apache,certbot_nginx
-cover-erase=1
-cover-tests=1

setup.py

@@ -67,7 +67,9 @@ dev_extras = [
     'astroid==1.3.5',
     'coverage',
     'ipdb',
-    'nose',
+    'pytest',
+    'pytest-cov',
+    'pytest-xdist',
     'pylint==1.4.2',  # upstream #248
     'tox',
     'twine',

tests/letstest/scripts/test_tests.sh

@@ -10,9 +10,9 @@ LE_AUTO_SUDO="" VENV_PATH=$VENV_NAME letsencrypt/certbot-auto --debug --no-boots
 
 # change to an empty directory to ensure CWD doesn't affect tests
 cd $(mktemp -d)
-pip install nose
+pip install pytest==3.2.5
 
 for module in $MODULES ; do
     echo testing $module
-    nosetests -v $module
+    pytest -v --pyargs $module
 done

tools/install_and_test.sh

@@ -16,6 +16,10 @@ for requirement in "$@" ; do
   pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
   if [ $pkg = "." ]; then
     pkg="certbot"
+  else
+    # Work around a bug in pytest/importlib for the deprecated Python 3.3.
+    # See https://travis-ci.org/certbot/certbot/jobs/308774157#L1333.
+    pkg=$(echo "$pkg" | tr - _)
   fi
-  nosetests -v $pkg --processes=-1 --process-timeout=100
+  pytest --numprocesses auto --quiet --pyargs $pkg
 done

tools/pip_constraints.txt

@@ -4,6 +4,7 @@
 # invocation, some constraints may be ignored due to pip's lack of dependency
 # resolution.
 alabaster==0.7.10
+apipkg==1.4
 astroid==1.3.5
 Babel==2.5.1
 backports.shutil-get-terminal-size==1.0.0
@@ -15,6 +16,7 @@ decorator==4.1.2
 dns-lexicon[dnsmadeeasy]==2.1.11
 dnspython==1.15.0
 docutils==0.14
+execnet==1.5.0
 future==0.16.0
 futures==3.1.1
 google-api-python-client==1.6.4
@@ -27,7 +29,6 @@ Jinja2==2.9.6
 jmespath==0.9.3
 logilab-common==1.4.1
 MarkupSafe==1.0
-nose==1.3.7
 oauth2client==4.1.2
 pathlib2==2.3.0
 pexpect==4.2.1
@@ -42,6 +43,10 @@ pyasn1==0.3.7
 pyasn1-modules==0.1.5
 Pygments==2.2.0
 pylint==1.4.2
+pytest==3.2.5
+pytest-cov==2.5.1
+pytest-forked==0.2
+pytest-xdist==1.20.1
 python-dateutil==2.6.1
 python-digitalocean==1.12
 PyYAML==3.12

tools/release.sh

@@ -55,7 +55,7 @@ SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO"
 subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)"
 # certbot_compatibility_test is not packaged because:
 # - it is not meant to be used by anyone else than Certbot devs
-# - it causes problems when running nosetests - the latter tries to
+# - it causes problems when running pytest - the latter tries to
 #   run everything that matches test*, while there are no unittests
 #   there
 
@@ -166,10 +166,10 @@ fi
 mkdir kgs
 kgs="kgs/$version"
 pip freeze | tee $kgs
-pip install nose
+pip install pytest
 for module in $subpkgs_modules ; do
     echo testing $module
-    nosetests $module
+    pytest --pyargs $module
 done
 cd ~-
 

tox.cover.sh

@@ -16,7 +16,7 @@ fi
 
 cover () {
   if [ "$1" = "certbot" ]; then
-    min=98
+    min=97
   elif [ "$1" = "acme" ]; then
     min=100
   elif [ "$1" = "certbot_apache" ]; then
@@ -24,23 +24,23 @@ cover () {
   elif [ "$1" = "certbot_dns_cloudflare" ]; then
     min=98
   elif [ "$1" = "certbot_dns_cloudxns" ]; then
-    min=99
+    min=98
   elif [ "$1" = "certbot_dns_digitalocean" ]; then
     min=98
   elif [ "$1" = "certbot_dns_dnsimple" ]; then
     min=98
   elif [ "$1" = "certbot_dns_dnsmadeeasy" ]; then
-    min=99
+    min=98
   elif [ "$1" = "certbot_dns_google" ]; then
     min=99
   elif [ "$1" = "certbot_dns_luadns" ]; then
     min=98
   elif [ "$1" = "certbot_dns_nsone" ]; then
-    min=99
+    min=98
   elif [ "$1" = "certbot_dns_rfc2136" ]; then
     min=99
   elif [ "$1" = "certbot_dns_route53" ]; then
-    min=99
+    min=91
   elif [ "$1" = "certbot_nginx" ]; then
     min=97
   elif [ "$1" = "letshelp_certbot" ]; then
@@ -50,17 +50,10 @@ cover () {
     exit 1
   fi
 
-  # "-c /dev/null" makes sure setup.cfg is not loaded (multiple
+  pytest --cov "$1" --cov-report term-missing \
-  # --with-cover add up, --cover-erase must not be set for coveralls
+         --cov-fail-under "$min" --numprocesses auto --pyargs "$1"
-  # to get all the data); --with-cover scopes coverage to only
-  # specific package, positional argument scopes tests only to
-  # specific package directory; --cover-tests makes sure every tests
-  # is run (c.f. #403)
-  nosetests -c /dev/null --with-cover --cover-tests --cover-package  \
-            "$1" --cover-min-percentage="$min" "$1"
 }
 
-rm -f .coverage  # --cover-erase is off, make sure stats are correct
 for pkg in $pkgs
 do
   cover $pkg

tox.ini

@@ -6,9 +6,6 @@
 skipsdist = true
 envlist = modification,py{26,33,34,35,36},cover,lint
 
-# nosetest -v => more verbose output, allows to detect busy waiting
-# loops, especially on Travis
-
 [base]
 # pip installs the requested packages in editable mode
 pip_install = {toxinidir}/tools/pip_install_editable.sh
@@ -96,6 +93,7 @@ deps =
     pyasn1==0.1.9
     pyparsing==1.5.6
     pyrfc3339==1.0
+    pytest==3.2.5
     python-augeas==0.4.1
     pytz==2012c
     requests[security]==2.6.0