Switch from nose to pytest (#5282)

* Use pipstrap to install a good version of pip

* Use pytest in cb-auto tests

* Remove nose usage in auto_test.py

* remove nose dev dep

* use pytest in test_tests

* Use pytest in tox

* Update dev dependency pinnings

* remove nose multiprocess lines

* Use pytest for coverage

* Use older py and pytest for old python versions

* Add test for Error.__str__

* pin pytest in oldest test

* Fix tests for DNS-DO plugin on py26

* Work around bug for Python 3.3

* Clarify dockerfile comments

letsencrypt-auto-source/Dockerfile.centos6

@@ -5,9 +5,13 @@ FROM centos:6
 
 RUN yum install -y epel-release
 
-# Install pip, sudo and nose:
+# Install pip and sudo:
 RUN yum install -y python-pip sudo
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups wheel --uid 1000 lea
@@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.precise

@@ -6,13 +6,16 @@ FROM ubuntu:precise
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea
 
-# Install pip, sudo, openssl, and nose:
+# Install pip, sudo, and openssl:
 RUN apt-get update && \
     apt-get -q -y install python-pip sudo openssl && \
     apt-get clean
 
-ENV PIP_INDEX_URL https://pypi.python.org/simple
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Let that user sudo:
 RUN sed -i.bkp -e \
@@ -30,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.trusty

@@ -11,11 +11,15 @@ RUN sed -i.bkp -e \
       's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
       /etc/sudoers
 
-# Install pip and nose:
+# Install pip:
 RUN apt-get update && \
     apt-get -q -y install python-pip && \
     apt-get clean
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 RUN mkdir -p /home/lea/certbot
 
@@ -29,4 +33,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/Dockerfile.wheezy

@@ -6,13 +6,15 @@ FROM debian:wheezy
 # Add an unprivileged user:
 RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea
 
-# Install pip, sudo, openssl, and nose:
+# Install pip, sudo, and openssl:
 RUN apt-get update && \
     apt-get -q -y install python-pip sudo openssl && \
     apt-get clean
-
-ENV PIP_INDEX_URL https://pypi.python.org/simple
-RUN pip install nose
+# Use pipstrap to update to a stable and tested version of pip
+COPY ./pieces/pipstrap.py /opt
+RUN /opt/pipstrap.py
+# Pin pytest version for increased stability
+RUN pip install pytest==3.2.5
 
 # Let that user sudo:
 RUN sed -i.bkp -e \
@@ -30,4 +32,4 @@ COPY . /home/lea/certbot/letsencrypt-auto-source
 USER lea
 WORKDIR /home/lea
 
-CMD ["nosetests", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]
+CMD ["pytest", "-v", "-s", "certbot/letsencrypt-auto-source/tests"]

letsencrypt-auto-source/tests/auto_test.py

@@ -17,10 +17,10 @@ from tempfile import mkdtemp
 from threading import Thread
 from unittest import TestCase
 
-from nose.tools import eq_, nottest, ok_
+from pytest import mark
 
 
-@nottest
+@mark.skip
 def tests_dir():
     """Return a path to the "tests" directory."""
     return dirname(abspath(__file__))
@@ -279,8 +279,8 @@ class AutoTests(TestCase):
                 # installed, and pip hashes verify:
                 install_le_auto(build_le_auto(version='50.0.0'), le_auto_path)
                 out, err = run_letsencrypt_auto()
-                ok_(re.match(r'letsencrypt \d+\.\d+\.\d+',
-                             err.strip().splitlines()[-1]))
+                self.assertTrue(re.match(r'letsencrypt \d+\.\d+\.\d+',
+                                err.strip().splitlines()[-1]))
                 # Make a few assertions to test the validity of the next tests:
                 self.assertTrue('Upgrading certbot-auto ' in out)
                 self.assertTrue('Creating virtual environment...' in out)
@@ -327,7 +327,7 @@ class AutoTests(TestCase):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("Couldn't verify signature of downloaded "
                                     "certbot-auto." in exc.output)
                 else:
@@ -348,10 +348,11 @@ class AutoTests(TestCase):
                 try:
                     out, err = run_le_auto(le_auto_path, venv_dir, base_url)
                 except CalledProcessError as exc:
-                    eq_(exc.returncode, 1)
+                    self.assertEqual(exc.returncode, 1)
                     self.assertTrue("THESE PACKAGES DO NOT MATCH THE HASHES "
                                     "FROM THE REQUIREMENTS FILE" in exc.output)
-                    ok_(not exists(venv_dir),
+                    self.assertFalse(
+                        exists(venv_dir),
                         msg="The virtualenv was left around, even though "
                             "installation didn't succeed. We shouldn't do "
                             "this, as it foils our detection of whether we "