Parallalelise nosetests from tox (#3836)

* Parallalelise nosetests from tox

* Parallelise even more things, break even more things

* Now unbreak all the tests that aren't ready for ||ism

* Try to pass tests!

 - Remove non-working hack in reporter_test
 - also be selective about ||ism in the cover environment

* Try again

* certbot-apache tests also work, given enough time

* Nginx may need more time in Travis's cloud

* Unbreak reporter_test under ||ism

* More timeout

* Working again?

* This goes way faster

* Another big win

* Split a couple more large test suites

* A last improvement

* More ||ism!

* ||ise lint too

* Allow nosetests to figure out how many cores to use

* simplify merge

* Mark the new CLI tests as ||izable

* Simplify reporter_test changes

* Rationalise ||ism flags

* Re-up coverage

* Clean up reporter tests

* Stop modifying testdata during tests

* remove unused os

acme/acme/crypto_util_test.py

@@ -18,6 +18,8 @@ from acme import test_util
 class SSLSocketAndProbeSNITest(unittest.TestCase):
     """Tests for acme.crypto_util.SSLSocket/probe_sni."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         self.cert = test_util.load_comparable_cert('rsa2048_cert.pem')
         key = test_util.load_pyopenssl_private_key('rsa2048_key.pem')
@@ -67,6 +69,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
 class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
     """Test for acme.crypto_util._pyopenssl_cert_or_req_san."""
 
+    _multiprocess_can_split_ = True
+
     @classmethod
     def _call(cls, loader, name):
         # pylint: disable=protected-access
@@ -131,6 +135,8 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase):
 class RandomSnTest(unittest.TestCase):
     """Test for random certificate serial numbers."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         self.cert_count = 5
         self.serial_num = []

acme/acme/standalone_test.py

@@ -21,6 +21,8 @@ from acme import test_util
 class TLSServerTest(unittest.TestCase):
     """Tests for acme.standalone.TLSServer."""
 
+    _multiprocess_can_split_ = True
+
     def test_bind(self):  # pylint: disable=no-self-use
         from acme.standalone import TLSServer
         server = TLSServer(
@@ -31,6 +33,8 @@ class TLSServerTest(unittest.TestCase):
 class TLSSNI01ServerTest(unittest.TestCase):
     """Test for acme.standalone.TLSSNI01Server."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         self.certs = {b'localhost': (
             test_util.load_pyopenssl_private_key('rsa2048_key.pem'),
@@ -57,6 +61,8 @@ class TLSSNI01ServerTest(unittest.TestCase):
 class HTTP01ServerTest(unittest.TestCase):
     """Tests for acme.standalone.HTTP01Server."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         self.account_key = jose.JWK.load(
             test_util.load_vector('rsa1024_key.pem'))
@@ -109,6 +115,8 @@ class HTTP01ServerTest(unittest.TestCase):
 class TestSimpleTLSSNI01Server(unittest.TestCase):
     """Tests for acme.standalone.simple_tls_sni_01_server."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         # mirror ../examples/standalone
         self.test_cwd = tempfile.mkdtemp()

certbot-apache/certbot_apache/tests/augeas_configurator_test.py

@@ -13,6 +13,8 @@ from certbot_apache.tests import util
 class AugeasConfiguratorTest(util.ApacheTest):
     """Test for Augeas Configurator base class."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):  # pylint: disable=arguments-differ
         super(AugeasConfiguratorTest, self).setUp()
 

certbot-apache/certbot_apache/tests/configurator_test.py

@@ -24,6 +24,8 @@ from certbot_apache.tests import util
 class MultipleVhostsTest(util.ApacheTest):
     """Test two standard well-configured HTTP vhosts."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):  # pylint: disable=arguments-differ
         super(MultipleVhostsTest, self).setUp()
 
@@ -1241,6 +1243,7 @@ class MultipleVhostsTest(util.ApacheTest):
 class AugeasVhostsTest(util.ApacheTest):
     """Test vhosts with illegal names dependant on augeas version."""
     # pylint: disable=protected-access
+    _multiprocess_can_split_ = True
 
     def setUp(self):  # pylint: disable=arguments-differ
         td = "debian_apache_2_4/augeas_vhosts"

certbot-nginx/certbot_nginx/tests/configurator_test.py

@@ -21,6 +21,8 @@ from certbot_nginx.tests import util
 class NginxConfiguratorTest(util.NginxTest):
     """Test a semi complex vhost configuration."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         super(NginxConfiguratorTest, self).setUp()
 

certbot-nginx/certbot_nginx/tests/nginxparser_test.py

@@ -1,7 +1,7 @@
 """Test for certbot_nginx.nginxparser."""
 import copy
 import operator
-import os
+import tempfile
 import unittest
 
 from pyparsing import ParseException
@@ -128,44 +128,34 @@ class TestRawNginxParser(unittest.TestCase):
                                  [['root', ' ', 'html'],
                                   ['index', ' ', 'index.html index.htm']]]]]))
 
-        with open(util.get_data_filename('nginx.new.conf'), 'w') as handle:
+        with tempfile.TemporaryFile() as f:
-            dump(parsed, handle)
+            dump(parsed, f)
-        with open(util.get_data_filename('nginx.new.conf')) as handle:
+            f.seek(0)
-            parsed_new = load(handle)
+            parsed_new = load(f)
-        try:
+        self.assertEqual(parsed, parsed_new)
-            self.maxDiff = None
-            self.assertEqual(parsed[0], parsed_new[0])
-            self.assertEqual(parsed[1:], parsed_new[1:])
-        finally:
-            os.unlink(util.get_data_filename('nginx.new.conf'))
 
     def test_comments(self):
         with open(util.get_data_filename('minimalistic_comments.conf')) as handle:
             parsed = load(handle)
 
-        with open(util.get_data_filename('minimalistic_comments.new.conf'), 'w') as handle:
+        with tempfile.TemporaryFile() as f:
-            dump(parsed, handle)
+            dump(parsed, f)
+            f.seek(0)
+            parsed_new = load(f)
 
-        with open(util.get_data_filename('minimalistic_comments.new.conf')) as handle:
+        self.assertEqual(parsed, parsed_new)
-            parsed_new = load(handle)
+        self.assertEqual(parsed_new, [
-
+            ['#', " Use bar.conf when it's a full moon!"],
-        try:
+            ['include', 'foo.conf'],
-            self.assertEqual(parsed, parsed_new)
+            ['#', ' Kilroy was here'],
-
+            ['check_status'],
-            self.assertEqual(parsed_new, [
+            [['server'],
-                ['#', " Use bar.conf when it's a full moon!"],
+             [['#', ''],
-                ['include', 'foo.conf'],
+              ['#', " Don't forget to open up your firewall!"],
-                ['#', ' Kilroy was here'],
+              ['#', ''],
-                ['check_status'],
+              ['listen', '1234'],
-                [['server'],
+              ['#', ' listen 80;']]],
-                 [['#', ''],
+        ])
-                  ['#', " Don't forget to open up your firewall!"],
-                  ['#', ''],
-                  ['listen', '1234'],
-                  ['#', ' listen 80;']]],
-            ])
-        finally:
-            os.unlink(util.get_data_filename('minimalistic_comments.new.conf'))
 
     def test_issue_518(self):
         parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }')

certbot/tests/cli_test.py

@@ -20,6 +20,9 @@ def reset_set_by_cli():
 
 class TestReadFile(unittest.TestCase):
     '''Test cli.read_file'''
+
+    _multiprocess_can_split_ = True
+
     def test_read_file(self):
         tmp_dir = tempfile.mkdtemp()
         rel_test_path = os.path.relpath(os.path.join(tmp_dir, 'foo'))
@@ -38,6 +41,8 @@ class TestReadFile(unittest.TestCase):
 class ParseTest(unittest.TestCase):
     '''Test the cli args entrypoint'''
 
+    _multiprocess_can_split_ = True
+
     @classmethod
     def setUpClass(cls):
         cls.plugins = disco.PluginsRegistry.find_all()
@@ -256,6 +261,8 @@ class ParseTest(unittest.TestCase):
 class DefaultTest(unittest.TestCase):
     """Tests for certbot.cli._Default."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         # pylint: disable=protected-access
         self.default1 = cli._Default()
@@ -275,6 +282,8 @@ class DefaultTest(unittest.TestCase):
 class SetByCliTest(unittest.TestCase):
     """Tests for certbot.set_by_cli and related functions."""
 
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         reload_module(cli)
 

certbot/tests/reporter_test.py

@@ -8,7 +8,6 @@ import six
 
 class ReporterTest(unittest.TestCase):
     """Tests for certbot.reporter.Reporter."""
-
     def setUp(self):
         from certbot import reporter
         self.reporter = reporter.Reporter(mock.MagicMock(quiet=False))
@@ -21,7 +20,7 @@ class ReporterTest(unittest.TestCase):
 
     def test_multiline_message(self):
         self.reporter.add_message("Line 1\nLine 2", self.reporter.LOW_PRIORITY)
-        self.reporter.atexit_print_messages()
+        self.reporter.print_messages()
         output = sys.stdout.getvalue()
         self.assertTrue("Line 1\n" in output)
         self.assertTrue("Line 2" in output)
@@ -39,9 +38,12 @@ class ReporterTest(unittest.TestCase):
             self.reporter.print_messages()
         self.assertEqual(sys.stdout.getvalue(), "")
 
-    def test_atexit_print_messages(self):
+    @mock.patch('certbot.reporter.os.getpid')
+    def test_atexit_print_messages(self, mock_getpid):
         self._add_messages()
-        self.reporter.atexit_print_messages()
+        mock_getpid.return_value = 42
+        with mock.patch('certbot.reporter.INITIAL_PID', 42):
+            self.reporter.atexit_print_messages()
         output = sys.stdout.getvalue()
         self.assertTrue("IMPORTANT NOTES:" in output)
         self.assertTrue("High" in output)

certbot/tests/storage_test.py

@@ -43,6 +43,8 @@ class BaseRenewableCertTest(unittest.TestCase):
     your test.  Check :class:`.cli_test.DuplicateCertTest` for an example.
 
     """
+    _multiprocess_can_split_ = True
+
     def setUp(self):
         from certbot import storage
         self.tempdir = tempfile.mkdtemp()

tox.cover.sh

@@ -36,8 +36,9 @@ cover () {
   # specific package, positional argument scopes tests only to
   # specific package directory; --cover-tests makes sure every tests
   # is run (c.f. #403)
-  nosetests -c /dev/null --with-cover --cover-tests --cover-package  \
+  nosetests -c /dev/null --with-cover --cover-tests --cover-package \
-            "$1" --cover-min-percentage="$min" "$1"
+            "$1" --cover-min-percentage="$min" "$1" --processes=-1 \
+            --process-timeout=100
 }
 
 rm -f .coverage  # --cover-erase is off, make sure stats are correct

tox.ini

@@ -14,15 +14,15 @@ envlist = modification,py{26,33,34,35},cover,lint
 # are detected, c.f. #1002
 commands =
     pip install -e acme[dns,dev]
-    nosetests -v acme
+    nosetests -v acme --processes=-1
     pip install -e .[dev]
-    nosetests -v certbot
+    nosetests -v certbot --processes=-1 --process-timeout=100
     pip install -e certbot-apache
-    nosetests -v certbot_apache
+    nosetests -v certbot_apache --processes=-1 --process-timeout=80
     pip install -e certbot-nginx
-    nosetests -v certbot_nginx
+    nosetests -v certbot_nginx --processes=-1
     pip install -e letshelp-certbot
-    nosetests -v letshelp_certbot
+    nosetests -v letshelp_certbot --processes=-1
 
 setenv =
     PYTHONPATH = {toxinidir}
@@ -45,23 +45,23 @@ deps =
 [testenv:py33]
 commands =
     pip install -e acme[dns,dev]
-    nosetests -v acme
+    nosetests -v acme --processes=-1
     pip install -e .[dev]
-    nosetests -v certbot
+    nosetests -v certbot --processes=-1 --process-timeout=100
 
 [testenv:py34]
 commands =
     pip install -e acme[dns,dev]
-    nosetests -v acme
+    nosetests -v acme --processes=-1
     pip install -e .[dev]
-    nosetests -v certbot
+    nosetests -v certbot --processes=-1 --process-timeout=100
 
 [testenv:py35]
 commands =
     pip install -e acme[dns,dev]
-    nosetests -v acme
+    nosetests -v acme --processes=-1
     pip install -e .[dev]
-    nosetests -v certbot
+    nosetests -v certbot --processes=-1 --process-timeout=100
 
 [testenv:cover]
 basepython = python2.7
@@ -78,12 +78,8 @@ basepython = python2.7
 commands =
     pip install -q -e acme[dns,dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot
     ./pep8.travis.sh
-    pylint --reports=n --rcfile=.pylintrc certbot
     pylint --reports=n --rcfile=acme/.pylintrc acme/acme
-    pylint --reports=n --rcfile=.pylintrc certbot-apache/certbot_apache
+    pylint -j 0 --reports=n --rcfile=.pylintrc certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot
-    pylint --reports=n --rcfile=.pylintrc certbot-nginx/certbot_nginx
-    pylint --reports=n --rcfile=.pylintrc certbot-compatibility-test/certbot_compatibility_test
-    pylint --reports=n --rcfile=.pylintrc letshelp-certbot/letshelp_certbot
 
 [testenv:apacheconftest]
 #basepython = python2.7