Add lockfile (#4449)

* add lock_file

* cleanup lock file

* Add LockFile tests

* add lock_dir

* add lock_dir_until_exit

* add set_up_core_dir and move lock_dir_until_exit

* Move lock_and_call to certbot.test.util

* Add lock to Apache

* Add lock to the Nginx plugin

* Improve permissions error message

* sort plugins

* add test_prepare_order

* provide more actionable permissions error

* Document and catch use of OSError

* don't lock a directory twice

* add conditional dependency on ordereddict

* Add lock_test

* expand sorted plugins comment

* Add lock_test to lint

* make make_lineage more conventional and flexible

* enhance lock_test.py

* add lock_test to tox

* Readd success message

* make py26 happy

* add test_acquire_without_deletion

certbot/tests/lock_test.py

@@ -0,0 +1,116 @@
+"""Tests for certbot.lock."""
+import functools
+import multiprocessing
+import os
+import unittest
+
+import mock
+
+from certbot import errors
+from certbot.tests import util as test_util
+
+
+class LockDirTest(test_util.TempDirTestCase):
+    """Tests for certbot.lock.lock_dir."""
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot.lock import lock_dir
+        return lock_dir(*args, **kwargs)
+
+    def test_it(self):
+        assert_raises = functools.partial(
+            self.assertRaises, errors.LockError, self._call, self.tempdir)
+        lock_path = os.path.join(self.tempdir, '.certbot.lock')
+        test_util.lock_and_call(assert_raises, lock_path)
+
+
+class LockFileTest(test_util.TempDirTestCase):
+    """Tests for certbot.lock.LockFile."""
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot.lock import LockFile
+        return LockFile(*args, **kwargs)
+
+    def setUp(self):
+        super(LockFileTest, self).setUp()
+        self.lock_path = os.path.join(self.tempdir, 'test.lock')
+
+    def test_acquire_without_deletion(self):
+        # acquire the lock in another process but don't delete the file
+        child = multiprocessing.Process(target=self._call,
+                                        args=(self.lock_path,))
+        child.start()
+        child.join()
+        self.assertEqual(child.exitcode, 0)
+        self.assertTrue(os.path.exists(self.lock_path))
+
+        # Test we're still able to properly acquire and release the lock
+        self.test_removed()
+
+    def test_contention(self):
+        assert_raises = functools.partial(
+            self.assertRaises, errors.LockError, self._call, self.lock_path)
+        test_util.lock_and_call(assert_raises, self.lock_path)
+
+    def test_locked_repr(self):
+        lock_file = self._call(self.lock_path)
+        locked_repr = repr(lock_file)
+        self._test_repr_common(lock_file, locked_repr)
+        self.assertTrue('acquired' in locked_repr)
+
+    def test_released_repr(self):
+        lock_file = self._call(self.lock_path)
+        lock_file.release()
+        released_repr = repr(lock_file)
+        self._test_repr_common(lock_file, released_repr)
+        self.assertTrue('released' in released_repr)
+
+    def _test_repr_common(self, lock_file, lock_repr):
+        self.assertTrue(lock_file.__class__.__name__ in lock_repr)
+        self.assertTrue(self.lock_path in lock_repr)
+
+    def test_race(self):
+        should_delete = [True, False]
+        stat = os.stat
+
+        def delete_and_stat(path):
+            """Wrap os.stat and maybe delete the file first."""
+            if path == self.lock_path and should_delete.pop(0):
+                os.remove(path)
+            return stat(path)
+
+        with mock.patch('certbot.lock.os.stat') as mock_stat:
+            mock_stat.side_effect = delete_and_stat
+            self._call(self.lock_path)
+        self.assertFalse(should_delete)
+
+    def test_removed(self):
+        lock_file = self._call(self.lock_path)
+        lock_file.release()
+        self.assertFalse(os.path.exists(self.lock_path))
+
+    @mock.patch('certbot.lock.fcntl.lockf')
+    def test_unexpected_lockf_err(self, mock_lockf):
+        msg = 'hi there'
+        mock_lockf.side_effect = IOError(msg)
+        try:
+            self._call(self.lock_path)
+        except IOError as err:
+            self.assertTrue(msg in str(err))
+        else:  # pragma: no cover
+            self.fail('IOError not raised')
+
+    @mock.patch('certbot.lock.os.stat')
+    def test_unexpected_stat_err(self, mock_stat):
+        msg = 'hi there'
+        mock_stat.side_effect = OSError(msg)
+        try:
+            self._call(self.lock_path)
+        except OSError as err:
+            self.assertTrue(msg in str(err))
+        else:  # pragma: no cover
+            self.fail('OSError not raised')
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

certbot/tests/main_test.py

@@ -792,12 +792,12 @@ class MainTest(test_util.TempDirTestCase):  # pylint: disable=too-many-public-me
                 print(lf.read())
 
     def test_renew_verb(self):
-        test_util.make_lineage(self, 'sample-renewal.conf')
+        test_util.make_lineage(self.config_dir, 'sample-renewal.conf')
         args = ["renew", "--dry-run", "-tvv"]
         self._test_renewal_common(True, [], args=args, should_renew=True)
 
     def test_quiet_renew(self):
-        test_util.make_lineage(self, 'sample-renewal.conf')
+        test_util.make_lineage(self.config_dir, 'sample-renewal.conf')
         args = ["renew", "--dry-run"]
         _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True)
         out = stdout.getvalue()
@@ -809,13 +809,13 @@ class MainTest(test_util.TempDirTestCase):  # pylint: disable=too-many-public-me
         self.assertEqual("", out)
 
     def test_renew_hook_validation(self):
-        test_util.make_lineage(self, 'sample-renewal.conf')
+        test_util.make_lineage(self.config_dir, 'sample-renewal.conf')
         args = ["renew", "--dry-run", "--post-hook=no-such-command"]
         self._test_renewal_common(True, [], args=args, should_renew=False,
                                   error_expected=True)
 
     def test_renew_no_hook_validation(self):
-        test_util.make_lineage(self, 'sample-renewal.conf')
+        test_util.make_lineage(self.config_dir, 'sample-renewal.conf')
         args = ["renew", "--dry-run", "--post-hook=no-such-command",
                 "--disable-hook-validation"]
         with mock.patch("certbot.hooks.post_hook"):
@@ -825,7 +825,8 @@ class MainTest(test_util.TempDirTestCase):  # pylint: disable=too-many-public-me
     @mock.patch("certbot.cli.set_by_cli")
     def test_ancient_webroot_renewal_conf(self, mock_set_by_cli):
         mock_set_by_cli.return_value = False
-        rc_path = test_util.make_lineage(self, 'sample-renewal-ancient.conf')
+        rc_path = test_util.make_lineage(
+            self.config_dir, 'sample-renewal-ancient.conf')
         args = mock.MagicMock(account=None, config_dir=self.config_dir,
                               logs_dir=self.logs_dir, work_dir=self.work_dir,
                               email=None, webroot_path=None)
@@ -846,7 +847,7 @@ class MainTest(test_util.TempDirTestCase):  # pylint: disable=too-many-public-me
         self._test_renewal_common(False, [], args=args, should_renew=False, error_expected=True)
 
     def test_renew_with_certname(self):
-        test_util.make_lineage(self, 'sample-renewal.conf')
+        test_util.make_lineage(self.config_dir, 'sample-renewal.conf')
         self._test_renewal_common(True, [], should_renew=True,
             args=['renew', '--dry-run', '--cert-name', 'sample-renewal'])
 

certbot/tests/renewal_test.py

@@ -21,7 +21,8 @@ class RenewalTest(util.TempDirTestCase):
     @mock.patch('certbot.cli.set_by_cli')
     def test_ancient_webroot_renewal_conf(self, mock_set_by_cli):
         mock_set_by_cli.return_value = False
-        rc_path = util.make_lineage(self, 'sample-renewal-ancient.conf')
+        rc_path = util.make_lineage(
+            self.config_dir, 'sample-renewal-ancient.conf')
         args = mock.MagicMock(account=None, config_dir=self.config_dir,
                               logs_dir="logs", work_dir="work",
                               email=None, webroot_path=None)

certbot/tests/util.py

@@ -3,6 +3,7 @@
 .. warning:: This module is not part of the public API.
 
 """
+import multiprocessing
 import os
 import pkg_resources
 import shutil
@@ -13,12 +14,14 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import mock
 import OpenSSL
+from six.moves import reload_module  # pylint: disable=import-error
 
 from acme import jose
 
 from certbot import constants
 from certbot import interfaces
 from certbot import storage
+from certbot import util
 
 from certbot.display import util as display_util
 
@@ -106,12 +109,13 @@ def skip_unless(condition, reason):  # pragma: no cover
         return lambda cls: None
 
 
-def make_lineage(self, testfile):
+def make_lineage(config_dir, testfile):
     """Creates a lineage defined by testfile.
 
     This creates the archive, live, and renewal directories if
     necessary and creates a simple lineage.
 
+    :param str config_dir: path to the configuration directory
     :param str testfile: configuration file to base the lineage on
 
     :returns: path to the renewal conf file for the created lineage
@@ -121,11 +125,11 @@ def make_lineage(self, testfile):
     lineage_name = testfile[:-len('.conf')]
 
     conf_dir = os.path.join(
-        self.config_dir, constants.RENEWAL_CONFIGS_DIR)
+        config_dir, constants.RENEWAL_CONFIGS_DIR)
     archive_dir = os.path.join(
-        self.config_dir, constants.ARCHIVE_DIR, lineage_name)
+        config_dir, constants.ARCHIVE_DIR, lineage_name)
     live_dir = os.path.join(
-        self.config_dir, constants.LIVE_DIR, lineage_name)
+        config_dir, constants.LIVE_DIR, lineage_name)
 
     for directory in (archive_dir, conf_dir, live_dir,):
         if not os.path.exists(directory):
@@ -140,11 +144,11 @@ def make_lineage(self, testfile):
         os.symlink(os.path.join(archive_dir, '{0}1.pem'.format(kind)),
                    os.path.join(live_dir, '{0}.pem'.format(kind)))
 
-    conf_path = os.path.join(self.config_dir, conf_dir, testfile)
+    conf_path = os.path.join(config_dir, conf_dir, testfile)
     with open(vector_path(testfile)) as src:
         with open(conf_path, 'w') as dst:
             dst.writelines(
-                line.replace('MAGICDIR', self.config_dir) for line in src)
+                line.replace('MAGICDIR', config_dir) for line in src)
 
     return conf_path
 
@@ -241,3 +245,47 @@ class TempDirTestCase(unittest.TestCase):
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
+
+
+def lock_and_call(func, lock_path):
+    """Grab a lock for lock_path and call func.
+
+    :param callable func: object to call after acquiring the lock
+    :param str lock_path: path to file or directory to lock
+
+    """
+    # Reload module to reset internal _LOCKS dictionary
+    reload_module(util)
+
+    # start child and wait for it to grab the lock
+    cv = multiprocessing.Condition()
+    cv.acquire()
+    child_args = (cv, lock_path,)
+    child = multiprocessing.Process(target=hold_lock, args=child_args)
+    child.start()
+    cv.wait()
+
+    # call func and terminate the child
+    func()
+    cv.notify()
+    cv.release()
+    child.join()
+    assert child.exitcode == 0
+
+
+def hold_lock(cv, lock_path):  # pragma: no cover
+    """Acquire a file lock at lock_path and wait to release it.
+
+    :param multiprocessing.Condition cv: condition for syncronization
+    :param str lock_path: path to the file lock
+
+    """
+    from certbot import lock
+    if os.path.isdir(lock_path):
+        my_lock = lock.lock_dir(lock_path)
+    else:
+        my_lock = lock.LockFile(lock_path)
+    cv.acquire()
+    cv.notify()
+    cv.wait()
+    my_lock.release()

certbot/tests/util_test.py

@@ -2,11 +2,13 @@
 import argparse
 import errno
 import os
+import shutil
 import stat
 import unittest
 
 import mock
 import six
+from six.moves import reload_module  # pylint: disable=import-error
 
 from certbot import errors
 import certbot.tests.util as test_util
@@ -73,19 +75,52 @@ class ExeExistsTest(unittest.TestCase):
         self.assertFalse(self._call("exe"))
 
 
-class MakeOrVerifyCoreDirTest(test_util.TempDirTestCase):
+class LockDirUntilExit(test_util.TempDirTestCase):
+    """Tests for certbot.util.lock_dir_until_exit."""
+    @classmethod
+    def _call(cls, *args, **kwargs):
+        from certbot.util import lock_dir_until_exit
+        return lock_dir_until_exit(*args, **kwargs)
+
+    def setUp(self):
+        super(LockDirUntilExit, self).setUp()
+        # reset global state from other tests
+        import certbot.util
+        reload_module(certbot.util)
+
+    @mock.patch('certbot.util.logger')
+    @mock.patch('certbot.util.atexit_register')
+    def test_it(self, mock_register, mock_logger):
+        subdir = os.path.join(self.tempdir, 'subdir')
+        os.mkdir(subdir)
+        self._call(self.tempdir)
+        self._call(subdir)
+        self._call(subdir)
+
+        self.assertEqual(mock_register.call_count, 1)
+        registered_func = mock_register.call_args[0][0]
+        shutil.rmtree(subdir)
+        registered_func()  # exception not raised
+        # logger.debug is only called once because the second call
+        # to lock subdir was ignored because it was already locked
+        self.assertEqual(mock_logger.debug.call_count, 1)
+
+
+class SetUpCoreDirTest(test_util.TempDirTestCase):
     """Tests for certbot.util.make_or_verify_core_dir."""
 
     def _call(self, *args, **kwargs):
-        from certbot.util import make_or_verify_core_dir
-        return make_or_verify_core_dir(*args, **kwargs)
+        from certbot.util import set_up_core_dir
+        return set_up_core_dir(*args, **kwargs)
 
-    def test_success(self):
+    @mock.patch('certbot.util.lock_dir_until_exit')
+    def test_success(self, mock_lock):
         new_dir = os.path.join(self.tempdir, 'new')
         self._call(new_dir, 0o700, os.geteuid(), False)
         self.assertTrue(os.path.exists(new_dir))
+        self.assertEqual(mock_lock.call_count, 1)
 
-    @mock.patch('certbot.main.util.make_or_verify_dir')
+    @mock.patch('certbot.util.make_or_verify_dir')
     def test_failure(self, mock_make_or_verify):
         mock_make_or_verify.side_effect = OSError
         self.assertRaises(errors.Error, self._call,