Merge remote-tracking branch 'origin/master' into multi-topic-help

certbot/plugins/disco.py

@@ -3,6 +3,7 @@ import collections
 import itertools
 import logging
 import pkg_resources
+import six
 
 import zope.interface
 import zope.interface.verify
@@ -194,12 +195,12 @@ class PluginsRegistry(collections.Mapping):
     def init(self, config):
         """Initialize all plugins in the registry."""
         return [plugin_ep.init(config) for plugin_ep
-                in self._plugins.itervalues()]
+                in six.itervalues(self._plugins)]
 
     def filter(self, pred):
         """Filter plugins based on predicate."""
         return type(self)(dict((name, plugin_ep) for name, plugin_ep
-                               in self._plugins.iteritems() if pred(plugin_ep)))
+                               in six.iteritems(self._plugins) if pred(plugin_ep)))
 
     def visible(self):
         """Filter plugins based on visibility."""
@@ -216,7 +217,7 @@ class PluginsRegistry(collections.Mapping):
 
     def prepare(self):
         """Prepare all plugins in the registry."""
-        return [plugin_ep.prepare() for plugin_ep in self._plugins.itervalues()]
+        return [plugin_ep.prepare() for plugin_ep in six.itervalues(self._plugins)]
 
     def available(self):
         """Filter plugins based on availability."""
@@ -238,7 +239,7 @@ class PluginsRegistry(collections.Mapping):
 
         """
         # use list instead of set because PluginEntryPoint is not hashable
-        candidates = [plugin_ep for plugin_ep in self._plugins.itervalues()
+        candidates = [plugin_ep for plugin_ep in six.itervalues(self._plugins)
                       if plugin_ep.initialized and plugin_ep.init() is plugin]
         assert len(candidates) <= 1
         if candidates:
@@ -249,7 +250,7 @@ class PluginsRegistry(collections.Mapping):
     def __repr__(self):
         return "{0}({1})".format(
             self.__class__.__name__, ','.join(
-                repr(p_ep) for p_ep in self._plugins.itervalues()))
+                repr(p_ep) for p_ep in six.itervalues(self._plugins)))
 
     def __str__(self):
         if not self._plugins:

certbot/plugins/manual.py

@@ -10,6 +10,7 @@ import sys
 import tempfile
 import time
 
+import six
 import zope.component
 import zope.interface
 
@@ -187,7 +188,7 @@ s.serve_forever()" """
         #answer = zope.component.getUtility(interfaces.IDisplay).notification(
         #    message=message, height=25, pause=True)
         sys.stdout.write(message)
-        raw_input("Press ENTER to continue")
+        six.moves.input("Press ENTER to continue")
 
     def cleanup(self, achalls):
         # pylint: disable=missing-docstring,no-self-use,unused-argument

certbot/plugins/selection.py

@@ -84,7 +84,7 @@ def pick_plugin(config, default, plugins, question, ifaces):
         else:
             return plugin_ep.init()
     elif len(prepared) == 1:
-        plugin_ep = prepared.values()[0]
+        plugin_ep = list(prepared.values())[0]
         logger.debug("Single candidate plugin: %s", plugin_ep)
         if plugin_ep.misconfigured:
             return None
@@ -174,7 +174,7 @@ def choose_configurator_plugins(config, plugins, verb):
     if verb == "install":
         need_inst = True
         if config.authenticator:
-            logger.warn("Specifying an authenticator doesn't make sense in install mode")
+            logger.warning("Specifying an authenticator doesn't make sense in install mode")
 
     # Try to meet the user's request and/or ask them to pick plugins
     authenticator = installer = None

certbot/plugins/util.py

@@ -3,15 +3,27 @@ import logging
 import os
 import socket
 
-import psutil
 import zope.component
 
 from certbot import interfaces
 from certbot import util
 
+try:
+    import psutil
+    USE_PSUTIL = True
+except ImportError:
+    USE_PSUTIL = False
 
 logger = logging.getLogger(__name__)
 
+RENEWER_EXTRA_MSG = (
+    " For automated renewal, you may want to use a script that stops"
+    " and starts your webserver. You can find an example at"
+    " https://certbot.eff.org/docs/using.html#renewal ."
+    " Alternatively you can use the webroot plugin to renew without"
+    " needing to stop and start your webserver.")
+
+
 def path_surgery(restart_cmd):
     """Attempt to perform PATH surgery to find restart_cmd
 
@@ -38,9 +50,11 @@ def path_surgery(restart_cmd):
         return True
     else:
         expanded = " expanded" if any(added) else ""
-        logger.warn("Failed to find %s in%s PATH: %s", restart_cmd, expanded, path)
+        logger.warning("Failed to find %s in%s PATH: %s", restart_cmd,
+                       expanded, path)
         return False
 
+
 def already_listening(port, renewer=False):
     """Check if a process is already listening on the port.
 
@@ -53,6 +67,50 @@ def already_listening(port, renewer=False):
     :param int port: The TCP port in question.
     :returns: True or False.
 
+    """
+
+    if USE_PSUTIL:
+        return already_listening_psutil(port, renewer=renewer)
+    else:
+        logger.debug("Psutil not found, using simple socket check.")
+        return already_listening_socket(port, renewer=renewer)
+
+
+def already_listening_socket(port, renewer=False):
+    """Simple socket based check to find out if port is already in use
+
+    :param int port: The TCP port in question.
+    :returns: True or False
+    """
+
+    try:
+        testsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+        try:
+            testsocket.bind(("", port))
+        except socket.error:
+            display = zope.component.getUtility(interfaces.IDisplay)
+            extra = ""
+            if renewer:
+                extra = RENEWER_EXTRA_MSG
+            display.notification(
+                "Port {0} is already in use by another process. This will "
+                "prevent us from binding to that port. Please stop the "
+                "process that is populating the port in question and try "
+                "again. {1}".format(port, extra), height=13)
+            return True
+        finally:
+            testsocket.close()
+    except socket.error:
+        pass
+    return False
+
+
+def already_listening_psutil(port, renewer=False):
+    """Psutil variant of the open port check
+
+    :param int port: The TCP port in question.
+    :returns: True or False.
+
     """
     try:
         net_connections = psutil.net_connections()
@@ -81,12 +139,7 @@ def already_listening(port, renewer=False):
             display = zope.component.getUtility(interfaces.IDisplay)
             extra = ""
             if renewer:
-                extra = (
-                    " For automated renewal, you may want to use a script that stops"
-                    " and starts your webserver. You can find an example at"
-                    " https://letsencrypt.org/howitworks/#writing-your-own-renewal-script"
-                    ". Alternatively you can use the webroot plugin to renew without"
-                    " needing to stop and start your webserver.")
+                extra = RENEWER_EXTRA_MSG
             display.notification(
                 "The program {0} (process ID {1}) is already listening "
                 "on TCP port {2}. This will prevent us from binding to "

certbot/plugins/util_test.py

@@ -1,14 +1,22 @@
 """Tests for certbot.plugins.util."""
 import os
 import unittest
+import sys
 
 import mock
-import psutil
+
+try:
+    # Python 3.5+
+    from importlib import reload as refresh  # pylint: disable=no-name-in-module
+except ImportError:
+    # Python 2-3.4
+    from imp import reload as refresh
+
 
 class PathSurgeryTest(unittest.TestCase):
     """Tests for certbot.plugins.path_surgery."""
 
-    @mock.patch("certbot.plugins.util.logger.warn")
+    @mock.patch("certbot.plugins.util.logger.warning")
     @mock.patch("certbot.plugins.util.logger.debug")
     def test_path_surgery(self, mock_debug, mock_warn):
         from certbot.plugins.util import path_surgery
@@ -16,20 +24,64 @@ class PathSurgeryTest(unittest.TestCase):
         with mock.patch.dict('os.environ', all_path):
             with mock.patch('certbot.util.exe_exists') as mock_exists:
                 mock_exists.return_value = True
-                self.assertEquals(path_surgery("eg"), True)
-                self.assertEquals(mock_debug.call_count, 0)
-                self.assertEquals(mock_warn.call_count, 0)
-                self.assertEquals(os.environ["PATH"], all_path["PATH"])
+                self.assertEqual(path_surgery("eg"), True)
+                self.assertEqual(mock_debug.call_count, 0)
+                self.assertEqual(mock_warn.call_count, 0)
+                self.assertEqual(os.environ["PATH"], all_path["PATH"])
         no_path = {"PATH": "/tmp/"}
         with mock.patch.dict('os.environ', no_path):
             path_surgery("thingy")
-            self.assertEquals(mock_debug.call_count, 1)
-            self.assertEquals(mock_warn.call_count, 1)
+            self.assertEqual(mock_debug.call_count, 1)
+            self.assertEqual(mock_warn.call_count, 1)
             self.assertTrue("Failed to find" in mock_warn.call_args[0][0])
             self.assertTrue("/usr/local/bin" in os.environ["PATH"])
             self.assertTrue("/tmp" in os.environ["PATH"])
 
-class AlreadyListeningTest(unittest.TestCase):
+
+class AlreadyListeningTestNoPsutil(unittest.TestCase):
+    """Tests for certbot.plugins.already_listening when
+    psutil is not available"""
+    def setUp(self):
+        import certbot.plugins.util
+        # Ensure we get importerror
+        self.psutil = None
+        if "psutil" in sys.modules:
+            self.psutil = sys.modules['psutil']
+        sys.modules['psutil'] = None
+        # Reload hackery to ensure getting non-psutil version
+        # loaded to memory
+        refresh(certbot.plugins.util)
+
+    def tearDown(self):
+        # Need to reload the module to ensure
+        # getting back to normal
+        import certbot.plugins.util
+        sys.modules["psutil"] = self.psutil
+        refresh(certbot.plugins.util)
+
+    @mock.patch("certbot.plugins.util.zope.component.getUtility")
+    def test_ports_available(self, mock_getutil):
+        import certbot.plugins.util as plugins_util
+        # Ensure we don't get error
+        with mock.patch("socket._socketobject.bind"):
+            self.assertFalse(plugins_util.already_listening(80))
+            self.assertFalse(plugins_util.already_listening(80, True))
+            self.assertEqual(mock_getutil.call_count, 0)
+
+    @mock.patch("certbot.plugins.util.zope.component.getUtility")
+    def test_ports_blocked(self, mock_getutil):
+        sys.modules["psutil"] = None
+        import certbot.plugins.util as plugins_util
+        import socket
+        with mock.patch("socket._socketobject.bind", side_effect=socket.error):
+            self.assertTrue(plugins_util.already_listening(80))
+            self.assertTrue(plugins_util.already_listening(80, True))
+        with mock.patch("socket.socket", side_effect=socket.error):
+            self.assertFalse(plugins_util.already_listening(80))
+        self.assertEqual(mock_getutil.call_count, 2)
+
+
+class AlreadyListeningTestPsutil(unittest.TestCase):
     """Tests for certbot.plugins.already_listening."""
     def _call(self, *args, **kwargs):
         from certbot.plugins.util import already_listening
@@ -42,6 +94,7 @@ class AlreadyListeningTest(unittest.TestCase):
         # This tests a race condition, or permission problem, or OS
         # incompatibility in which, for some reason, no process name can be
         # found to match the identified listening PID.
+        import psutil
         from psutil._common import sconn
         conns = [
             sconn(fd=-1, family=2, type=1, laddr=("0.0.0.0", 30),
@@ -94,7 +147,7 @@ class AlreadyListeningTest(unittest.TestCase):
                   raddr=(), status="LISTEN", pid=4416)]
         mock_net.return_value = conns
         mock_process.name.return_value = "inetd"
-        result = self._call(17)
+        result = self._call(17, True)
         self.assertTrue(result)
         self.assertEqual(mock_get_utility.call_count, 1)
         mock_process.assert_called_once_with(4416)
@@ -124,6 +177,7 @@ class AlreadyListeningTest(unittest.TestCase):
 
     @mock.patch("certbot.plugins.util.psutil.net_connections")
     def test_access_denied_exception(self, mock_net):
+        import psutil
         mock_net.side_effect = psutil.AccessDenied("")
         self.assertFalse(self._call(12345))