Merge remote-tracking branch 'upstream/master' into plugin_tests

.travis.yml

@@ -22,6 +22,11 @@ env:
     - TOXENV=lint
     - TOXENV=cover
 
+# make sure simplehttp simple verification works (custom /etc/hosts)
+addons:
+  hosts:
+    - le.wtf
+
 install: "travis_retry pip install tox coveralls"
 before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh amqp'
 script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))'

letsencrypt/auth_handler.py

@@ -153,13 +153,18 @@ class AuthHandler(object):
         """
         active_achalls = []
         for achall, resp in itertools.izip(achalls, resps):
+            # XXX: make sure that all achalls, including those
+            # corresponding to None or False returned from
+            # Authenticator are removed from the queue and thus avoid
+            # infinite loop
+            active_achalls.append(achall)
+
             # Don't send challenges for None and False authenticator responses
-            if resp:
+            if resp is not None and resp:
                 self.acme.answer_challenge(achall.challb, resp)
                 # TODO: answer_challenge returns challr, with URI,
                 # that can be used in _find_updated_challr
                 # comparisons...
-                active_achalls.append(achall)
                 if achall.domain in chall_update:
                     chall_update[achall.domain].append(achall)
                 else:

letsencrypt/plugins/manual.py

@@ -1,6 +1,12 @@
 """Manual plugin."""
 import os
+import logging
+import shutil
+import signal
+import subprocess
 import sys
+import tempfile
+import time
 
 import zope.component
 import zope.interface
@@ -8,10 +14,14 @@ import zope.interface
 from acme import challenges
 from acme import jose
 
+from letsencrypt import errors
 from letsencrypt import interfaces
 from letsencrypt.plugins import common
 
 
+logger = logging.getLogger(__name__)
+
+
 class ManualAuthenticator(common.Plugin):
     """Manual Authenticator.
 
@@ -43,8 +53,8 @@ command on the target server (as root):
     # anything recursively under the cwd
 
     HTTP_TEMPLATE = """\
-mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH}
+mkdir -p {root}/public_html/{response.URI_ROOT_PATH}
-cd /tmp/letsencrypt/public_html
+cd {root}/public_html
 echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path}
 # run only once per server:
 python -c "import BaseHTTPServer, SimpleHTTPServer; \\
@@ -55,8 +65,8 @@ s.serve_forever()" """
 
     # https://www.piware.de/2011/01/creating-an-https-server-in-python/
     HTTPS_TEMPLATE = """\
-mkdir -p /tmp/letsencrypt/public_html/{response.URI_ROOT_PATH}
+mkdir -p {root}/public_html/{response.URI_ROOT_PATH}
-cd /tmp/letsencrypt/public_html
+cd {root}/public_html
 echo -n {achall.token} > {response.URI_ROOT_PATH}/{response.path}
 # run only once per server:
 openssl req -new -newkey rsa:4096 -subj "/" -days 1 -nodes -x509 -keyout ../key.pem -out ../cert.pem
@@ -77,6 +87,15 @@ s.serve_forever()" """
         super(ManualAuthenticator, self).__init__(*args, **kwargs)
         self.template = (self.HTTP_TEMPLATE if self.config.no_simple_http_tls
                          else self.HTTPS_TEMPLATE)
+        self._root = (tempfile.mkdtemp() if self.conf("test-mode")
+                      else "/tmp/letsencrypt")
+        self._httpd = None
+
+    @classmethod
+    def add_parser_arguments(cls, add):
+        add("test-mode", action="store_true",
+            help="Test mode. Executes the manual command in subprocess. "
+            "Requires openssl to be installed unless --no-simple-http-tls.")
 
     def prepare(self):  # pylint: disable=missing-docstring,no-self-use
         pass  # pragma: no cover
@@ -110,17 +129,44 @@ binary for temporary key/certificate generation.""".replace("\n", "")
             tls=(not self.config.no_simple_http_tls))
         assert response.good_path  # is encoded os.urandom(18) good?
 
+        command = self.template.format(
+            root=self._root, achall=achall, response=response,
+            ct=response.CONTENT_TYPE, port=(
+                response.port if self.config.simple_http_port is None
+                else self.config.simple_http_port))
+        if self.conf("test-mode"):
+            logger.debug("Test mode. Executing the manual command: %s", command)
+            try:
+                self._httpd = subprocess.Popen(
+                    command,
+                    # don't care about setting stdout and stderr,
+                    # we're in test mode anyway
+                    shell=True,
+                    # "preexec_fn" is UNIX specific, but so is "command"
+                    preexec_fn=os.setsid)
+            except OSError as error:  # ValueError should not happen!
+                logger.debug(
+                    "Couldn't execute manual command: %s", error, exc_info=True)
+                return False
+            logger.debug("Manual command running as PID %s.", self._httpd.pid)
+            # give it some time to bootstrap, before we try to verify
+            # (cert generation in case of simpleHttpS might take time)
+            time.sleep(4)  # XXX
+            if self._httpd.poll() is not None:
+                raise errors.Error("Couldn't execute manual command")
+        else:
             self._notify_and_wait(self.MESSAGE_TEMPLATE.format(
-            achall=achall, response=response, uri=response.uri(achall.domain),
+                achall=achall, response=response,
-            ct=response.CONTENT_TYPE, command=self.template.format(
+                uri=response.uri(achall.domain), ct=response.CONTENT_TYPE,
-                achall=achall, response=response, ct=response.CONTENT_TYPE,
+                command=command))
-                port=(response.port if self.config.simple_http_port is None
-                      else self.config.simple_http_port))))
 
         if response.simple_verify(
                 achall.challb, achall.domain, self.config.simple_http_port):
             return response
         else:
+            if self.conf("test-mode") and self._httpd.poll() is not None:
+                # simply verify cause command failure...
+                return False
             return None
 
     def _notify_and_wait(self, message):  # pylint: disable=no-self-use
@@ -130,5 +176,15 @@ binary for temporary key/certificate generation.""".replace("\n", "")
         sys.stdout.write(message)
         raw_input("Press ENTER to continue")
 
-    def cleanup(self, achalls):  # pylint: disable=missing-docstring,no-self-use
+    def cleanup(self, achalls):
-        pass  # pragma: no cover
+        # pylint: disable=missing-docstring,no-self-use,unused-argument
+        if self.conf("test-mode"):
+            assert self._httpd is not None, (
+                "cleanup() must be called after perform()")
+            if self._httpd.poll() is None:
+                logger.debug("Terminating manual command process")
+                os.killpg(self._httpd.pid, signal.SIGTERM)
+            else:
+                logger.debug("Manual command process already terminated "
+                             "with %s code", self._httpd.returncode)
+            shutil.rmtree(self._root)

letsencrypt/plugins/manual_test.py

@@ -1,4 +1,5 @@
 """Tests for letsencrypt.plugins.manual."""
+import signal
 import unittest
 
 import mock
@@ -6,6 +7,8 @@ import mock
 from acme import challenges
 
 from letsencrypt import achallenges
+from letsencrypt import errors
+
 from letsencrypt.tests import acme_util
 
 
@@ -15,11 +18,18 @@ class ManualAuthenticatorTest(unittest.TestCase):
     def setUp(self):
         from letsencrypt.plugins.manual import ManualAuthenticator
         self.config = mock.MagicMock(
-            no_simple_http_tls=True, simple_http_port=4430)
+            no_simple_http_tls=True, simple_http_port=4430,
+            manual_test_mode=False)
         self.auth = ManualAuthenticator(config=self.config, name="manual")
         self.achalls = [achallenges.SimpleHTTP(
             challb=acme_util.SIMPLE_HTTP, domain="foo.com", key=None)]
 
+        config_test_mode = mock.MagicMock(
+            no_simple_http_tls=True, simple_http_port=4430,
+            manual_test_mode=True)
+        self.auth_test_mode = ManualAuthenticator(
+            config=config_test_mode, name="manual")
+
     def test_more_info(self):
         self.assertTrue(isinstance(self.auth.more_info(), str))
 
@@ -51,6 +61,45 @@ class ManualAuthenticatorTest(unittest.TestCase):
         mock_verify.return_value = False
         self.assertEqual([None], self.auth.perform(self.achalls))
 
+    @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True)
+    def test_perform_test_command_oserror(self, mock_popen):
+        mock_popen.side_effect = OSError
+        self.assertEqual([False], self.auth_test_mode.perform(self.achalls))
+
+    @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True)
+    @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True)
+    def test_perform_test_command_run_failure(
+            self, mock_popen, unused_mock_sleep):
+        mock_popen.poll.return_value = 10
+        mock_popen.return_value.pid = 1234
+        self.assertRaises(
+            errors.Error, self.auth_test_mode.perform, self.achalls)
+
+    @mock.patch("letsencrypt.plugins.manual.time.sleep", autospec=True)
+    @mock.patch("acme.challenges.SimpleHTTPResponse.simple_verify",
+                autospec=True)
+    @mock.patch("letsencrypt.plugins.manual.subprocess.Popen", autospec=True)
+    def test_perform_test_mode(self, mock_popen, mock_verify, mock_sleep):
+        mock_popen.return_value.poll.side_effect = [None, 10]
+        mock_popen.return_value.pid = 1234
+        mock_verify.return_value = False
+        self.assertEqual([False], self.auth_test_mode.perform(self.achalls))
+        self.assertEqual(1, mock_sleep.call_count)
+
+    def test_cleanup_test_mode_already_terminated(self):
+        # pylint: disable=protected-access
+        self.auth_test_mode._httpd = httpd = mock.Mock()
+        httpd.poll.return_value = 0
+        self.auth_test_mode.cleanup(self.achalls)
+
+    @mock.patch("letsencrypt.plugins.manual.os.killpg", autospec=True)
+    def test_cleanup_test_mode_kills_still_running(self, mock_killpg):
+        # pylint: disable=protected-access
+        self.auth_test_mode._httpd = httpd = mock.Mock(pid=1234)
+        httpd.poll.return_value = None
+        self.auth_test_mode.cleanup(self.achalls)
+        mock_killpg.assert_called_once_with(1234, signal.SIGTERM)
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

letsencrypt/plugins/standalone/authenticator.py

@@ -196,6 +196,10 @@ class StandaloneAuthenticator(common.Plugin):
         """
         signal.signal(signal.SIGINT, self.subproc_signal_handler)
         self.sock = socket.socket()
+        # SO_REUSEADDR flag tells the kernel to reuse a local socket
+        # in TIME_WAIT state, without waiting for its natural timeout
+        # to expire.
+        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         try:
             self.sock.bind(("0.0.0.0", port))
         except socket.error, error:

tests/boulder-integration.sh

@@ -23,6 +23,8 @@ common() {
 
 common --domains le1.wtf auth
 common --domains le2.wtf run
+common -a manual -d le.wtf auth
+common -a manual -d le.wtf --no-simple-http-tls auth
 
 export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \
        OPENSSL_CNF=examples/openssl.cnf

tests/integration/_common.sh

@@ -10,11 +10,12 @@ store_flags="$store_flags --logs-dir $root/logs"
 export root store_flags
 
 letsencrypt_test () {
-    # first three flags required, rest is handy defaults
     letsencrypt \
         --server "${SERVER:-http://localhost:4000/acme/new-reg}" \
         --no-verify-ssl \
         --dvsni-port 5001 \
+        --simple-http-port 5001 \
+        --manual-test-mode \
         $store_flags \
         --text \
         --agree-eula \