Merge remote-tracking branch 'upstream/master' into multios_apache

# Conflicts:
#	letsencrypt-apache/letsencrypt_apache/configurator.py
#	letsencrypt-apache/letsencrypt_apache/parser.py
#	letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py

Dockerfile

@@ -49,7 +49,6 @@ COPY letsencrypt-apache /opt/letsencrypt/src/letsencrypt-apache/
 COPY letsencrypt-nginx /opt/letsencrypt/src/letsencrypt-nginx/
 
 
-# py26reqs.txt not installed!
 RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \
     /opt/letsencrypt/venv/bin/pip install \
     -e /opt/letsencrypt/src/acme \

Dockerfile-dev

@@ -32,7 +32,6 @@ RUN /opt/letsencrypt/src/ubuntu.sh && \
 # the above is not likely to change, so by putting it further up the
 # Dockerfile we make sure we cache as much as possible
 
-# py26reqs.txt not installed!
 COPY setup.py README.rst CHANGES.rst MANIFEST.in linter_plugin.py tox.cover.sh tox.ini pep8.travis.sh .pep8 .pylintrc /opt/letsencrypt/src/
 
 # all above files are necessary for setup.py, however, package source

MANIFEST.in

@@ -1,4 +1,3 @@
-include py26reqs.txt
 include README.rst
 include CHANGES.rst
 include CONTRIBUTING.md

acme/acme/__init__.py

@@ -1,12 +1,12 @@
 """ACME protocol implementation.
 
 This module is an implementation of the `ACME protocol`_. Latest
-supported version: `v02`_.
+supported version: `draft-ietf-acme-01`_.
 
-.. _`ACME protocol`: https://github.com/letsencrypt/acme-spec
+.. _`ACME protocol`: https://github.com/ietf-wg-acme/acme/
 
-.. _`v02`:
+.. _`draft-ietf-acme-01`:
-  https://github.com/letsencrypt/acme-spec/commit/d328fea2d507deb9822793c512830d827a4150c4
+  https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01
 
 
 """

acme/setup.py

@@ -10,8 +10,6 @@ install_requires = [
     # load_pem_private/public_key (>=0.6)
     # rsa_recover_prime_factors (>=0.8)
     'cryptography>=0.8',
-    'ndg-httpsclient',  # urllib3 InsecurePlatformWarning (#304)
-    'pyasn1',  # urllib3 InsecurePlatformWarning (#304)
     # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15)
     'PyOpenSSL>=0.15',
     'pyrfc3339',
@@ -32,6 +30,11 @@ if sys.version_info < (2, 7):
 else:
     install_requires.append('mock')
 
+if sys.version_info < (2, 7, 9):
+    # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning)
+    install_requires.append('ndg-httpsclient')
+    install_requires.append('pyasn1')
+
 docs_extras = [
     'Sphinx>=1.0',  # autodoc_member_order = 'bysource', autodoc_default_flags
     'sphinx_rtd_theme',

bootstrap/README

@@ -2,6 +2,5 @@ This directory contains scripts that install necessary OS-specific
 prerequisite dependencies (see docs/using.rst).
 
 General dependencies:
-- git-core: py26reqs.txt git+https://*
 - ca-certificates: communication with demo ACMO server at
-  https://www.letsencrypt-demo.org, py26reqs.txt git+https://*
+  https://www.letsencrypt-demo.org

bootstrap/_arch_common.sh

@@ -8,7 +8,6 @@
 # ./bootstrap/dev/_common_venv.sh
 
 deps="
-  git
   python2
   python-virtualenv
   gcc

bootstrap/_deb_common.sh

@@ -24,26 +24,56 @@ apt-get update
 # distro version (#346)
 
 virtualenv=
-if apt-cache show virtualenv > /dev/null ; then
+if apt-cache show virtualenv > /dev/null 2>&1; then
   virtualenv="virtualenv"
 fi
 
-if apt-cache show python-virtualenv > /dev/null ; then
+if apt-cache show python-virtualenv > /dev/null 2>&1; then
   virtualenv="$virtualenv python-virtualenv"
 fi
 
+augeas_pkg=libaugeas0
+AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2`
+
+if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then
+    if lsb_release -a | grep -q wheezy ; then
+        if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q wheezy-backports ; then
+            # This can theoretically error if sources.list.d is empty, but in that case we don't care.
+            if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q wheezy-backports ; then
+                /bin/echo -n "Installing augeas from wheezy-backports in 3 seconds..."
+                sleep 1s
+                /bin/echo -ne "\e[0K\rInstalling augeas from wheezy-backports in 2 seconds..."
+                sleep 1s
+                /bin/echo -e "\e[0K\rInstalling augeas from wheezy-backports in 1 second ..."
+                sleep 1s
+                /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")'
+
+                echo deb http://http.debian.net/debian wheezy-backports main >> /etc/apt/sources.list.d/wheezy-backports.list
+                apt-get update
+            fi
+        fi
+        apt-get install -y --no-install-recommends -t wheezy-backports libaugeas0
+        augeas_pkg=
+    else
+        echo "No libaugeas0 version is available that's new enough to run the"
+        echo "Let's Encrypt apache plugin..."
+    fi
+    # XXX add a case for ubuntu PPAs
+fi
+
 apt-get install -y --no-install-recommends \
-  git \
   python \
   python-dev \
   $virtualenv \
   gcc \
   dialog \
-  libaugeas0 \
+  $augeas_pkg \
   libssl-dev \
   libffi-dev \
   ca-certificates \
 
+
+
 if ! command -v virtualenv > /dev/null ; then
   echo Failed to install a working \"virtualenv\" command, exiting
   exit 1

bootstrap/_gentoo_common.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-PACKAGES="dev-vcs/git
+PACKAGES="
   dev-lang/python:2.7
   dev-python/virtualenv
   dev-util/dialog

bootstrap/_rpm_common.sh

@@ -33,9 +33,7 @@ then
   fi
 fi
 
-# "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails)
 if ! $tool install -y \
-       git-core \
        gcc \
        dialog \
        augeas-libs \

bootstrap/_suse_common.sh

@@ -2,7 +2,7 @@
 
 # SLE12 don't have python-virtualenv
 
-zypper -nq in -l git-core \
+zypper -nq in -l \
   python \
   python-devel \
   python-virtualenv \

bootstrap/dev/venv.sh

@@ -4,7 +4,6 @@
 export VENV_ARGS="--python python2"
 
 ./bootstrap/dev/_venv_common.sh \
-  -r py26reqs.txt \
   -e acme[testing] \
   -e .[dev,docs,testing] \
   -e letsencrypt-apache \

bootstrap/freebsd.sh

@@ -1,7 +1,6 @@
 #!/bin/sh -xe
 
 pkg install -Ay \
-  git \
   python \
   py27-virtualenv \
   augeas \

bootstrap/venv.sh

@@ -20,7 +20,7 @@ fi
 pip install -U setuptools
 pip install -U pip
 
-pip install -U -r py26reqs.txt letsencrypt letsencrypt-apache # letsencrypt-nginx
+pip install -U letsencrypt letsencrypt-apache # letsencrypt-nginx
 
 echo
 echo "Congratulations, Let's Encrypt has been successfully installed/updated!"

docs/using.rst

@@ -371,7 +371,7 @@ If you run Debian Stretch or Debian Sid, you can install letsencrypt packages.
    sudo apt-get update
    sudo apt-get install letsencrypt python-letsencrypt-apache
 
-If you don't want to use the Apache plugin, you can ommit the
+If you don't want to use the Apache plugin, you can omit the
 ``python-letsencrypt-apache`` package.
 
 Packages for Debian Jessie are coming in the next few weeks.

letsencrypt-apache/letsencrypt_apache/configurator.py

@@ -556,10 +556,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator):
         :param str port: Port to listen on
 
         """
-        if self.conf("handle-modules"):
         if "ssl_module" not in self.parser.modules:
             self.enable_mod("ssl", temp=temp)
-
+        if self.version >= (2, 4) and "socache_shmcb_module" not in self.parser.modules:
+            self.enable_mod("socache_shmcb", temp=temp)
         # Check for Listen <port>
         # Note: This could be made to also look for ip:443 combo
         listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")]
@@ -1423,7 +1423,7 @@ def _get_mod_deps(mod_name):
 
     """
     deps = {
-        "ssl": ["setenvif", "mime", "socache_shmcb"]
+        "ssl": ["setenvif", "mime"]
     }
     return deps.get(mod_name, [])
 

letsencrypt-apache/letsencrypt_apache/parser.py

@@ -20,7 +20,6 @@ class ApacheParser(object):
 
     :ivar str root: Normalized absolute path to the server root
         directory. Without trailing slash.
-    :ivar str root: Server root
     :ivar set modules: All module names that are currently enabled.
     :ivar dict loc: Location to place directives, root - configuration origin,
         default - user config file, name - NameVirtualHost,
@@ -37,7 +36,7 @@ class ApacheParser(object):
         # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine
         # This only handles invocation parameters and Define directives!
         self.variables = {}
-        if version >= (2, 4):
+        self.unparsable = False
         self.update_runtime_variables(ctl)
 
         self.aug = aug
@@ -63,6 +62,11 @@ class ApacheParser(object):
         # Must also attempt to parse virtual host root
         self._parse_file(self.vhostroot + "/*.conf")
 
+        #check to see if there were unparsed define statements
+        if self.unparsable:
+            if self.find_dir("Define", exclude=False):
+                raise errors.PluginError("Error parsing runtime variables")
+
     def init_modules(self):
         """Iterates on the configuration until no new modules are loaded.
 
@@ -104,7 +108,8 @@ class ApacheParser(object):
         try:
             matches.remove("DUMP_RUN_CFG")
         except ValueError:
-            raise errors.PluginError("Unable to parse runtime variables")
+            self.unparsable = True
+            return
 
         for match in matches:
             if match.count("=") > 1:

letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py

@@ -27,11 +27,22 @@ class TwoVhost80Test(util.ApacheTest):
         super(TwoVhost80Test, self).setUp()
 
         self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir, self.work_dir)
+            self.config_path, self.config_dir, self.work_dir)
-
+        self.config = self.mock_deploy_cert(self.config)
         self.vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/two_vhost_80")
 
+    def mock_deploy_cert(self, config):
+        """A test for a mock deploy cert"""
+        self.config.real_deploy_cert = self.config.deploy_cert
+        def mocked_deploy_cert(*args, **kwargs):
+            """a helper to mock a deployed cert"""
+            with mock.patch(
+                 "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"):
+                config.real_deploy_cert(*args, **kwargs)
+        self.config.deploy_cert = mocked_deploy_cert
+        return self.config
+
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
         shutil.rmtree(self.config_dir)
@@ -274,6 +285,7 @@ class TwoVhost80Test(util.ApacheTest):
 
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
+        self.config = self.mock_deploy_cert(self.config)
         self.config.deploy_cert(
             "random.demo", "example/cert.pem", "example/key.pem",
             "example/cert_chain.pem", "example/fullchain.pem")
@@ -299,7 +311,8 @@ class TwoVhost80Test(util.ApacheTest):
 
     def test_deploy_cert_newssl_no_fullchain(self):
         self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16))
+            self.config_path, self.config_dir, self.work_dir, version=(2, 4, 16))
+        self.config = self.mock_deploy_cert(self.config)
 
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")
@@ -312,7 +325,8 @@ class TwoVhost80Test(util.ApacheTest):
 
     def test_deploy_cert_old_apache_no_chain(self):
         self.config = util.get_apache_configurator(
-            self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7))
+            self.config_path, self.config_dir, self.work_dir, version=(2, 4, 7))
+        self.config = self.mock_deploy_cert(self.config)
 
         self.config.parser.modules.add("ssl_module")
         self.config.parser.modules.add("mod_ssl.c")

letsencrypt-apache/letsencrypt_apache/tests/parser_test.py

@@ -151,8 +151,8 @@ class BasicParserTest(util.ParserTest):
     @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
     def test_update_runtime_vars_bad_output(self, mock_cfg):
         mock_cfg.return_value = "Define: TLS=443=24"
-        self.assertRaises(
+        self.parser.update_runtime_variables("ctl")
-            errors.PluginError, self.parser.update_runtime_variables, "ctl")
+        self.assertTrue(self.parser.unparsable)
 
         mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24"
         self.assertRaises(
@@ -185,6 +185,21 @@ class ParserInitTest(util.ApacheTest):
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_unparsable(self, mock_cfg):
+        from letsencrypt_apache.parser import ApacheParser
+        def unparsable_true(self, arg):
+            """a helper to set the self unparsabale to true"""
+            print "side effect has passed in arg: %s", arg
+            self.unparsable = True
+        with mock.patch.object(ApacheParser, 'update_runtime_variables', autospec=True) as urv:
+            urv.side_effect = unparsable_true
+            mock_cfg.return_value = ('Define: TEST')
+            self.assertRaises(
+                errors.PluginError,
+                ApacheParser, self.aug, os.path.relpath(self.config_path), "ctl")
+        self.assertEquals(1, 1)
+
     def test_root_normalized(self):
         from letsencrypt_apache.parser import ApacheParser
 

letsencrypt-apache/letsencrypt_apache/tests/tls_sni_01_test.py

@@ -78,6 +78,8 @@ class TlsSniPerformTest(util.ApacheTest):
         # pylint: disable=protected-access
         self.sni._setup_challenge_cert = mock_setup_cert
 
+        with mock.patch(
+             "letsencrypt_apache.configurator.ApacheConfigurator.enable_mod"):
             sni_responses = self.sni.perform()
 
         self.assertEqual(mock_setup_cert.call_count, 2)

letsencrypt-auto

@@ -47,13 +47,13 @@ if test "`id -u`" -ne "0" ; then
       args=""
       # This `while` loop iterates over all parameters given to this function.
       # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string
-      # will be wrap in a pair of `'`, then append to `$args` string
+      # will be wrapped in a pair of `'`, then appended to `$args` string
       # For example, `echo "It's only 1\$\!"` will be escaped to:
       #   'echo' 'It'"'"'s only 1$!'
       #     │       │└┼┘│
       #     │       │ │ └── `'s only 1$!'` the literal string
       #     │       │ └── `\"'\"` is a single quote (as a string)
-      #     │       └── `'It'`, to be concatenated with the strings followed it
+      #     │       └── `'It'`, to be concatenated with the strings following it
       #     └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself
       while [ $# -ne 0 ]; do
         args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' "
@@ -176,7 +176,7 @@ if [ "$VERBOSE" = 1 ]  ; then
   echo
   $VENV_BIN/pip install -U setuptools
   $VENV_BIN/pip install -U pip
-  $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt -U letsencrypt letsencrypt-apache
+  $VENV_BIN/pip install -U letsencrypt letsencrypt-apache
   # nginx is buggy / disabled for now, but upgrade it if the user has
   # installed it manually
   if $VENV_BIN/pip freeze | grep -q letsencrypt-nginx ; then
@@ -188,8 +188,6 @@ else
   $VENV_BIN/pip install -U pip > /dev/null
   printf .
   # nginx is buggy / disabled for now...
-  $VENV_BIN/pip install -r "$LEA_PATH"/py26reqs.txt > /dev/null
-  printf .
   $VENV_BIN/pip install -U letsencrypt > /dev/null
   printf .
   $VENV_BIN/pip install -U letsencrypt-apache > /dev/null
@@ -202,5 +200,5 @@ fi
 
 # Explain what's about to happen, for the benefit of those getting sudo
 # password prompts...
-echo "Running with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@"
+echo "Requesting root privileges to run with virtualenv:" $SUDO $VENV_BIN/letsencrypt "$@"
 $SUDO $VENV_BIN/letsencrypt "$@"

letsencrypt/cli.py

@@ -1189,7 +1189,7 @@ def _plugins_parsing(helpful, plugins):
 
     # These would normally be a flag within the webroot plugin, but because
     # they are parsed in conjunction with --domains, they live here for
-    # legibiility. helpful.add_plugin_ags must be called first to add the
+    # legibility. helpful.add_plugin_ags must be called first to add the
     # "webroot" topic
     helpful.add("webroot", "-w", "--webroot-path", action=WebrootPathProcessor,
                 help="public_html / webroot path. This can be specified multiple times to "

letsencrypt/plugins/webroot_test.py

@@ -66,7 +66,15 @@ class AuthenticatorTest(unittest.TestCase):
 
     def test_prepare_reraises_other_errors(self):
         self.auth.full_path = os.path.join(self.path, "null")
+        permission_canary = os.path.join(self.path, "rnd")
+        with open(permission_canary, "w") as f:
+            f.write("thingimy")
         os.chmod(self.path, 0o000)
+        try:
+            open(permission_canary, "r")
+            print "Warning, running tests as root skips permissions tests..."
+        except IOError:
+            # ok, permissions work, test away...
             self.assertRaises(errors.PluginError, self.auth.prepare)
         os.chmod(self.path, 0o700)
 

letsencrypt/renewer.py

@@ -116,6 +116,7 @@ def renew(cert, old_version):
 def _cli_log_handler(args, level, fmt):  # pylint: disable=unused-argument
     handler = colored_logging.StreamHandler()
     handler.setFormatter(logging.Formatter(fmt))
+    handler.setLevel(level)
     return handler
 
 

py26reqs.txt

@@ -1,2 +0,0 @@
-# https://github.com/bw2/ConfigArgParse/issues/17
-git+https://github.com/kuba/ConfigArgParse.git@python2.6-0.9.3#egg=ConfigArgParse

setup.py

@@ -32,7 +32,6 @@ version = meta['version']
 
 install_requires = [
     'acme=={0}'.format(version),
-    'ConfigArgParse',
     'configobj',
     'cryptography>=0.7',  # load_pem_x509_certificate
     'parsedatetime',
@@ -53,10 +52,19 @@ if sys.version_info < (2, 7):
     install_requires.extend([
         # only some distros recognize stdlib argparse as already satisfying
         'argparse',
+        'ConfigArgParse>=0.10.0',  # python2.6 support, upstream #17
         'mock<1.1.0',
     ])
 else:
-    install_requires.append('mock')
+    install_requires.extend([
+        'ConfigArgParse',
+        'mock',
+    ])
+
+if sys.version_info < (2, 7, 9):
+    # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning)
+    install_requires.append('ndg-httpsclient')
+    install_requires.append('pyasn1')
 
 dev_extras = [
     # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289

tests/apache-conf-files/passing/rewrite-quote-1960.conf

@@ -0,0 +1,7 @@
+<IfModule mod_rewrite.c>
+ RewriteEngine On
+ RewriteCond %{REQUEST_URI} ^.*(,|;|:|<|>|">|"<|/|\\\.\.\\).* [NC,OR]
+ RewriteCond %{REQUEST_URI} ^.*(\=|\@|\[|\]|\^|\`|\{|\}|\~).* [NC,OR]
+ RewriteCond %{REQUEST_URI} ^.*(\'|%0A|%0D|%27|%3C|%3E|%00).* [NC]
+ RewriteRule ^(.*)$ - [F,L]
+</IfModule>

tests/letstest/README.md

@@ -0,0 +1,44 @@
+# letstest
+simple aws testfarm scripts for letsencrypt client testing
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+## Notes
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+## Usage
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+```
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem
+```
+then:
+```
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_apache2.sh
+```
+
+## Scripts
+example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed
+to them at runtime via environment variables.  test_apache2.sh is a useful reference.
+
+Note that the <pre>test_letsencrypt_auto_*</pre> scripts pull code from PyPI using the letsencrypt-auto script,
+__not__ the local python code.  test_apache2 runs the dev venv and does local tests.
+
+see:
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+
+main repos:
+- https://github.com/letsencrypt/boulder
+- https://github.com/letsencrypt/letsencrypt

tests/letstest/apache2_targets.yaml

@@ -0,0 +1,57 @@
+targets:
+  #-----------------------------------------------------------------------------
+  # Apache 2.4
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Apache 2.2
+  # - ami: ami-0611546c
+  #   name: ubuntu12.04LTS
+  #   type: ubuntu
+  #   virt: hvm
+  #   user: ubuntu
+  # - ami: ami-e0efab88
+  #   name: debian7.8.aws.1
+  #   type: debian
+  #   virt: hvm
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]
+  # - ami: ami-e6eeaa8e
+  #   name: debian7.8.aws.1_32bit
+  #   type: debian
+  #   virt: pv
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]

tests/letstest/multitester.py

@@ -0,0 +1,492 @@
+"""
+Letsencrypt Integration Test Tool
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+Notes:
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+Usage:
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
+ --query 'KeyMaterial' --output text > MyKeyPair.pem
+then:
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+"""
+
+from __future__ import print_function
+from __future__ import with_statement
+
+import sys, os, time, argparse, socket
+import multiprocessing as mp
+from multiprocessing import Manager
+import urllib2
+import yaml
+import boto3
+import fabric
+from fabric.api import run, execute, local, env, sudo, cd, lcd
+from fabric.operations import get, put
+from fabric.context_managers import shell_env
+
+# Command line parser
+#-------------------------------------------------------------------------------
+parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.')
+parser.add_argument('config_file',
+                    help='yaml configuration file for AWS server cluster')
+parser.add_argument('key_file',
+                    help='key file (<keyname>.pem) for AWS')
+parser.add_argument('aws_profile',
+                    help='profile for AWS (i.e. as in ~/.aws/certificates)')
+parser.add_argument('test_script',
+                    default='test_letsencrypt_auto_certonly_standalone.sh',
+                    help='path of bash script in to deploy and run')
+#parser.add_argument('--script_args',
+#                    nargs='+',
+#                    help='space-delimited list of arguments to pass to the bash test script',
+#                    required=False)
+parser.add_argument('--repo',
+                    default='https://github.com/letsencrypt/letsencrypt.git',
+                    help='letsencrypt git repo to use')
+parser.add_argument('--branch',
+                    default='~',
+                    help='letsencrypt git branch to trial')
+parser.add_argument('--pull_request',
+                    default='~',
+                    help='letsencrypt/letsencrypt pull request to trial')
+parser.add_argument('--merge_master',
+                    action='store_true',
+                    help="if set merges PR into master branch of letsencrypt/letsencrypt")
+parser.add_argument('--saveinstances',
+                    action='store_true',
+                    help="don't kill EC2 instances after run, useful for debugging")
+parser.add_argument('--alt_pip',
+                    default='https://certainly.isnot.org/pip/',
+                    help="server from which to pull candidate release packages")
+cl_args = parser.parse_args()
+
+# Credential Variables
+#-------------------------------------------------------------------------------
+# assumes naming: <key_filename> = <keyname>.pem
+KEYFILE = cl_args.key_file
+KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0]
+PROFILE = cl_args.aws_profile
+
+# Globals
+#-------------------------------------------------------------------------------
+BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1
+LOGDIR = "" #points to logging / working directory
+# boto3/AWS api globals
+AWS_SESSION = None
+EC2 = None
+
+# Boto3/AWS automation functions
+#-------------------------------------------------------------------------------
+def make_security_group():
+    # will fail if security group of GroupName already exists
+    # cannot have duplicate SGs of the same name
+    mysg = EC2.create_security_group(GroupName="letsencrypt_test",
+                                     Description='security group for automated testing')
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443)
+    # for boulder wfe (http) server
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000)
+    # for mosh
+    mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000)
+    return mysg
+
+def make_instance(instance_name,
+                  ami_id,
+                  keyname,
+                  machine_type='t2.micro',
+                  security_groups=['letsencrypt_test'],
+                  userdata=""): #userdata contains bash or cloud-init script
+
+    new_instance = EC2.create_instances(
+        ImageId=ami_id,
+        SecurityGroups=security_groups,
+        KeyName=keyname,
+        MinCount=1,
+        MaxCount=1,
+        UserData=userdata,
+        InstanceType=machine_type)[0]
+
+    # brief pause to prevent rare error on EC2 delay, should block until ready instead
+    time.sleep(1.0)
+
+    # give instance a name
+    new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
+    return new_instance
+
+def terminate_and_clean(instances):
+    """
+    Some AMIs specify EBS stores that won't delete on instance termination.
+    These must be manually deleted after shutdown.
+    """
+    volumes_to_delete = []
+    for instance in instances:
+        for bdmap in instance.block_device_mappings:
+            if 'Ebs' in bdmap.keys():
+                if not bdmap['Ebs']['DeleteOnTermination']:
+                    volumes_to_delete.append(bdmap['Ebs']['VolumeId'])
+
+    for instance in instances:
+        instance.terminate()
+
+    # can't delete volumes until all attaching instances are terminated
+    _ids = [instance.id for instance in instances]
+    all_terminated = False
+    while not all_terminated:
+        all_terminated = True
+        for _id in _ids:
+            # necessary to reinit object for boto3 to get true state
+            inst = EC2.Instance(id=_id)
+            if inst.state['Name'] != 'terminated':
+                all_terminated = False
+        time.sleep(5)
+
+    for vol_id in volumes_to_delete:
+        volume = EC2.Volume(id=vol_id)
+        volume.delete()
+
+    return volumes_to_delete
+
+
+# Helper Routines
+#-------------------------------------------------------------------------------
+def block_until_http_ready(urlstring, wait_time=10, timeout=240):
+    "Blocks until server at urlstring can respond to http requests"
+    server_ready = False
+    t_elapsed = 0
+    while not server_ready and t_elapsed < timeout:
+        try:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            req = urllib2.Request(urlstring)
+            response = urllib2.urlopen(req)
+            #if response.code == 200:
+            server_ready = True
+        except urllib2.URLError:
+            pass
+        time.sleep(wait_time)
+        t_elapsed += wait_time
+
+def block_until_ssh_open(ipstring, wait_time=10, timeout=120):
+    "Blocks until server at ipstring has an open port 22"
+    reached = False
+    t_elapsed = 0
+    while not reached and t_elapsed < timeout:
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect((ipstring, 22))
+            reached = True
+        except socket.error as err:
+            time.sleep(wait_time)
+            t_elapsed += wait_time
+    sock.close()
+
+def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20):
+    "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections"
+    # the reinstantiation from id is necessary to force boto3
+    # to correctly update the 'state' variable during init
+    _id = booting_instance.id
+    _instance = EC2.Instance(id=_id)
+    _state = _instance.state['Name']
+    _ip = _instance.public_ip_address
+    while _state != 'running' or _ip is None:
+        time.sleep(wait_time)
+        _instance = EC2.Instance(id=_id)
+        _state = _instance.state['Name']
+        _ip = _instance.public_ip_address
+    block_until_ssh_open(_ip)
+    time.sleep(extra_wait_time)
+    return _instance
+
+
+# Fabric Routines
+#-------------------------------------------------------------------------------
+def local_git_clone(repo_url):
+    "clones master of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_branch(repo_url, branch_name):
+    "clones branch <branch_name> of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s --branch %s --single-branch'%(repo_url, branch_name))
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_PR(repo_url, PRnumstr, merge_master=True):
+    "clones specified pull request from repo_url and optionally merges into master"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr)
+        local('cd letsencrypt && git co lePRtest')
+        if merge_master:
+            local('cd letsencrypt && git remote update origin')
+            local('cd letsencrypt && git merge origin/master -m "testmerge"')
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_repo_to_remote():
+    "copies local tarball of repo to remote"
+    with lcd(LOGDIR):
+        put(local_path='le.tar.gz', remote_path='')
+        run('tar xzf le.tar.gz')
+
+def local_repo_clean():
+    "delete tarball"
+    with lcd(LOGDIR):
+        local('rm le.tar.gz')
+
+def deploy_script(scriptpath, *args):
+    "copies to remote and executes local script"
+    #with lcd('scripts'):
+    put(local_path=scriptpath, remote_path='', mirror_local_mode=True)
+    scriptfile = os.path.split(scriptpath)[1]
+    args_str = ' '.join(args)
+    run('./'+scriptfile+' '+args_str)
+
+def run_boulder():
+    with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
+        run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost')
+        run('nohup ./start.py >& /dev/null < /dev/null &')
+
+def config_and_launch_boulder(instance):
+    execute(deploy_script, 'scripts/boulder_config.sh')
+    execute(run_boulder)
+
+def install_and_launch_letsencrypt(instance, boulder_url, target):
+    execute(local_repo_to_remote)
+    with shell_env(BOULDER_URL=boulder_url,
+                   PUBLIC_IP=instance.public_ip_address,
+                   PRIVATE_IP=instance.private_ip_address,
+                   PUBLIC_HOSTNAME=instance.public_dns_name,
+                   PIP_EXTRA_INDEX_URL=cl_args.alt_pip,
+                   OS_TYPE=target['type']):
+        execute(deploy_script, cl_args.test_script)
+
+def grab_letsencrypt_log():
+    "grabs letsencrypt.log via cat into logged stdout"
+    sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \
+    cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi')
+    # fallback file if /var/log is unwriteable...? correct?
+    sudo('if [ -f ./letsencrypt.log ]; then \
+    cat ./letsencrypt.log; else echo "[nolocallog]"; fi')
+
+#-------------------------------------------------------------------------------
+# SCRIPT BEGINS
+#-------------------------------------------------------------------------------
+
+# Fabric library controlled through global env parameters
+env.key_filename = KEYFILE
+env.shell = '/bin/bash -l -i -c'
+env.connection_attempts = 5
+env.timeout = 10
+# replace default SystemExit thrown by fabric during trouble
+class FabricException(Exception):
+    pass
+env['abort_exception'] = FabricException
+
+# Set up local copy of git repo
+#-------------------------------------------------------------------------------
+LOGDIR = "letest-%d"%int(time.time())
+print("Making local dir for test repo and logs: %s"%LOGDIR)
+local('mkdir %s'%LOGDIR)
+
+# figure out what git object to test and locally create it in LOGDIR
+print("Making local git repo")
+try:
+    if cl_args.pull_request != '~':
+        print('Testing PR %s '%cl_args.pull_request,
+              "MERGING into master" if cl_args.merge_master else "")
+        execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
+    elif cl_args.branch != '~':
+        print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
+        execute(local_git_branch, cl_args.repo, cl_args.branch)
+    else:
+        print('Testing master of %s'%cl_args.repo)
+        execute(local_git_clone, cl_args.repo)
+except FabricException:
+    print("FAIL: trouble with git repo")
+    exit()
+
+
+# Set up EC2 instances
+#-------------------------------------------------------------------------------
+configdata = yaml.load(open(cl_args.config_file, 'r'))
+targetlist = configdata['targets']
+print('Testing against these images: [%d total]'%len(targetlist))
+for target in targetlist:
+    print(target['ami'], target['name'])
+
+print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
+AWS_SESSION = boto3.session.Session(profile_name=PROFILE)
+EC2 = AWS_SESSION.resource('ec2')
+
+print("Making Security Group")
+sg_exists = False
+for sg in EC2.security_groups.all():
+    if sg.group_name == 'letsencrypt_test':
+        sg_exists = True
+        print("  %s already exists"%'letsencrypt_test')
+if not sg_exists:
+    make_security_group()
+    time.sleep(30)
+
+print("Requesting Instances...")
+boulder_server = make_instance('le-boulderserver',
+                               BOULDER_AMI,
+                               KEYNAME,
+                               #machine_type='t2.micro',
+                               machine_type='t2.medium',
+                               security_groups=['letsencrypt_test'])
+
+instances = []
+for target in targetlist:
+    if target['virt'] == 'hvm':
+        machine_type = 't2.micro'
+    else:
+        machine_type = 't1.micro'
+    if 'userdata' in target.keys():
+        userdata = target['userdata']
+    else:
+        userdata = ''
+    instances.append(make_instance('le-%s'%target['name'],
+                                   target['ami'],
+                                   KEYNAME,
+                                   machine_type=machine_type,
+                                   userdata=userdata))
+
+
+# Configure and launch boulder server
+#-------------------------------------------------------------------------------
+print("Waiting on Boulder Server")
+boulder_server = block_until_instance_ready(boulder_server)
+print(" server %s"%boulder_server)
+
+print("Configuring and Launching Boulder")
+
+# env.host_string defines the ssh user and host for connection
+env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
+print("Boulder Server at (SSH):", env.host_string)
+config_and_launch_boulder(boulder_server)
+# blocking often unnecessary, but cheap EC2 VMs can get very slow
+block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
+                       wait_time=10,
+                       timeout=500)
+
+boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
+print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
+print("Boulder Server at (EC2 private ip): %s"%boulder_url)
+
+# Install and launch client scripts in parallel
+#-------------------------------------------------------------------------------
+print("Uploading and running test script in parallel: %s"%cl_args.test_script)
+print("Output routed to log files in %s"%LOGDIR)
+# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
+# the latter has implementation flaws that deadlock it in some circumstances)
+manager = Manager()
+outqueue = manager.Queue()
+inqueue = manager.Queue()
+SENTINEL = None #queue kill signal
+
+# launch as many processes as clients to test
+num_processes = len(targetlist)
+jobs = [] #keep a reference to current procs
+
+def test_client_process(inqueue, outqueue):
+    cur_proc = mp.current_process()
+    for inreq in iter(inqueue.get, SENTINEL):
+        ii, target = inreq
+
+        #save all stdout to log file
+        sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
+
+        print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
+        instances[ii] = block_until_instance_ready(instances[ii])
+        print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
+        env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
+        print(env.host_string)
+
+        try:
+            install_and_launch_letsencrypt(instances[ii], boulder_url, target)
+            outqueue.put((ii, target, 'pass'))
+            print("%s - %s SUCCESS"%(target['ami'], target['name']))
+        except:
+            outqueue.put((ii, target, 'fail'))
+            print("%s - %s FAIL"%(target['ami'], target['name']))
+            pass
+
+        # append server letsencrypt.log to each per-machine output log
+        print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
+        try:
+            execute(grab_letsencrypt_log)
+        except:
+            print("log fail\n")
+            pass
+
+# initiate process execution
+for i in range(num_processes):
+    p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
+    jobs.append(p)
+    p.daemon = True  # kills subprocesses if parent is killed
+    p.start()
+
+# fill up work queue
+for ii, target in enumerate(targetlist):
+    inqueue.put((ii, target))
+
+# add SENTINELs to end client processes
+for i in range(num_processes):
+    inqueue.put(SENTINEL)
+# wait on termination of client processes
+for p in jobs:
+    p.join()
+# add SENTINEL to output queue
+outqueue.put(SENTINEL)
+
+# clean up
+execute(local_repo_clean)
+
+# print and save summary results
+results_file = open(LOGDIR+'/results', 'w')
+outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
+outputs.sort(key=lambda x: x[0])
+for outq in outputs:
+    ii, target, status = outq
+    print('%d %s %s'%(ii, target['name'], status))
+    results_file.write('%d %s %s\n'%(ii, target['name'], status))
+results_file.close()
+
+if not cl_args.saveinstances:
+    print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
+    boulder_server.terminate()
+    terminate_and_clean(instances)
+else:
+    # print login information for the boxes for debugging
+    for ii, target in enumerate(targetlist):
+        print(target['name'],
+              target['ami'],
+              "%s@%s"%(target['user'], instances[ii].public_ip_address))
+
+# kill any connections
+fabric.network.disconnect_all()

tests/letstest/scripts/boulder_config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Configures and Launches Boulder Server installed on
+# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab)
+
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# get local DNS resolver for VPC
+resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1)
+resolver=$resolver_ip':53'
+
+# modifies integration testing boulder setup for local AWS VPC network
+# connections instead of localhost
+cd $GOPATH/src/github.com/letsencrypt/boulder
+# configure boulder to receive outside connection on 4000
+sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+# change test ports to real
+sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json
+sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json
+sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json
+# set local dns resolver
+sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json
+
+# start rabbitMQ
+#go run cmd/rabbitmq-setup/main.go -server amqp://localhost
+# start acme services
+#nohup ./start.py >& /dev/null < /dev/null &
+#./start.py

tests/letstest/scripts/boulder_install.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# >>>> only tested on Ubuntu 14.04LTS <<<<
+
+# non-interactive install of mariadb and other dependencies
+export DEBIAN_FRONTEND=noninteractive
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS'
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS'
+apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server
+sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');"
+
+# install go
+wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+tar xzvf go1.5.1.linux-amd64.tar.gz
+mkdir gocode
+echo "export GOROOT=/home/ubuntu/go \n\
+      export GOPATH=/home/ubuntu/gocode\n\
+      export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc
+
+# install boulder and its go dependencies
+go get -d github.com/letsencrypt/boulder/...
+cd $GOPATH/src/github.com/letsencrypt/boulder
+wget https://github.com/jsha/boulder-tools/raw/master/goose.gz
+mkdir $GOPATH/bin
+zcat goose.gz > $GOPATH/bin/goose
+chmod +x $GOPATH/bin/goose
+./test/create_db.sh
+go get github.com/jsha/listenbuddy

tests/letstest/scripts/test_apache2.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -x
+
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
+
+if [ "$OS_TYPE" = "ubuntu" ]
+then
+    CONFFILE=/etc/apache2/sites-available/000-default.conf
+    sudo apt-get update
+    sudo apt-get -y --no-upgrade install apache2 #curl
+    # For apache 2.4, set up ServerName
+    sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE
+    sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE
+elif [ "$OS_TYPE" = "centos" ]
+then
+    CONFFILE=/etc/httpd/conf/httpd.conf
+    sudo setenforce 0 || true #disable selinux
+    sudo yum -y install httpd
+    sudo service httpd start
+    sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html
+    sudo chmod -R oug+rwx /var/www
+    sudo chmod -R oug+rw /etc/httpd
+    sudo echo '<html><head><title>foo</title></head><body>bar</body></html>' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html
+    sudo mkdir /etc/httpd/sites-available #letsencrypt requires this...
+    sudo mkdir /etc/httpd/sites-enabled #letsencrypt requires this...
+    #sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf
+    sudo echo """
+<VirtualHost *:80>
+    ServerName $PUBLIC_HOSTNAME
+    DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html
+    ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log
+    CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined
+</VirtualHost>""" >> /etc/httpd/conf.d/$PUBLIC_HOSTNAME.conf
+    #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/
+fi
+
+# run letsencrypt-apache2 via letsencrypt-auto
+cd letsencrypt
+./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect --register-unsafely-without-email \
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+if [ "$OS_TYPE" = "ubuntu" ] ; then
+    export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt"
+    tests/apache-conf-files/hackish-apache-test --debian-modules
+else
+    echo Not running hackish apache tests on $OS_TYPE
+fi
+
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+# return error if any of the subtests failed
+if [ "$FAIL" = 1 ] ; then
+    return 1
+fi

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -xe
+
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
+
+cd letsencrypt
+#git checkout v0.1.0     use --branch instead
+SAVE="$PIP_EXTRA_INDEX_URL"
+unset PIP_EXTRA_INDEX_URL
+./letsencrypt-auto -v --debug --version 
+
+export PIP_EXTRA_INDEX_URL="$SAVE"
+
+if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then
+    echo upgrade appeared to fail
+    exit 1
+fi
+echo upgrade appeared to be successful

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -x
+
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+
+# with curl, instance metadata available from EC2 metadata service:
+#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+cd letsencrypt
+./letsencrypt-auto certonly -v --standalone --debug \
+                   --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect \
+                   --register-unsafely-without-email \
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL

tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -x
+
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+
+cd letsencrypt
+# help installs virtualenv and does nothing else
+./letsencrypt-auto -v --help all

tests/letstest/scripts/test_tox.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -x
+XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share}
+VENV_NAME="venv"
+# The path to the letsencrypt-auto script.  Everything that uses these might
+# at some point be inlined...
+LEA_PATH=./letsencrypt/
+VENV_PATH=${LEA_PATH/$VENV_NAME}
+VENV_BIN=${VENV_PATH}/bin
+BOOTSTRAP=${LEA_PATH}/bootstrap
+
+SUDO=sudo
+
+ExperimentalBootstrap() {
+  # Arguments: Platform name, boostrap script name, SUDO command (iff needed)
+    if [ "$2" != "" ]  ; then
+      echo "Bootstrapping dependencies for $1..."
+      if [ "$3" != "" ] ; then
+        "$3" "$BOOTSTRAP/$2"
+      else
+        "$BOOTSTRAP/$2"
+      fi
+    fi
+}
+
+# virtualenv call is not idempotent: it overwrites pip upgraded in
+# later steps, causing "ImportError: cannot import name unpack_url"
+if [ ! -f $BOOTSTRAP/debian.sh ] ; then
+  echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP"
+  exit 1
+fi
+
+if [ -f /etc/debian_version ] ; then
+  echo "Bootstrapping dependencies for Debian-based OSes..."
+  $SUDO $BOOTSTRAP/_deb_common.sh
+elif [ -f /etc/redhat-release ] ; then
+  echo "Bootstrapping dependencies for RedHat-based OSes..."
+  $SUDO $BOOTSTRAP/_rpm_common.sh
+elif `grep -q openSUSE /etc/os-release` ; then
+  echo "Bootstrapping dependencies for openSUSE-based OSes..."
+  $SUDO $BOOTSTRAP/_suse_common.sh
+elif [ -f /etc/arch-release ] ; then
+  if [ "$DEBUG" = 1 ] ; then
+    echo "Bootstrapping dependencies for Archlinux..."
+    $SUDO $BOOTSTRAP/archlinux.sh
+  else
+    echo "Please use pacman to install letsencrypt packages:"
+    echo "# pacman -S letsencrypt letsencrypt-apache"
+    echo
+    echo "If you would like to use the virtualenv way, please run the script again with the"
+    echo "--debug flag."
+    exit 1
+  fi
+elif [ -f /etc/manjaro-release ] ; then
+  ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO"
+elif [ -f /etc/gentoo-release ] ; then
+  ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO"
+elif uname | grep -iq FreeBSD ; then
+  ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO"
+elif uname | grep -iq Darwin ; then
+  ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root
+elif grep -iq "Amazon Linux" /etc/issue ; then
+  ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO"
+else
+  echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!"
+  echo
+  echo "You will need to bootstrap, configure virtualenv, and run a pip install manually"
+  echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites"
+  echo "for more info"
+fi
+echo "Bootstrapped!"
+
+cd letsencrypt
+./bootstrap/dev/venv.sh
+PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+
+if [ $PYVER -eq 26 ] ; then
+    venv/bin/tox -e py26
+else
+    venv/bin/tox -e py27
+fi

tests/letstest/targets.yaml

@@ -0,0 +1,99 @@
+targets:
+  #-----------------------------------------------------------------------------
+  #Ubuntu
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-0611546c
+    name: ubuntu12.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  #-----------------------------------------------------------------------------
+  # Debian
+  - ami: ami-116d857a
+    name: debian8.1
+    type: ubuntu
+    virt: hvm
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  - ami: ami-e0efab88
+    name: debian7.8.aws.1
+    type: ubuntu
+    virt: hvm
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  - ami: ami-e6eeaa8e
+    name: debian7.8.aws.1_32bit
+    type: ubuntu
+    virt: pv
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Other Redhat Distros
+  - ami: ami-60b6c60a
+    name: amazonlinux-2015.09.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-0d4cfd66
+    name: amazonlinux-2015.03.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-a8d369c0
+    name: RHEL7
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-518bfb3b
+    name: fedora23
+    type: centos
+    virt: hvm
+    user: fedora
+  #-----------------------------------------------------------------------------
+  # CentOS
+  # These Marketplace AMIs must, irritatingly, have their terms manually
+  # agreed to on the AWS marketplace site for any new AWS account using them...
+  - ami: ami-61bbf104
+    name: centos7
+    type: centos
+    virt: hvm
+    user: centos
+  # centos6 requires EPEL repo added
+  - ami: ami-57cd8732
+    name: centos6
+    type: centos
+    virt: hvm
+    user: centos
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum install -y epel-release
+        - iptables -F

tools/release.sh

@@ -86,7 +86,7 @@ SetVersion() {
     done
     sed -i "s/^__version.*/__version__ = '$ver'/" letsencrypt/__init__.py
 
-    git add -p $SUBPKGS # interactive user input
+    git add -p letsencrypt $SUBPKGS # interactive user input
 }
 SetVersion "$version"
 git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"

tox.ini

@@ -17,7 +17,7 @@ envlist = py26,py27,py33,py34,py35,cover,lint
 commands =
     pip install -e acme[testing]
     nosetests -v acme
-    pip install -r py26reqs.txt -e .[testing]
+    pip install -e .[testing]
     nosetests -v letsencrypt
     pip install -e letsencrypt-apache
     nosetests -v letsencrypt_apache