initial commit of scripts

.pylintrc

@@ -0,0 +1,335 @@
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+#load-plugins=linter_plugin
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time. See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes
+# abstract-class-not-used cannot be disabled locally (at least in
+# pylint 1.4.1), same for abstract-class-little-used
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input,file
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=f,i,j,k,ex,Run,_,fd,logger
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Regular expression matching correct function names
+function-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for function names
+function-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct variable names
+variable-rgx=[a-z_][a-z0-9_]{1,30}$
+
+# Naming hint for variable names
+variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for attribute names
+attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for argument names
+argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct method names
+method-rgx=[a-z_][a-z0-9_]{2,50}$
+
+# Naming hint for method names
+method-name-hint=[a-z_][a-z0-9_]{2,50}$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=(__.*__)|(test_[A-Za-z0-9_]*)|(_.*)|(.*Test$)
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging,logger
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=(unused)?_.*|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=6
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=yes
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+# List of optional constructs for which whitespace checking is disabled
+no-space-check=trailing-comma
+
+# Maximum number of lines in a module
+max-module-lines=1250
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging or continued line.
+# This does something silly/broken...
+#indent-after-paren=4
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis
+ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
+# import errors ignored only in 1.4.4
+# https://bitbucket.org/logilab/pylint/commits/cd000904c9e2
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=yes
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defined in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by,implementedBy,providedBy
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=6
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=12
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

README.md

@@ -1,2 +1,36 @@
 # letstest
 simple aws testfarm scripts for letsencrypt client testing
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+## Notes
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+## Usage
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+```
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem
+```
+then:
+```
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
+```
+
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+
+https://github.com/letsencrypt/boulder
+https://github.com/letsencrypt/letsencrypt

apache2_targets.yaml

@@ -0,0 +1,57 @@
+targets:
+  #-----------------------------------------------------------------------------
+  # Apache 2.4
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Apache 2.2
+  # - ami: ami-0611546c
+  #   name: ubuntu12.04LTS
+  #   type: ubuntu
+  #   virt: hvm
+  #   user: ubuntu
+  # - ami: ami-e0efab88
+  #   name: debian7.8.aws.1
+  #   type: debian
+  #   virt: hvm
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]
+  # - ami: ami-e6eeaa8e
+  #   name: debian7.8.aws.1_32bit
+  #   type: debian
+  #   virt: pv
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]

multitester.py

@@ -0,0 +1,482 @@
+"""
+Letsencrypt Integration Test Tool
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+Notes:
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+Usage:
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
+ --query 'KeyMaterial' --output text > MyKeyPair.pem
+then:
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker test_letsencrypt_auto_venv_only.sh
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+"""
+
+from __future__ import print_function
+from __future__ import with_statement
+
+import sys, os, time, argparse, socket
+import multiprocessing as mp
+from multiprocessing import Manager
+import urllib2
+import yaml
+import boto3
+import fabric
+from fabric.api import run, execute, local, env, sudo, cd, lcd
+from fabric.operations import get, put
+from fabric.context_managers import shell_env
+
+# Command line parser
+#-------------------------------------------------------------------------------
+parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.')
+parser.add_argument('config_file',
+                    help='yaml configuration file for AWS server cluster')
+parser.add_argument('key_file',
+                    help='key file (<keyname>.pem) for AWS')
+parser.add_argument('aws_profile',
+                    help='profile for AWS (i.e. as in ~/.aws/certificates)')
+parser.add_argument('test_script',
+                    default='test_letsencrypt_auto_certonly_standalone.sh',
+                    help='name of bash script in /scripts to deploy and run')
+#parser.add_argument('--script_args',
+#                    nargs='+',
+#                    help='space-delimited list of arguments to pass to the bash test script',
+#                    required=False)
+parser.add_argument('--repo',
+                    default='https://github.com/letsencrypt/letsencrypt.git',
+                    help='letsencrypt git repo to use')
+parser.add_argument('--branch',
+                    default='~',
+                    help='letsencrypt git branch to trial')
+parser.add_argument('--pull_request',
+                    default='~',
+                    help='letsencrypt/letsencrypt pull request to trial')
+parser.add_argument('--merge_master',
+                    action='store_true',
+                    help="if set merges PR into master branch of letsencrypt/letsencrypt")
+parser.add_argument('--saveinstances',
+                    action='store_true',
+                    help="don't kill EC2 instances after run, useful for debugging")
+cl_args = parser.parse_args()
+
+# Credential Variables
+#-------------------------------------------------------------------------------
+# assumes naming: <key_filename> = <keyname>.pem
+KEYFILE = cl_args.key_file
+KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0]
+PROFILE = cl_args.aws_profile
+
+# Globals
+#-------------------------------------------------------------------------------
+BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1
+LOGDIR = "" #points to logging / working directory
+# boto3/AWS api globals
+AWS_SESSION = None
+EC2 = None
+
+# Boto3/AWS automation functions
+#-------------------------------------------------------------------------------
+def make_security_group():
+    # will fail if security group of GroupName already exists
+    # cannot have duplicate SGs of the same name
+    mysg = EC2.create_security_group(GroupName="letsencrypt_test",
+                                     Description='security group for automated testing')
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443)
+    # for boulder wfe (http) server
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000)
+    # for mosh
+    mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000)
+    return mysg
+
+def make_instance(instance_name,
+                  ami_id,
+                  keyname,
+                  machine_type='t2.micro',
+                  security_groups=['letsencrypt_test'],
+                  userdata=""): #userdata contains bash or cloud-init script
+
+    new_instance = EC2.create_instances(
+        ImageId=ami_id,
+        SecurityGroups=security_groups,
+        KeyName=keyname,
+        MinCount=1,
+        MaxCount=1,
+        UserData=userdata,
+        InstanceType=machine_type)[0]
+
+    # brief pause to prevent rare EC2 error
+    time.sleep(0.5)
+
+    # give instance a name
+    new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
+    return new_instance
+
+def terminate_and_clean(instances):
+    """
+    Some AMIs specify EBS stores that won't delete on instance termination.
+    These must be manually deleted after shutdown.
+    """
+    volumes_to_delete = []
+    for instance in instances:
+        for bdmap in instance.block_device_mappings:
+            if 'Ebs' in bdmap.keys():
+                if not bdmap['Ebs']['DeleteOnTermination']:
+                    volumes_to_delete.append(bdmap['Ebs']['VolumeId'])
+
+    for instance in instances:
+        instance.terminate()
+
+    # can't delete volumes until all attaching instances are terminated
+    _ids = [instance.id for instance in instances]
+    all_terminated = False
+    while not all_terminated:
+        all_terminated = True
+        for _id in _ids:
+            # necessary to reinit object for boto3 to get true state
+            inst = EC2.Instance(id=_id)
+            if inst.state['Name'] != 'terminated':
+                all_terminated = False
+        time.sleep(5)
+
+    for vol_id in volumes_to_delete:
+        volume = EC2.Volume(id=vol_id)
+        volume.delete()
+
+    return volumes_to_delete
+
+
+# Helper Routines
+#-------------------------------------------------------------------------------
+def block_until_http_ready(urlstring, wait_time=10, timeout=240):
+    "Blocks until server at urlstring can respond to http requests"
+    server_ready = False
+    t_elapsed = 0
+    while not server_ready and t_elapsed < timeout:
+        try:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            req = urllib2.Request(urlstring)
+            response = urllib2.urlopen(req)
+            #if response.code == 200:
+            server_ready = True
+        except urllib2.URLError:
+            pass
+        time.sleep(wait_time)
+        t_elapsed += wait_time
+
+def block_until_ssh_open(ipstring, wait_time=10, timeout=120):
+    "Blocks until server at ipstring has an open port 22"
+    reached = False
+    t_elapsed = 0
+    while not reached and t_elapsed < timeout:
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect((ipstring, 22))
+            reached = True
+        except socket.error as err:
+            time.sleep(wait_time)
+            t_elapsed += wait_time
+    sock.close()
+
+def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20):
+    "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections"
+    # the reinstantiation from id is necessary to force boto3
+    # to correctly update the 'state' variable during init
+    _id = booting_instance.id
+    _instance = EC2.Instance(id=_id)
+    _state = _instance.state['Name']
+    _ip = _instance.public_ip_address
+    while _state != 'running' or _ip is None:
+        time.sleep(wait_time)
+        _instance = EC2.Instance(id=_id)
+        _state = _instance.state['Name']
+        _ip = _instance.public_ip_address
+    block_until_ssh_open(_ip)
+    time.sleep(extra_wait_time)
+    return _instance
+
+
+# Fabric Routines
+#-------------------------------------------------------------------------------
+def local_git_clone(repo_url):
+    "clones master of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_branch(repo_url, branch_name):
+    "clones branch <branch_name> of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s --branch %s --single-branch'%(repo_url, branch_name))
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_PR(repo_url, PRnumstr, merge_master=True):
+    "clones specified pull request from repo_url and optionally merges into master"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr)
+        local('cd letsencrypt && git co lePRtest')
+        if merge_master:
+            local('cd letsencrypt && git remote update origin')
+            local('cd letsencrypt && git merge origin/master -m "testmerge"')
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_repo_to_remote():
+    "copies local tarball of repo to remote"
+    with lcd(LOGDIR):
+        put(local_path='le.tar.gz', remote_path='')
+        run('tar xzf le.tar.gz')
+
+def local_repo_clean():
+    "delete tarball"
+    with lcd(LOGDIR):
+        local('rm le.tar.gz')
+
+def deploy_script(scriptname, *args):
+    "copies to remote and executes local script"
+    with lcd('scripts'):
+        put(local_path=scriptname, remote_path='', mirror_local_mode=True)
+    args_str = ' '.join(args)
+    run('./'+scriptname+' '+args_str)
+
+def run_boulder():
+    with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
+        run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost')
+        run('nohup ./start.py >& /dev/null < /dev/null &')
+
+def config_and_launch_boulder(instance):
+    execute(deploy_script, 'boulder_config.sh')
+    execute(run_boulder)
+
+def install_and_launch_letsencrypt(instance, boulder_url):
+    execute(local_repo_to_remote)
+    with shell_env(BOULDER_URL=boulder_url):
+        execute(deploy_script, cl_args.test_script)
+
+def grab_letsencrypt_log():
+    "grabs letsencrypt.log via cat into logged stdout"
+    sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \
+    cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi')
+    # fallback file if /var/log is unwriteable...? correct?
+    sudo('if [ -f ./letsencrypt.log ]; then \
+    cat ./letsencrypt.log; else echo "[nolocallog]"; fi')
+
+#-------------------------------------------------------------------------------
+# SCRIPT BEGINS
+#-------------------------------------------------------------------------------
+
+# Set up local copy of git repo
+#-------------------------------------------------------------------------------
+LOGDIR = "letest-%d"%int(time.time())
+print("Making local dir for test repo and logs: %s"%LOGDIR)
+local('mkdir %s'%LOGDIR)
+
+# figure out what git object to test and locally create it in LOGDIR
+print("Making local git repo")
+try:
+    if cl_args.pull_request != '~':
+        print('Testing PR %s '%cl_args.pull_request,
+              "MERGING into master" if cl_args.merge_master else "")
+        execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
+    elif cl_args.branch != '~':
+        print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
+        execute(local_git_branch, cl_args.repo, cl_args.branch)
+    else:
+        print('Testing master of %s'%cl_args.repo)
+        execute(local_git_clone, cl_args.repo)
+except FabricException:
+    print("FAIL: trouble with git repo")
+    exit()
+
+
+# Set up EC2 instances
+#-------------------------------------------------------------------------------
+configdata = yaml.load(open(cl_args.config_file, 'r'))
+targetlist = configdata['targets']
+print('Testing against these images: [%d total]'%len(targetlist))
+for target in targetlist:
+    print(target['ami'], target['name'])
+
+print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
+AWS_SESSION = boto3.session.Session(profile_name=PROFILE)
+EC2 = AWS_SESSION.resource('ec2')
+
+print("Making Security Group")
+sg_exists = False
+for sg in EC2.security_groups.all():
+    if sg.group_name == 'letsencrypt_test':
+        sg_exists = True
+        print("  %s already exists"%'letsencrypt_test')
+if not sg_exists:
+    make_security_group()
+    time.sleep(30)
+
+print("Requesting Instances...")
+boulder_server = make_instance('le-boulderserver',
+                               BOULDER_AMI,
+                               KEYNAME,
+                               #machine_type='t2.micro',
+                               machine_type='t2.medium',
+                               security_groups=['letsencrypt_test'])
+
+instances = []
+for target in targetlist:
+    if target['virt'] == 'hvm':
+        machine_type = 't2.micro'
+    else:
+        machine_type = 't1.micro'
+    if 'userdata' in target.keys():
+        userdata = target['userdata']
+    else:
+        userdata = ''
+    instances.append( make_instance('le-%s'%target['name'],
+                                   target['ami'],
+                                   KEYNAME,
+                                   machine_type=machine_type,
+                                   userdata=userdata) )
+
+
+# Configure and launch boulder server
+#-------------------------------------------------------------------------------
+print("Waiting on Boulder Server")
+boulder_server = block_until_instance_ready(boulder_server)
+print(" server %s"%boulder_server)
+
+print("Configuring and Launching Boulder")
+
+# Fabric library controlled through global env parameters
+env.key_filename = KEYFILE
+env.shell = '/bin/bash -l -i -c'
+env.connection_attempts = 5
+env.timeout = 10
+# replace default SystemExit thrown by fabric during trouble
+class FabricException(Exception):
+    pass
+env['abort_exception'] = FabricException
+
+# env.host_string defines the ssh user and host for connection
+env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
+print("Boulder Server at (SSH):", env.host_string)
+config_and_launch_boulder(boulder_server)
+# blocking often unnecessary, but cheap EC2 VMs can get very slow
+block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
+                       wait_time=10,
+                       timeout=500)
+
+boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
+print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
+print("Boulder Server at (EC2 private ip): %s"%boulder_url)
+
+# Install and launch client scripts in parallel
+#-------------------------------------------------------------------------------
+print("Running letsencrypt clients in parallel - output routed to log files.")
+# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
+# the latter has implementation flaws that deadlock it in some circumstances)
+manager = Manager()
+outqueue = manager.Queue()
+inqueue = manager.Queue()
+SENTINEL = None #queue kill signal
+
+# launch as many processes as clients to test
+num_processes = len(targetlist)
+jobs = [] #keep a reference to current procs
+
+def test_client_process(inqueue, outqueue):
+    cur_proc = mp.current_process()
+    for inreq in iter(inqueue.get, SENTINEL):
+        ii, target = inreq
+
+        #save all stdout to log file
+        sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
+
+        print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
+        instances[ii] = block_until_instance_ready(instances[ii])
+        print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
+        env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
+        print(env.host_string)
+
+        try:
+            install_and_launch_letsencrypt(instances[ii], boulder_url)
+            outqueue.put((ii, target, 'pass'))
+            print("%s - %s SUCCESS"%(target['ami'], target['name']))
+        except:
+            outqueue.put((ii, target, 'fail'))
+            print("%s - %s FAIL"%(target['ami'], target['name']))
+            pass
+
+        # append server letsencrypt.log to each per-machine output log
+        print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
+        try:
+            execute(grab_letsencrypt_log)
+        except:
+            print("log fail\n")
+            pass
+
+# initiate process execution
+for i in range(num_processes):
+    p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
+    jobs.append(p)
+    p.daemon = True  # kills subprocesses if parent is killed
+    p.start()
+
+# fill up work queue
+for ii, target in enumerate(targetlist):
+    inqueue.put((ii, target))
+
+# add SENTINELs to end client processes
+for i in range(num_processes):
+    inqueue.put(SENTINEL)
+# wait on termination of client processes
+for p in jobs:
+    p.join()
+# add SENTINEL to output queue
+outqueue.put(SENTINEL)
+
+# clean up
+execute(local_repo_clean)
+
+# print and save summary results
+results_file = open(LOGDIR+'/results', 'w')
+outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
+outputs.sort(key=lambda x: x[0])
+for outq in outputs:
+    ii, target, status = outq
+    print('%d %s %s'%(ii, target['name'], status))
+    results_file.write('%d %s %s\n'%(ii, target['name'], status))
+results_file.close()
+
+if not cl_args.saveinstances:
+    print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
+    boulder_server.terminate()
+    terminate_and_clean(instances)
+else:
+    # print login information for the boxes for debugging
+    for ii, target in enumerate(targetlist):
+        print(target['name'],
+              target['ami'],
+              "%s@%s"%(target['user'],instances[ii].public_ip_address))
+
+# kill any connections
+fabric.network.disconnect_all()

scripts/boulder_config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Configures and Launches Boulder Server installed on
+# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab)
+
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# get local DNS resolver for VPC
+resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1)
+resolver=$resolver_ip':53'
+
+# modifies integration testing boulder setup for local AWS VPC network
+# connections instead of localhost
+cd $GOPATH/src/github.com/letsencrypt/boulder
+# configure boulder to receive outside connection on 4000
+sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+# change test ports to real
+sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json
+sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json
+sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json
+# set local dns resolver
+sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json
+
+# start rabbitMQ
+#go run cmd/rabbitmq-setup/main.go -server amqp://localhost
+# start acme services
+#nohup ./start.py >& /dev/null < /dev/null &
+#./start.py

scripts/boulder_install.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# >>>> only tested on Ubuntu 14.04LTS <<<<
+
+# non-interactive install of mariadb and other dependencies
+export DEBIAN_FRONTEND=noninteractive
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS'
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS'
+apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server
+sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');"
+
+# install go
+wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+tar xzvf go1.5.1.linux-amd64.tar.gz
+mkdir gocode
+echo "export GOROOT=/home/ubuntu/go \n\
+      export GOPATH=/home/ubuntu/gocode\n\
+      export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc
+
+# install boulder and its go dependencies
+go get -d github.com/letsencrypt/boulder/...
+cd $GOPATH/src/github.com/letsencrypt/boulder
+wget https://github.com/jsha/boulder-tools/raw/master/goose.gz
+mkdir $GOPATH/bin
+zcat goose.gz > $GOPATH/bin/goose
+chmod +x $GOPATH/bin/goose
+./test/create_db.sh
+go get github.com/jsha/listenbuddy

scripts/test_letsencrypt_auto_apache2.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -x
+
+#install apache2 on apt systems
+# debian doesn't come with curl
+sudo apt-get update
+sudo apt-get -y --no-upgrade install apache2 curl
+
+# $BOULDER_URL is dynamically set at execution
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# For apache 2.4, set up ServerName
+sudo sed -i '/ServerName/ s/#ServerName/ServerName/' \
+     /etc/apache2/sites-available/000-default.conf
+sudo sed -i '/ServerName/ s/www.example.com/'$public_host'/' \
+    /etc/apache2/sites-available/000-default.conf
+
+# run letsencrypt-apache2 via letsencrypt-auto
+cd letsencrypt
+./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect --register-unsafely-without-email \
+                   --domain $public_host --server $BOULDER_URL

scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -x
+
+# $BOULDER_URL is dynamically set at execution
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+cd letsencrypt
+./letsencrypt-auto certonly -v --standalone --debug \
+                   --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect \
+                   --register-unsafely-without-email \
+                   --domain $public_host --server $BOULDER_URL

scripts/test_letsencrypt_auto_venv_only.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -x
+
+# $BOULDER_URL is dynamically set at execution
+
+cd letsencrypt
+# help installs virtualenv and does nothing else
+./letsencrypt-auto -v --help all

targets.yaml

@@ -0,0 +1,99 @@
+targets:
+  #-----------------------------------------------------------------------------
+  #Ubuntu
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-0611546c
+    name: ubuntu12.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  #-----------------------------------------------------------------------------
+  # Debian
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  - ami: ami-e0efab88
+    name: debian7.8.aws.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  - ami: ami-e6eeaa8e
+    name: debian7.8.aws.1_32bit
+    type: debian
+    virt: pv
+    user: admin
+    userdata: |
+      cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Other Redhat Distros
+  - ami: ami-60b6c60a
+    name: amazonlinux-2015.09.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-0d4cfd66
+    name: amazonlinux-2015.03.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-a8d369c0
+    name: RHEL7
+    type: redhat
+    virt: hvm
+    user: ec2-user
+  - ami: ami-518bfb3b
+    name: fedora23
+    type: fedora
+    virt: hvm
+    user: fedora
+  #-----------------------------------------------------------------------------
+  # CentOS
+  # These Marketplace AMIs must, irritatingly, have their terms manually
+  # agreed to on the AWS marketplace site for any new AWS account using them...
+  # - ami: ami-61bbf104
+  #   name: centos7
+  #   type: centos
+  #   virt: hvm
+  #   user: centos
+  # # centos6 requires EPEL repo added
+  # - ami: ami-57cd8732
+  #   name: centos6
+  #   type: centos
+  #   virt: hvm
+  #   user: centos
+  #   userdata: |
+  #     #cloud-config
+  #     runcmd:
+  #       - [ yum, install, -y, epel-release ]
+  #       - [ iptables, -F ]