Merge branch 'master' into squashed-postfix

2026-08-03 03:32:13 +02:00 · 2018-06-15 16:21:15 -07:00
parent 4ba153949d adc07ef933
commit 40c50231ed
377 changed files with 20544 additions and 7996 deletions
@@ -15,10 +15,10 @@ mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
 virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
 . ./$VENV_NAME/bin/activate

-# Separately install setuptools and pip to make sure following
-# invocations use latest
-pip install -U pip
-pip install -U setuptools
+# Use pipstrap to update Python packaging tools to only update to a well tested
+# version and to work around https://github.com/pypa/pip/issues/4817 on older
+# systems.
+python letsencrypt-auto-source/pieces/pipstrap.py
 ./tools/pip_install.sh "$@"

 set +x
@@ -18,10 +18,10 @@ import sys
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives import serialization
+import josepy as jose

 from acme import client as acme_client
 from acme import errors as acme_errors
-from acme import jose
 from acme import messages

 DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory')
@@ -0,0 +1,74 @@
+# Specifies Python package versions for packages not specified in
+# letsencrypt-auto's requirements file.
+alabaster==0.7.10
+apipkg==1.4
+asn1crypto==0.22.0
+astroid==1.3.5
+attrs==17.3.0
+Babel==2.5.1
+backports.shutil-get-terminal-size==1.0.0
+boto3==1.4.7
+botocore==1.7.41
+cloudflare==1.5.1
+coverage==4.4.2
+decorator==4.1.2
+dns-lexicon==2.2.1
+dnspython==1.15.0
+docutils==0.14
+execnet==1.5.0
+future==0.16.0
+futures==3.1.1
+google-api-python-client==1.5
+httplib2==0.10.3
+imagesize==0.7.1
+ipdb==0.10.2
+ipython==5.5.0
+ipython-genutils==0.2.0
+Jinja2==2.9.6
+jmespath==0.9.3
+josepy==1.0.1
+logger==1.4
+logilab-common==1.4.1
+MarkupSafe==1.0
+mypy==0.600
+ndg-httpsclient==0.3.2
+oauth2client==2.0.0
+pathlib2==2.3.0
+pexpect==4.2.1
+pickleshare==0.7.4
+pkginfo==1.4.2
+pluggy==0.5.2
+prompt-toolkit==1.0.15
+ptyprocess==0.5.2
+py==1.4.34
+pyasn1==0.1.9
+pyasn1-modules==0.0.10
+Pygments==2.2.0
+pylint==1.4.2
+pytest==3.2.5
+pytest-cov==2.5.1
+pytest-forked==0.2
+pytest-xdist==1.20.1
+python-dateutil==2.6.1
+python-digitalocean==1.11
+PyYAML==3.12
+repoze.sphinx.autointerface==0.8
+requests-file==1.4.2
+requests-toolbelt==0.8.0
+rsa==3.4.2
+s3transfer==0.1.11
+scandir==1.6
+simplegeneric==0.8.1
+snowballstemmer==1.2.1
+Sphinx==1.5.6
+sphinx-rtd-theme==0.2.4
+tldextract==2.2.0
+tox==2.9.1
+tqdm==4.19.4
+traitlets==4.3.2
+twine==1.11.0
+typed-ast==1.1.0
+typing==3.6.4
+uritemplate==0.6
+virtualenv==15.1.0
+wcwidth==0.1.7
@@ -2,20 +2,28 @@
 # pip installs the requested packages in editable mode and runs unit tests on
 # them. Each package is installed and tested in the order they are provided
 # before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using certbot-auto's requirements file as
-# constraints.
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.sh for more information on the versions pinned
+# to.

 if [ "$CERTBOT_NO_PIN" = 1 ]; then
-  pip_install="pip install -e"
+  pip_install="pip install -q -e"
 else
  pip_install="$(dirname $0)/pip_install_editable.sh"
 fi

+temp_cwd=$(mktemp -d)
+trap "rm -rf $temp_cwd" EXIT
+
+set -x
 for requirement in "$@" ; do
  $pip_install $requirement
  pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
+  pkg=$(echo "$pkg" | tr - _ )  # convert package names to Python import names
  if [ $pkg = "." ]; then
    pkg="certbot"
  fi
-  nosetests -v $pkg --processes=-1 --process-timeout=100
+  cd "$temp_cwd"
+  pytest --numprocesses auto --quiet --pyargs $pkg
+  cd -
 done
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""Merges multiple Python requirements files into one file.
+
+Requirements files specified later take precedence over earlier ones. Only
+simple SomeProject==1.2.3 format is currently supported.
+
+"""
+
+from __future__ import print_function
+
+import sys
+
+
+def read_file(file_path):
+    """Reads in a Python requirements file.
+
+    :param str file_path: path to requirements file
+
+    :returns: mapping from a project to its pinned version
+    :rtype: dict
+
+    """
+    d = {}
+    with open(file_path) as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith('#'):
+                project, version = line.split('==')
+                if not version:
+                    raise ValueError("Unexpected syntax '{0}'".format(line))
+                d[project] = version
+    return d
+
+
+def print_requirements(requirements):
+    """Prints requirements to stdout.
+
+    :param dict requirements: mapping from a project to its pinned version
+
+    """
+    print('\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items())))
+
+
+def merge_requirements_files(*files):
+    """Merges multiple requirements files together and prints the result.
+
+    Requirement files specified later in the list take precedence over earlier
+    files.
+
+    :param tuple files: paths to requirements files
+
+    """
+    d = {}
+    for f in files:
+        d.update(read_file(f))
+    print_requirements(d)
+
+
+if __name__ == '__main__':
+    merge_requirements_files(*sys.argv[1:])
@@ -2,17 +2,17 @@

 set -o errexit

-if ! `which festival > /dev/null` ; then
-    echo Please install \'festival\'!
-    exit 1
-fi
-
 function sayhash { # $1 <-- HASH ; $2 <---SIGFILEBALL
  while read -p "Press Enter to read the hash aloud or type 'done':  " INP && [ "$INP" = "" ] ; do
-    cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \
-                echo -n '(SayText "'; \
-                sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \
-                echo '")' ) | festival
+    if ! `which festival > /dev/null` ; then
+      echo \`festival\` is not installed!
+      echo Please install it to read the hash aloud
+    else
+      cat $1 | (echo "(Parameter.set 'Duration_Stretch 1.8)"; \
+                  echo -n '(SayText "'; \
+                  sha256sum | cut -c1-64 | fold -1 | sed 's/^a$/alpha/; s/^b$/bravo/; s/^c$/charlie/; s/^d$/delta/; s/^e$/echo/; s/^f$/foxtrot/'; \
+                  echo '")' ) | festival
+    fi
  done

  echo 'Paste in the data from the QR code, then type Ctrl-D:'
@@ -0,0 +1,51 @@
+# This file contains the oldest versions of our dependencies we say we require
+# in our packages or versions we need to support to maintain compatibility with
+# the versions included in the various Linux distros where we are packaged.
+
+# CentOS/RHEL 7 EPEL constraints
+cffi==1.6.0
+chardet==2.2.1
+configobj==4.7.2
+ipaddress==1.0.16
+mock==1.0.1
+ndg-httpsclient==0.3.2
+ply==3.4
+pyasn1==0.1.9
+pycparser==2.14
+pyOpenSSL==0.13.1
+pyparsing==1.5.6
+pyRFC3339==1.0
+python-augeas==0.5.0
+six==1.9.0
+# setuptools 0.9.8 is the actual version packaged, but some other dependencies
+# in this file require setuptools>=1.0 and there are no relevant changes for us
+# between these versions.
+setuptools==1.0.0
+urllib3==1.10.2
+zope.component==4.1.0
+zope.event==4.0.3
+zope.interface==4.0.5
+
+# Debian Jessie Backports constraints
+PyICU==1.8
+colorama==0.3.2
+enum34==1.0.3
+html5lib==0.999
+idna==2.0
+pbr==1.8.0
+pytz==2012rc0
+
+# Our setup.py constraints
+cloudflare==1.5.1
+cryptography==1.2.0
+google-api-python-client==1.5
+oauth2client==2.0
+parsedatetime==1.3
+pyparsing==1.5.5
+python-digitalocean==1.11
+requests[security]==2.4.1
+
+# Ubuntu Xenial constraints
+ConfigArgParse==0.10.0
+funcsigs==0.4
+zope.hookable==4.0.4
@@ -1,14 +1,44 @@
 #!/bin/sh -e
-# pip installs packages using Certbot's requirements file as constraints
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt,
+# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# top level of the package's directory is used, otherwise, a combination of
+# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
+# other file always takes precedence over tools/dev_constraints.txt. If
+# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
+# no other arguments.

 # get the root of the Certbot repo
-my_path=$("$(dirname $0)/readlink.py" $0)
-repo_root=$(dirname $(dirname $my_path))
-requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
-constraints=$(mktemp)
-trap "rm -f $constraints" EXIT
-# extract pinned requirements without hashes
-sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $constraints
+tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
+all_constraints=$(mktemp)
+test_constraints=$(mktemp)
+trap "rm -f $all_constraints $test_constraints" EXIT
+
+if [ "$CERTBOT_OLDEST" = 1 ]; then
+    if [ "$1" != "-e" -o "$#" -ne "2" ]; then
+        echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument."
+        exit 1
+    fi
+    pkg_dir=$(echo $2 | cut -f1 -d\[)  # remove any extras such as [dev]
+    requirements="$pkg_dir/local-oldest-requirements.txt"
+    # packages like acme don't have any local oldest requirements
+    if [ ! -f "$requirements" ]; then
+        unset requirements
+    fi
+    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
+else
+    repo_root=$(dirname "$tools_dir")
+    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
+    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
+fi
+
+"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \
+                                   "$test_constraints" > "$all_constraints"
+
+set -x

 # install the requested packages using the pinned requirements as constraints
-pip install --constraint $constraints "$@"
+if [ -n "$requirements" ]; then
+    pip install -q --constraint "$all_constraints" --requirement "$requirements"
+fi
+pip install -q --constraint "$all_constraints" "$@"
@@ -55,7 +55,7 @@ SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO"
 subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)"
 # certbot_compatibility_test is not packaged because:
 # - it is not meant to be used by anyone else than Certbot devs
-# - it causes problems when running nosetests - the latter tries to
+# - it causes problems when running pytest - the latter tries to
 #   run everything that matches test*, while there are no unittests
 #   there

@@ -166,10 +166,10 @@ fi
 mkdir kgs
 kgs="kgs/$version"
 pip freeze | tee $kgs
-pip install nose
+pip install pytest
 for module in $subpkgs_modules ; do
    echo testing $module
-    nosetests $module
+    pytest --pyargs $module
 done
 cd ~-

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""A version of Python 2.x's SimpleHTTPServer that flushes its output."""
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+import sys
+
+def serve_forever(port=0):
+    """Spins up an HTTP server on all interfaces and the given port.
+
+    A message is printed to stdout specifying the address and port being used
+    by the server.
+
+    :param int port: port number to use.
+
+    """
+    server = HTTPServer(('', port), SimpleHTTPRequestHandler)
+    print('Serving HTTP on {0} port {1} ...'.format(*server.server_address))
+    sys.stdout.flush()
+    server.serve_forever()
+
+
+if __name__ == '__main__':
+    kwargs = {}
+    if len(sys.argv) > 1:
+        kwargs['port'] = int(sys.argv[1])
+    serve_forever(**kwargs)
@@ -25,7 +25,7 @@ API Documentation
   :glob:

   api/**" > api.rst
-sed -i -e "s|   :caption: Contents:|   :caption: Contents:\n\n.. toctree::\n   :maxdepth: 1\n\n   api\n\n.. automodule:: ${PROJECT//-/_}\n   :members:|" index.rst
+sed -i -e "s|   :caption: Contents:|   :caption: Contents:\n\n.. automodule:: ${PROJECT//-/_}\n   :members:\n\n.. toctree::\n   :maxdepth: 1\n\n   api|" index.rst

 echo "Suggested next steps:
 * Add API docs to: $PROJECT/docs/api/