Move OS-package bootstrappers to a private folder.

They're now used only by the le-auto build process. The new public interface for OS-level bootstrapping is le-auto --os-packages-only, which dispatches by OS automatically. That obsoletes install-deps.sh as well, saving some repetition.

Also, switch to mustache-style templating to avoid colliding with shell variable references.

To optimize for the docker cache, we could later add a shim script that sources just deb_common.sh and calls its bootstrap function.

Dockerfile

@@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt
 # directories in its path.
 
 
-COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh
+COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto
-RUN /opt/letsencrypt/src/ubuntu.sh && \
+RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \
            /tmp/* \

Dockerfile-dev

@@ -22,8 +22,8 @@ WORKDIR /opt/letsencrypt
 
 # TODO: Install non-default Python versions for tox.
 # TODO: Install Apache/Nginx for plugin development.
-COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/ubuntu.sh
+COPY letsencrypt_auto/letsencrypt-auto /opt/letsencrypt/src/letsencrypt-auto
-RUN /opt/letsencrypt/src/ubuntu.sh && \
+RUN /opt/letsencrypt/src/letsencrypt-auto --os-packages-only && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \
            /tmp/* \

Vagrantfile

@@ -7,7 +7,7 @@ VAGRANTFILE_API_VERSION = "2"
 # Setup instructions from docs/contributing.rst
 $ubuntu_setup_script = <<SETUP_SCRIPT
 cd /vagrant
-./bootstrap/install-deps.sh
+./letsencrypt-auto/letsencrypt-auto --os-packages-only
 ./bootstrap/dev/venv.sh
 SETUP_SCRIPT
 

bootstrap/_arch_common.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-
-# Tested with:
-#   - ArchLinux (x86_64)
-#
-# "python-virtualenv" is Python3, but "python2-virtualenv" provides
-# only "virtualenv2" binary, not "virtualenv" necessary in
-# ./bootstrap/dev/_common_venv.sh
-
-deps="
-  git
-  python2
-  python-virtualenv
-  gcc
-  dialog
-  augeas
-  openssl
-  libffi
-  ca-certificates
-  pkg-config
-"
-
-missing=$("$SUDO" pacman -T $deps)
-
-if [ "$missing" ]; then
-  "$SUDO" pacman -S --needed $missing
-fi

bootstrap/_deb_common.sh

@@ -1,50 +0,0 @@
-#!/bin/sh
-
-# Current version tested with:
-#
-# - Ubuntu
-#     - 14.04 (x64)
-#     - 15.04 (x64)
-# - Debian
-#     - 7.9 "wheezy" (x64)
-#     - sid (2015-10-21) (x64)
-
-# Past versions tested with:
-#
-# - Debian 8.0 "jessie" (x64)
-# - Raspbian 7.8 (armhf)
-
-# Believed not to work:
-#
-# - Debian 6.0.10 "squeeze" (x64)
-
-apt-get update
-
-# virtualenv binary can be found in different packages depending on
-# distro version (#346)
-
-virtualenv=
-if apt-cache show virtualenv > /dev/null ; then
-  virtualenv="virtualenv"
-fi
-
-if apt-cache show python-virtualenv > /dev/null ; then
-  virtualenv="$virtualenv python-virtualenv"
-fi
-
-apt-get install -y --no-install-recommends \
-  git \
-  python \
-  python-dev \
-  $virtualenv \
-  gcc \
-  dialog \
-  libaugeas0 \
-  libssl-dev \
-  libffi-dev \
-  ca-certificates \
-
-if ! command -v virtualenv > /dev/null ; then
-  echo Failed to install a working \"virtualenv\" command, exiting
-  exit 1
-fi

bootstrap/_gentoo_common.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-
-PACKAGES="dev-vcs/git
-  dev-lang/python:2.7
-  dev-python/virtualenv
-  dev-util/dialog
-  app-admin/augeas
-  dev-libs/openssl
-  dev-libs/libffi
-  app-misc/ca-certificates
-  virtual/pkgconfig"
-
-case "$PACKAGE_MANAGER" in
-  (paludis)
-    "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x
-    ;;
-  (pkgcore)
-    "$SUDO" pmerge --noreplace $PACKAGES
-    ;;
-  (portage|*)
-    "$SUDO" emerge --noreplace $PACKAGES
-    ;;
-esac

bootstrap/_rpm_common.sh

@@ -1,49 +0,0 @@
-#!/bin/sh
-
-# Tested with:
-#   - Fedora 22, 23 (x64)
-#   - Centos 7 (x64: onD igitalOcean droplet)
-
-if type dnf 2>/dev/null
-then
-  tool=dnf
-elif type yum 2>/dev/null
-then
-  tool=yum
-
-else
-  echo "Neither yum nor dnf found. Aborting bootstrap!"
-  exit 1
-fi
-
-# Some distros and older versions of current distros use a "python27"
-# instead of "python" naming convention. Try both conventions.
-if ! $tool install -y \
-       python \
-       python-devel \
-       python-virtualenv
-then
-  if ! $tool install -y \
-         python27 \
-         python27-devel \
-         python27-virtualenv
-  then
-    echo "Could not install Python dependencies. Aborting bootstrap!"
-    exit 1
-  fi
-fi
-
-# "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails)
-if ! $tool install -y \
-       git-core \
-       gcc \
-       dialog \
-       augeas-libs \
-       openssl-devel \
-       libffi-devel \
-       redhat-rpm-config \
-       ca-certificates
-then
-    echo "Could not install additional dependencies. Aborting bootstrap!"
-    exit 1
-fi

bootstrap/_suse_common.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# SLE12 don't have python-virtualenv
-
-zypper -nq in -l git-core \
-  python \
-  python-devel \
-  python-virtualenv \
-  gcc \
-  dialog \
-  augeas-lenses \
-  libopenssl-devel \
-  libffi-devel \
-  ca-certificates \

bootstrap/archlinux.sh

@@ -1 +0,0 @@
-_arch_common.sh

bootstrap/centos.sh

@@ -1 +0,0 @@
-_rpm_common.sh

bootstrap/debian.sh

@@ -1 +0,0 @@
-_deb_common.sh

bootstrap/fedora.sh

@@ -1 +0,0 @@
-_rpm_common.sh

bootstrap/freebsd.sh

@@ -1,8 +0,0 @@
-#!/bin/sh -xe
-
-"$SUDO" pkg install -Ay \
-  git \
-  python \
-  py27-virtualenv \
-  augeas \
-  libffi \

bootstrap/gentoo.sh

@@ -1 +0,0 @@
-_gentoo_common.sh

bootstrap/install-deps.sh

@@ -1,46 +0,0 @@
-#!/bin/sh -e
-#
-# Install OS dependencies.  In the glorious future, letsencrypt-auto will
-# source this...
-
-if test "`id -u`" -ne "0" ; then
-  SUDO=sudo
-else
-  SUDO=
-fi
-
-BOOTSTRAP=`dirname $0`
-if [ ! -f $BOOTSTRAP/debian.sh ] ; then
-  echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP"
-  exit 1
-fi
-if [ -f /etc/debian_version ] ; then
-  echo "Bootstrapping dependencies for Debian-based OSes..."
-  $SUDO $BOOTSTRAP/_deb_common.sh
-elif [ -f /etc/arch-release ] ; then
-  echo "Bootstrapping dependencies for Archlinux..."
-  $SUDO $BOOTSTRAP/archlinux.sh
-elif [ -f /etc/redhat-release ] ; then
-  echo "Bootstrapping dependencies for RedHat-based OSes..."
-  $SUDO $BOOTSTRAP/_rpm_common.sh
-elif [ -f /etc/gentoo-release ] ; then
-  echo "Bootstrapping dependencies for Gentoo-based OSes..."
-  $SUDO $BOOTSTRAP/_gentoo_common.sh
-elif uname | grep -iq FreeBSD ; then
-  echo "Bootstrapping dependencies for FreeBSD..."
-  $SUDO $BOOTSTRAP/freebsd.sh
-elif `grep -qs openSUSE /etc/os-release` ; then
-  echo "Bootstrapping dependencies for openSUSE.."
-  $SUDO $BOOTSTRAP/suse.sh
-elif uname | grep -iq Darwin ; then
-  echo "Bootstrapping dependencies for Mac OS X..."
-  echo "WARNING: Mac support is very experimental at present..."
-  $BOOTSTRAP/mac.sh
-else
-  echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!"
-  echo
-  echo "You will need to bootstrap, configure virtualenv, and run a pip install manually"
-  echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites"
-  echo "for more info"
-  exit 1
-fi

bootstrap/mac.sh

@@ -1,18 +0,0 @@
-#!/bin/sh -e
-if ! hash brew 2>/dev/null; then
-    echo "Homebrew Not Installed\nDownloading..."
-    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-fi
-
-brew install augeas
-brew install dialog
-
-if ! hash pip 2>/dev/null; then
-    echo "pip Not Installed\nInstalling python from Homebrew..."
-    brew install python
-fi
-
-if ! hash virtualenv 2>/dev/null; then
-    echo "virtualenv Not Installed\nInstalling with pip"
-    pip install virtualenv
-fi

bootstrap/manjaro.sh

@@ -1 +0,0 @@
-_arch_common.sh

bootstrap/suse.sh

@@ -1 +0,0 @@
-_suse_common.sh

bootstrap/ubuntu.sh

@@ -1 +0,0 @@
-_deb_common.sh

docs/contributing.rst

@@ -22,7 +22,7 @@ once:
 
    git clone https://github.com/letsencrypt/letsencrypt
    cd letsencrypt
-   ./bootstrap/install-deps.sh
+   ./letsencrypt-auto/letsencrypt-auto --os-packages-only
    ./bootstrap/dev/venv.sh
 
 Then in each shell where you're working on the client, do:

letsencrypt_auto/letsencrypt-auto.template

@@ -49,6 +49,14 @@ DeterminePythonVersion() {
   fi
 }
 
+{{bootstrap_deb_common}}
+{{bootstrap_rpm_common}}
+{{bootstrap_suse_common}}
+{{bootstrap_arch_common}}
+{{bootstrap_gentoo_common}}
+{{bootstrap_free_bsd}}
+{{bootstrap_mac}}
+
 # Install required OS packages:
 Bootstrap() {
   if [ -f /etc/debian_version ] ; then
@@ -63,7 +71,7 @@ Bootstrap() {
   elif [ -f /etc/arch-release ] ; then
     if [ "$DEBUG" = 1 ] ; then
       echo "Bootstrapping dependencies for Archlinux..."
-      BootstrapArchLinux
+      BootstrapArchCommon
     else
       echo "Please use pacman to install letsencrypt packages:"
       echo "# pacman -S letsencrypt letsencrypt-apache"
@@ -73,7 +81,7 @@ Bootstrap() {
       exit 1
     fi
   elif [ -f /etc/manjaro-release ] ; then
-    ExperimentalBootstrap "Manjaro Linux" BootstrapManjaro
+    ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon
   elif [ -f /etc/gentoo-release ] ; then
     ExperimentalBootstrap "Gentoo" BootstrapGentooCommon
   elif uname | grep -iq FreeBSD ; then
@@ -172,7 +180,7 @@ elif [ "$1" != "--_skip-to-install" ]; then
     # There is no $ interpolation due to quotes on heredoc delimiters.
     set +e
     TEMP_DIR=`$PYTHON - << "UNLIKELY_EOF"
-{download_upgrade}
+{{download_upgrade}}
 UNLIKELY_EOF`
     DOWNLOAD_STATUS=$?
     set -e
@@ -197,11 +205,11 @@ else  # --_skip-to-install was passed.
   TEMP_DIR="$2"
   shift 2
   cat << "UNLIKELY_EOF" > $TEMP_DIR/letsencrypt-auto-requirements.txt
-{requirements}
+{{requirements}}
 UNLIKELY_EOF
 
   cat << "UNLIKELY_EOF" > $TEMP_DIR/peep.py
-{peep}
+{{peep}}
 UNLIKELY_EOF
 
   set +e

letsencrypt_auto/pieces/bootstrappers/arch_common.sh

@@ -0,0 +1,27 @@
+BootstrapArchCommon() {
+  # Tested with:
+  #   - ArchLinux (x86_64)
+  #
+  # "python-virtualenv" is Python3, but "python2-virtualenv" provides
+  # only "virtualenv2" binary, not "virtualenv" necessary in
+  # ./bootstrap/dev/_common_venv.sh
+
+  deps="
+    git
+    python2
+    python-virtualenv
+    gcc
+    dialog
+    augeas
+    openssl
+    libffi
+    ca-certificates
+    pkg-config
+  "
+
+  missing=$("$SUDO" pacman -T $deps)
+
+  if [ "$missing" ]; then
+    "$SUDO" pacman -S --needed $missing
+  fi
+}

letsencrypt_auto/pieces/bootstrappers/deb_common.sh

@@ -0,0 +1,50 @@
+BootstrapDebCommon() {
+  # Current version tested with:
+  #
+  # - Ubuntu
+  #     - 14.04 (x64)
+  #     - 15.04 (x64)
+  # - Debian
+  #     - 7.9 "wheezy" (x64)
+  #     - sid (2015-10-21) (x64)
+
+  # Past versions tested with:
+  #
+  # - Debian 8.0 "jessie" (x64)
+  # - Raspbian 7.8 (armhf)
+
+  # Believed not to work:
+  #
+  # - Debian 6.0.10 "squeeze" (x64)
+
+  $SUDO apt-get update
+
+  # virtualenv binary can be found in different packages depending on
+  # distro version (#346)
+
+  virtualenv=
+  if apt-cache show virtualenv > /dev/null ; then
+    virtualenv="virtualenv"
+  fi
+
+  if apt-cache show python-virtualenv > /dev/null ; then
+    virtualenv="$virtualenv python-virtualenv"
+  fi
+
+  $SUDO apt-get install -y --no-install-recommends \
+    git \
+    python \
+    python-dev \
+    $virtualenv \
+    gcc \
+    dialog \
+    libaugeas0 \
+    libssl-dev \
+    libffi-dev \
+    ca-certificates \
+
+  if ! command -v virtualenv > /dev/null ; then
+    echo Failed to install a working \"virtualenv\" command, exiting
+    exit 1
+  fi
+}

letsencrypt_auto/pieces/bootstrappers/freebsd.sh

@@ -0,0 +1,8 @@
+BootstrapFreeBsd() {
+  "$SUDO" pkg install -Ay \
+    git \
+    python \
+    py27-virtualenv \
+    augeas \
+    libffi \
+}

letsencrypt_auto/pieces/bootstrappers/gentoo_common.sh

@@ -0,0 +1,23 @@
+BootstrapGentooCommon() {
+  PACKAGES="dev-vcs/git
+    dev-lang/python:2.7
+    dev-python/virtualenv
+    dev-util/dialog
+    app-admin/augeas
+    dev-libs/openssl
+    dev-libs/libffi
+    app-misc/ca-certificates
+    virtual/pkgconfig"
+
+  case "$PACKAGE_MANAGER" in
+    (paludis)
+      "$SUDO" cave resolve --keep-targets if-possible $PACKAGES -x
+      ;;
+    (pkgcore)
+      "$SUDO" pmerge --noreplace $PACKAGES
+      ;;
+    (portage|*)
+      "$SUDO" emerge --noreplace $PACKAGES
+      ;;
+  esac
+}

letsencrypt_auto/pieces/bootstrappers/mac.sh

@@ -0,0 +1,19 @@
+BootstrapMac() {
+  if ! hash brew 2>/dev/null; then
+      echo "Homebrew Not Installed\nDownloading..."
+      ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+  fi
+
+  brew install augeas
+  brew install dialog
+
+  if ! hash pip 2>/dev/null; then
+      echo "pip Not Installed\nInstalling python from Homebrew..."
+      brew install python
+  fi
+
+  if ! hash virtualenv 2>/dev/null; then
+      echo "virtualenv Not Installed\nInstalling with pip"
+      pip install virtualenv
+  fi
+}

letsencrypt_auto/pieces/bootstrappers/rpm_common.sh

@@ -0,0 +1,49 @@
+BootstrapRpmCommon() {
+  # Tested with:
+  #   - Fedora 22, 23 (x64)
+  #   - Centos 7 (x64: onD igitalOcean droplet)
+
+  if type dnf 2>/dev/null
+  then
+    tool=dnf
+  elif type yum 2>/dev/null
+  then
+    tool=yum
+
+  else
+    echo "Neither yum nor dnf found. Aborting bootstrap!"
+    exit 1
+  fi
+
+  # Some distros and older versions of current distros use a "python27"
+  # instead of "python" naming convention. Try both conventions.
+  if ! $SUDO $tool install -y \
+         python \
+         python-devel \
+         python-virtualenv
+  then
+    if ! $SUDO $tool install -y \
+           python27 \
+           python27-devel \
+           python27-virtualenv
+    then
+      echo "Could not install Python dependencies. Aborting bootstrap!"
+      exit 1
+    fi
+  fi
+
+  # "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails)
+  if ! $SUDO $tool install -y \
+         git-core \
+         gcc \
+         dialog \
+         augeas-libs \
+         openssl-devel \
+         libffi-devel \
+         redhat-rpm-config \
+         ca-certificates
+  then
+      echo "Could not install additional dependencies. Aborting bootstrap!"
+      exit 1
+  fi
+}

letsencrypt_auto/pieces/bootstrappers/suse_common.sh

@@ -0,0 +1,14 @@
+BootstrapSuseCommon() {
+  # SLE12 don't have python-virtualenv
+
+  $SUDO zypper -nq in -l git-core \
+    python \
+    python-devel \
+    python-virtualenv \
+    gcc \
+    dialog \
+    augeas-lenses \
+    libopenssl-devel \
+    libffi-devel \
+    ca-certificates \
+}