Finished script besides commit and started unit tests

This commit is contained in:
Brad Warren
2015-07-27 19:26:28 -07:00
parent 0a60122cf6
commit 12b9970787
10 changed files with 493 additions and 200 deletions
@@ -0,0 +1 @@
"""Tools for submitting server configurations"""
@@ -0,0 +1,302 @@
#!/usr/bin/env python
"""Let's Encrypt Apache configuration submission script"""
import argparse
import os
import subprocess
import sys
import tarfile
import tempfile
import textwrap
_DESCRIPTION = """
Let's Help is a simple script you can run to help out the Let's Encrypt
project. Since Let's Encrypt will support automatically configuring HTTPS on
many servers, we want to test this functionality on as many configurations as
possible. This script will create a sanitized copy of your Apache
configuration, notifying you of the files that have been selected. If (and only
if) you approve this selection, these files will be sent to the Let's Encrypt
developers.
"""
_NO_APACHECTL = """
Unable to find `apachectl` which is required for this script to work. If it is
installed, please run this script again with the --apache-ctl command line
argument and the path to the binary.
"""
def make_and_verify_selection(server_root, temp_dir):
"""Copies server_root to temp_dir and verifies selection with the user
:param str server_root: Path to the Apache server root
:param str temp_dir: Path to the temporary directory to copy files to
"""
copied_files, copied_dirs = copy_config(server_root, temp_dir)
print textwrap.fill("A secure copy of the files that have been selected "
"for submission has been created under {0}. All "
"comments have been removed and the files are only "
"accessible by the current user. A list of the files "
"that have been included is shown below. Please make "
"sure that this selection does not contain private "
"keys, passwords, or any other sensitive "
"information.".format(temp_dir))
print "\nFiles:"
for copied_file in copied_files:
print copied_file
print "Directories (including all contained files):"
for copied_dir in copied_dirs:
print copied_dir
sys.stdout.write("\nIs it safe to submit these files? ")
while True:
ans = raw_input("(Y)es/(N)o: ")
if ans.startswith("Y") or ans.startswith("y"):
return
elif ans.startswith("N") or ans.startswith("n"):
sys.exit("Your files were not submitted")
def copy_config(server_root, temp_dir):
"""Safely copies server_root to temp_dir and returns copied files
:param str server_root: Absolute path to the Apache server root
:param str temp_dir: Path to the temporary directory to copy files to
:returns: List of copied files and a list of leaf directories where
all contained files were copied
:rtype: `tuple` of `list` of `str`
"""
copied_files, copied_dirs = list(), list()
dir_len = len(os.path.dirname(server_root))
for config_path, config_dirs, config_files in os.walk(server_root):
temp_path = os.path.join(temp_dir, config_path[dir_len+1:])
os.mkdir(temp_path)
copied_all = True
copied_files_in_current_dir = list()
for config_file in config_files:
config_file_path = os.path.join(config_path, config_file)
temp_file_path = os.path.join(temp_path, config_file)
if os.path.islink(config_file_path):
os.symlink(os.readlink(config_file_path), temp_file_path)
elif safe_config_file(config_file_path):
copy_file_without_comments(config_file_path, temp_file_path)
copied_files_in_current_dir.append(config_file_path)
else:
copied_all = False
# If copied all files in leaf directory
if copied_all and not config_dirs:
copied_dirs.append(config_path)
else:
copied_files += copied_files_in_current_dir
return copied_files, copied_dirs
def copy_file_without_comments(source, destination):
"""Copies source to destination, removing comments
:param str source: Path to the file to be copied
:param str destination: Path where source should be copied to
"""
with open(source, "r") as infile:
with open(destination, "w") as outfile:
for line in infile:
if not (line.isspace() or line.lstrip().startswith("#")):
outfile.write(line)
def safe_config_file(config_file):
"""Returns True if config_file can be safely copied
:param str config_file: Path to an Apache configuration file
:returns: True if config_file can be safely copied
:rtype: bool
"""
file_output = subprocess.check_output(
["file", config_file], close_fds=True, stderr=open(os.devnull, "w")
)
if "ASCII" in file_output:
possible_password_file = empty_or_all_comments = True
with open(config_file) as config_fd:
for line in config_fd:
if not (line.isspace() or line.lstrip().startswith("#")):
empty_or_all_comments = False
if line.startswith("-----BEGIN"):
return False
elif not ":" in line:
possible_password_file = False
# If file isn't empty or commented out and could be a password file,
# don't include it in selection. It is safe to include the file if
# it consists solely of comments because comments are removed before
# submission.
return empty_or_all_comments or not possible_password_file
return False
def setup_tempdir(args):
"""Creates a temporary directory and necessary files for config
:param argparse.Namespace args: Parsed command line arguments
:returns: Path to temporary directory
:rtype: str
"""
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, "config_file"), "w") as config_fd:
config_fd.write(args.config_file + "\n")
with open(os.path.join(tempdir, "version"), "w") as version_fd:
version_fd.write(subprocess.check_output(
[args.apache_ctl, "-v"], close_fds=True,
stderr=open(os.devnull, "w")
))
with open(os.path.join(tempdir, "modules"), "w") as modules_fd:
modules_fd.write(subprocess.check_output(
[args.apache_ctl, "-d", args.server_root,
"-f", args.config_file, "-M"], close_fds=True,
stderr=open(os.devnull, "w")
))
with open(os.path.join(tempdir, "vhosts"), "w") as vhosts_fd:
vhosts_fd.write(subprocess.check_output(
[args.apache_ctl, "-d", args.server_root, "-f",
args.config_file, "-t", "-D", "DUMP_VHOSTS"], close_fds=True,
stderr=open(os.devnull, "w")
))
return tempdir
def verify_config(args):
"""Verifies server_root and config_file specify a valid config
:param argparse.Namespace args: Parsed command line arguments
"""
try:
subprocess.check_call(
[args.apache_ctl, "-d", args.server_root,
"-f", args.config_file, "-t"], close_fds=True,
stdout=open(os.devnull, "w"), stderr=open(os.devnull, "w")
)
except OSError:
sys.exit(_NO_APACHECTL)
except subprocess.CalledProcessError:
sys.exit("Syntax check from apachectl failed")
def locate_config(apache_ctl):
"""Uses the apachectl binary to find configuration files
:param str apache_ctl: Path to `apachectl` binary
:returns: Path to Apache server root and main configuration file
:rtype: `tuple` of `str`
"""
try:
output = subprocess.check_output(
[apache_ctl, "-V"], close_fds=True, stderr=open(os.devnull, "w")
)
except OSError:
sys.exit(_NO_APACHECTL)
server_root = config_file = ""
for line in output.splitlines():
# Relevant output lines are of the form: -D DIRECTIVE="VALUE"
if "HTTPD_ROOT" in line:
server_root = line[line.find("\"")+1:-1]
elif "SERVER_CONFIG_FILE" in line:
config_file = line[line.find("\"")+1:-1]
if not (server_root and config_file):
sys.exit(
"Unable to locate Apache configuration. Please run this script "
"again and specify --server-root and --config-file"
)
return server_root, config_file
def get_args():
"""Parses command line arguments
:returns: Parsed command line options
:rtype: argparse.Namespace
"""
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument(
"-c", "--apache-ctl", default="apachectl",
help="path to the `apachectl` binary"
)
parser.add_argument(
"-d", "--server-root",
help="location of the root directory of your Apache configuration"
)
parser.add_argument(
"-f", "--config-file",
help="location of your main Apache configuration file relative to the "
"server root"
)
args = parser.parse_args()
# args.server_root XOR args.config_file
if bool(args.server_root) != bool(args.config_file):
sys.exit(
"If either --server-root and --config-file are specified, they "
"both must be included"
)
elif args.server_root and args.config_file:
args.server_root = os.path.abspath(args.server_root)
args.config_file = os.path.abspath(args.config_file)
if args.config_file.startswith(args.server_root):
args.config_file = args.config_file[len(args.server_root)+1:]
else:
sys.exit(
"This script expects the Apache configuration file to be "
"inside the server root"
)
return args
def main():
"""Main script execution"""
args = get_args()
if not args.server_root:
args.server_root, args.config_file = locate_config(args.apache_ctl)
verify_config(args)
tempdir = setup_tempdir(args)
make_and_verify_selection(args.server_root, tempdir)
tarpath = os.path.join(tempdir, "config.tar.gz")
with tarfile.open(tarpath, mode="w:gz") as tar:
tar.add(tempdir, arcname=".")
# Submit tarpath
if __name__ == "__main__":
main()
@@ -0,0 +1,162 @@
"""Tests for letshelp.letshelp_letsencrypt_apache.py"""
import argparse
import functools
import os
import pkg_resources
import tempfile
import unittest
import mock
import letshelp_letsencrypt.letshelp_letsencrypt_apache as letshelp_le_apache
_PASSWD_FILE = pkg_resources.resource_filename(__name__, "testdata/passwd")
_CONF_FILE = pkg_resources.resource_filename(
__name__, "testdata/conf-available/charset.conf")
_MODULE_NAME = "letshelp_letsencrypt.letshelp_letsencrypt_apache"
_COMPILE_SETTINGS = """Server version: Apache/2.4.10 (Debian)
Server built: Mar 15 2015 09:51:43
Server's Module Magic Number: 20120211:37
Server loaded: APR 1.5.1, APR-UTIL 1.5.4
Compiled using: APR 1.5.1, APR-UTIL 1.5.4
Architecture: 64-bit
Server MPM: event
threaded: yes (fixed thread count)
forked: yes (variable process count)
Server compiled with....
-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=256
-D HTTPD_ROOT="/etc/apache2"
-D SUEXEC_BIN="/usr/lib/apache2/suexec"
-D DEFAULT_PIDLOG="/var/run/apache2.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="mime.types"
-D SERVER_CONFIG_FILE="apache2.conf"
"""
class LetsHelpApacheTest(unittest.TestCase):
@mock.patch(_MODULE_NAME + ".copy_config")
def test_make_and_verify_selection(self, mock_copy_config):
mock_copy_config.return_value = ["apache2.conf"], ["apache2"]
with mock.patch("__builtin__.raw_input") as mock_input:
with mock.patch(_MODULE_NAME + ".sys.stdout"):
mock_input.side_effect = ["Yes", "No"]
letshelp_le_apache.make_and_verify_selection("root", "temp")
self.assertRaises(
SystemExit, letshelp_le_apache.make_and_verify_selection,
"server_root", "temp_dir")
def test_copy_config(self):
tempdir = tempfile.mkdtemp()
server_root = pkg_resources.resource_filename(__name__, "testdata")
letshelp_le_apache.copy_config(server_root, tempdir)
temp_testdata = os.path.join(tempdir, "testdata")
self.assertFalse(os.path.exists(os.path.join(temp_testdata, "passwd")))
self.assertTrue(os.path.exists(os.path.join(
temp_testdata, "conf-available", "charset.conf")))
self.assertTrue(os.path.exists(os.path.join(
temp_testdata, "conf-enabled", "charset.conf")))
def test_copy_file_without_comments(self):
dest = tempfile.mkstemp()[1]
letshelp_le_apache.copy_file_without_comments(_PASSWD_FILE, dest)
with open(_PASSWD_FILE) as original:
with open(dest) as copy:
for original_line, copied_line in zip(original, copy):
self.assertEqual(original_line, copied_line)
@mock.patch(_MODULE_NAME + ".subprocess.check_output")
def test_safe_config_file(self, mock_check_output):
mock_check_output.return_value = "PEM RSA private key"
self.assertFalse(letshelp_le_apache.safe_config_file("filename"))
mock_check_output.return_value = "ASCII text"
self.assertFalse(letshelp_le_apache.safe_config_file(_PASSWD_FILE))
self.assertTrue(letshelp_le_apache.safe_config_file(_CONF_FILE))
@mock.patch(_MODULE_NAME + ".subprocess.check_output")
def test_tempdir(self, mock_check_output):
mock_check_output.side_effect = ["version", "modules", "vhosts"]
args = argparse.Namespace()
args.apache_ctl = "apache_ctl"
args.config_file = "config_file"
args.server_root = "server_root"
tempdir = letshelp_le_apache.setup_tempdir(args)
with open(os.path.join(tempdir, "config_file")) as config_fd:
self.assertEqual(config_fd.read(), args.config_file + "\n")
with open(os.path.join(tempdir, "version")) as version_fd:
self.assertEqual(version_fd.read(), "version")
with open(os.path.join(tempdir, "modules")) as modules_fd:
self.assertEqual(modules_fd.read(), "modules")
with open(os.path.join(tempdir, "vhosts")) as vhosts_fd:
self.assertEqual(vhosts_fd.read(), "vhosts")
@mock.patch(_MODULE_NAME + ".subprocess.check_output")
def test_locate_config(self, mock_check_output):
mock_check_output.side_effect = [OSError, "bad_output",
_COMPILE_SETTINGS,]
self.assertRaises(
SystemExit, letshelp_le_apache.locate_config, "ctl")
self.assertRaises(
SystemExit, letshelp_le_apache.locate_config, "ctl")
server_root, config_file = letshelp_le_apache.locate_config("ctl")
self.assertEqual(server_root, "/etc/apache2")
self.assertEqual(config_file, "apache2.conf")
@mock.patch(_MODULE_NAME + ".argparse")
def test_get_args(self, mock_argparse):
argv = ["-d", "/etc/apache2"]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
self.assertRaises(SystemExit, letshelp_le_apache.get_args)
server_root = "/etc/apache2"
config_file = server_root + "/apache2.conf"
argv = ["-d", server_root, "-f", config_file]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
args = letshelp_le_apache.get_args()
self.assertEqual(args.apache_ctl, "apachectl")
self.assertEqual(args.server_root, server_root)
self.assertEqual(args.config_file, os.path.basename(config_file))
server_root = "/etc/apache2"
config_file = "/etc/httpd/httpd.conf"
argv = ["-d", server_root, "-f", config_file]
mock_argparse.ArgumentParser.return_value = _create_mock_parser(argv)
self.assertRaises(SystemExit, letshelp_le_apache.get_args)
def _create_mock_parser(argv):
parser = argparse.ArgumentParser()
mock_parser = mock.MagicMock()
mock_parser.add_argument = parser.add_argument
mock_parser.parse_args = functools.partial(parser.parse_args, argv)
return mock_parser
if __name__ == "__main__":
unittest.main() # pragma: no cover
@@ -0,0 +1,8 @@
# Read the documentation before enabling AddDefaultCharset.
# In general, it is only a good idea if you know that all your files
# have this encoding. It will override any encoding given in the files
# in meta http-equiv or xml encoding tags.
#AddDefaultCharset UTF-8
# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
@@ -0,0 +1 @@
../conf-available/charset.conf
@@ -0,0 +1 @@
johntheripper:$apr1$fIGE9.JL$jTCwNWZy9Ak/yvOLuOyzQ1
+11
View File
@@ -0,0 +1,11 @@
from setuptools import setup
from setuptools import find_packages
install_requires = ["mock<1.1.0",]
setup(
name="letshelp-letsencrypt",
packages=find_packages(),
install_requires=install_requires,
)
-196
View File
@@ -1,196 +0,0 @@
#!/usr/bin/env python
"""Let's Encrypt Apache configuration submission script"""
import argparse
import os
import subprocess
import sys
import tempfile
DESCRIPTION = """
Let's Help is a simple script you can run to help out the Let's Encrypt
project. Since Let's Encrypt will support automatically configuring HTTPS on
many servers, we want to test this functionality on as many configurations as
possible. This script will create a sanitized copy of your Apache
configuration, notifying you of the files that have been selected. If (and only
if) you approve this selection, these files will be sent to the Let's Encrypt
developers. Of course, your submission will be encrypted.
"""
NO_APACHECTL = """
Unable to find `apachectl` which is required for this script to work. If it is
installed, please run this script again with the --apache-ctl command line
argument and path to the binary.
"""
def copy_config(server_root, temp_dir):
"""Safely copies server_root to temp_dir and returns copied files"""
copied_files, copied_dirs = list(), list()
dir_len = len(os.path.dirname(server_root))
for config_path, config_dirs, config_files in os.walk(server_root):
relative_path = config_path if not dir_len else config_path[dir_len+1:]
temp_path = os.path.join(temp_dir, relative_path)
os.mkdir(temp_path)
copied_all = True
copied_files_in_current_dir = list()
for config_file in config_files:
config_file_path = os.path.join(config_path, config_file)
temp_file_path = os.path.join(temp_path, config_file)
if os.path.islink(config_file_path):
os.symlink(os.readlink(config_file_path), temp_file_path)
elif _safe_config_file(config_file_path):
_copy_file_without_comments(config_file_path, temp_file_path)
copied_files_in_current_dir.append(config_file_path)
else:
copied_all = False
# If copied all files in leaf directory
if copied_all and not config_dirs:
copied_dirs.append(config_path)
else:
copied_files += copied_files_in_current_dir
return copied_files, copied_dirs
def _copy_file_without_comments(source, destination):
"""Copies source to destination, removing comments"""
with open(source, "r") as infile:
with open(destination, "w") as outfile:
for line in infile:
if not (line.isspace() or line.lstrip().startswith("#")):
outfile.write(line)
def _safe_config_file(config_file):
"""Returns True if config_file can be safely copied"""
if "ASCII" in subprocess.check_output(["file", config_file]):
if not config_file.endswith(".pem"):
with open(config_file) as f:
for line in f:
if line.startswith("-----BEGIN"):
return False
return True
return False
def setup_tempdir(args):
"""Creates a temporary directory and necessary files for config"""
tempdir = tempfile.mkdtemp()
with open(os.path.join(tempdir, "config_file"), "w") as f:
f.write(args.config_file + "\n")
with open(os.path.join(tempdir, "version"), "w") as f:
f.write(subprocess.check_output([args.apache_ctl, "-v"]))
with open(os.path.join(tempdir, "modules"), "w") as f:
f.write(subprocess.check_output(
[args.apache_ctl, "-d", args.server_root,
"-f", args.config_file, "-M"]
))
with open(os.path.join(tempdir, "vhosts"), "w") as f:
f.write(subprocess.check_output(
[args.apache_ctl, "-d", args.server_root, "-f",
args.config_file, "-t", "-D", "DUMP_VHOSTS"]
))
return tempdir
def verify_config(args):
"""Verifies server_root and config_file specify a valid config"""
try:
subprocess.check_call(
[args.apache_ctl, "-d", args.server_root,
"-f", args.config_file, "-t"]
)
except OSError:
sys.exit(NO_APACHECTL)
except subprocess.CalledProcessError:
sys.exit("Syntax check from apachectl failed")
def locate_config(apache_ctl):
"""Uses the apachectl binary to find configuration files"""
try:
output = subprocess.check_output([apache_ctl, "-V"])
except OSError:
sys.exit(NO_APACHECTL)
for line in output.splitlines():
if "HTTPD_ROOT" in line:
server_root = line[line.find("\"")+1:-1]
elif "SERVER_CONFIG_FILE" in line:
config_file = line[line.find("\"")+1:-1]
if not (server_root and config_file):
sys.exit(
"Unable to locate Apache configuration. Please run this script "
"again and specify --server-root and --config-file."
)
return server_root, config_file
def get_args():
"""Parses command line arguments"""
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
"-c", "--apache-ctl", default="apachectl",
help="path to the `apachectl` binary"
)
parser.add_argument(
"-d", "--server-root",
help="location of the root directory of your Apache configuration"
)
parser.add_argument(
"-f", "--config-file",
help="location of your main Apache configuration file"
)
args = parser.parse_args()
# args.server_root XOR args.config_file
if bool(args.server_root) != bool(args.config_file):
sys.exit(
"If either --server-root and --config-file are specified, they "
"both must be included."
)
elif args.server_root and args.config_file:
if args.config_file.startswith(args.server_root):
args.config_file = args.config_file[len(args.server_root):]
else:
sys.exit(
"This script expects the Apache configuration file to be "
"inside the server root."
)
return args
def main():
"""Main script execution"""
args = get_args()
if not args.server_root:
args.server_root, args.config_file = locate_config(args.apache_ctl)
verify_config(args)
tempdir = setup_tempdir(args)
files, dirs = copy_config(args.server_root, tempdir)
print "Copied these files:"
for f in files:
print f
print "Copied all files in these directories:"
for d in dirs:
print d
if __name__ == "__main__":
main()
+2 -1
View File
@@ -23,4 +23,5 @@ rm -f .coverage # --cover-erase is off, make sure stats are correct
cover letsencrypt 97 && \ cover letsencrypt 97 && \
cover acme 100 && \ cover acme 100 && \
cover letsencrypt_apache 78 && \ cover letsencrypt_apache 78 && \
cover letsencrypt_nginx 96 cover letsencrypt_nginx 96 && \
cover letshelp_letsencrypt 100
+5 -3
View File
@@ -10,12 +10,13 @@ envlist = py26,py27,py33,py34,cover,lint
[testenv] [testenv]
commands = commands =
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt
# -q does not suppress errors # -q does not suppress errors
python setup.py test -q python setup.py test -q
python setup.py test -q -s acme python setup.py test -q -s acme
python setup.py test -q -s letsencrypt_apache python setup.py test -q -s letsencrypt_apache
python setup.py test -q -s letsencrypt_nginx python setup.py test -q -s letsencrypt_nginx
python setup.py test -q -s letshelp_letsencrypt
setenv = setenv =
PYTHONPATH = {toxinidir} PYTHONPATH = {toxinidir}
@@ -35,7 +36,7 @@ commands =
[testenv:cover] [testenv:cover]
basepython = python2.7 basepython = python2.7
commands = commands =
pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx pip install -r requirements.txt -e acme -e .[testing] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt
./tox.cover.sh ./tox.cover.sh
[testenv:lint] [testenv:lint]
@@ -45,8 +46,9 @@ basepython = python2.7
# duplicate code checking; if one of the commands fails, others will # duplicate code checking; if one of the commands fails, others will
# continue, but tox return code will reflect previous error # continue, but tox return code will reflect previous error
commands = commands =
pip install -r requirements.txt -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx pip install -r requirements.txt -e acme -e .[dev] -e letsencrypt-apache -e letsencrypt-nginx -e letshelp-letsencrypt
pylint --rcfile=.pylintrc letsencrypt pylint --rcfile=.pylintrc letsencrypt
pylint --rcfile=.pylintrc acme/acme pylint --rcfile=.pylintrc acme/acme
pylint --rcfile=.pylintrc letsencrypt-apache/letsencrypt_apache pylint --rcfile=.pylintrc letsencrypt-apache/letsencrypt_apache
pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx pylint --rcfile=.pylintrc letsencrypt-nginx/letsencrypt_nginx
pylint --rcfile=.pylintrc letshelp-letsencrypt/letshelp_letsencrypt