diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..087900105 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[report] +# show lines missing coverage in output +show_missing = True diff --git a/.gitattributes b/.gitattributes index 5eee84cce..0f8bbbfb5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1,16 @@ -* text=auto eol=lf +#Default, normalize CRLF into LF in non-binary files +# Files identified as binary by Git are not changed +* crlf=auto # special files +*.sh crlf=input +*.py crlf=input + *.bat text eol=crlf + +*.der binary +*.gz binary *.jpeg binary *.jpg binary *.png binary +*.gz binary diff --git a/.travis.yml b/.travis.yml index 16b5700e5..6f964dbec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,26 +4,18 @@ cache: directories: - $HOME/.cache/pip -services: - - rabbitmq - - mariadb - # apacheconftest - #- apache2 +# This makes sure we get a host with docker-compose present. +dist: trusty -# http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS -# gimme has to be kept in sync with Boulder's Go version setting in .travis.yml before_install: - 'dpkg -s libaugeas0' - - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5.1)"' # using separate envs with different TOXENVs creates 4x1 Travis build # matrix, which allows us to clearly distinguish which component under # test has failed env: global: - - GOPATH=/tmp/go - - PATH=$GOPATH/bin:$PATH - - GO15VENDOREXPERIMENT=1 # Fixes problems with vendor directories + - BOULDERPATH=$PWD/boulder/ matrix: include: @@ -93,7 +85,6 @@ addons: - boulder - boulder-mysql - boulder-rabbitmq - mariadb: "10.0" apt: sources: - augeas @@ -109,13 +100,11 @@ addons: # For certbot-nginx integration testing - nginx-light - openssl - # For Boulder integration testing - - rsyslog # for apacheconftest - #- apache2 - #- libapache2-mod-wsgi - #- libapache2-mod-macro - #- sudo + - apache2 + - libapache2-mod-wsgi + - libapache2-mod-macro + - sudo install: "travis_retry pip install tox coveralls" script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/travis-integration.sh)' diff --git a/CHANGES.rst b/CHANGES.rst index 55e4bd796..4ce41a8bc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,9 +3,9 @@ ChangeLog Please note: the change log will only get updated after first release - for now please use the -`commit log `_. +`commit log `_. To see the changes in a given release, inspect the github milestone for the release. For instance: -https://github.com/letsencrypt/letsencrypt/issues?utf8=%E2%9C%93&q=milestone%3A0.3.0 +https://github.com/certbot/certbot/issues?utf8=%E2%9C%93&q=milestone%3A0.3.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf19b18e1..5f1625658 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,4 +15,4 @@ to the Sphinx generated docs is provided below. --> -https://letsencrypt.readthedocs.org/en/latest/contributing.html +https://certbot.eff.org/docs/contributing.html diff --git a/Dockerfile b/Dockerfile index 3e4c9430e..d42b632d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ WORKDIR /opt/certbot # If doesn't exist, it is created along with all missing # directories in its path. +ENV DEBIAN_FRONTEND=noninteractive COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ diff --git a/README.rst b/README.rst index 91a3cfcb5..c71079f9a 100644 --- a/README.rst +++ b/README.rst @@ -11,43 +11,63 @@ For more information regarding the status of the project, please see https://letsencrypt.org. Be sure to checkout the `Frequently Asked Questions (FAQ) `_. -About the Let's Encrypt Client +About Certbot ============================== -The Let's Encrypt Client is a fully-featured, extensible client for the Let's +Certbot is a fully-featured, extensible client for the Let's Encrypt CA (or any other CA that speaks the `ACME `_ protocol) that can automate the tasks of obtaining certificates and configuring webservers to use them. This client runs on Unix-based operating systems. +Until May 2016, Certbot was named simply ``letsencrypt`` or ``letsencrypt-auto``, +depending on install method. Instructions on the Internet, and some pieces of the +software, may still refer to this older name. + +Contributing +------------ + +If you'd like to contribute to this project please read `Developer Guide +`_. + +.. _installation: + Installation ------------ -If ``letsencrypt`` is packaged for your Unix OS, you can install it from -there, and run it by typing ``letsencrypt``. Because not all operating -systems have packages yet, we provide a temporary solution via the -``letsencrypt-auto`` wrapper script, which obtains some dependencies -from your OS and puts others in a python virtual environment:: +If ``certbot`` (or ``letsencrypt``) is packaged for your Unix OS (visit +certbot.eff.org_ to find out), you can install it +from there, and run it by typing ``certbot`` (or ``letsencrypt``). Because +not all operating systems have packages yet, we provide a temporary solution +via the ``certbot-auto`` wrapper script, which obtains some dependencies from +your OS and puts others in a python virtual environment:: - user@webserver:~$ git clone https://github.com/letsencrypt/letsencrypt - user@webserver:~$ cd letsencrypt - user@webserver:~/letsencrypt$ ./letsencrypt-auto --help + user@webserver:~$ wget https://dl.eff.org/certbot-auto + user@webserver:~$ chmod a+x ./certbot-auto + user@webserver:~$ ./certbot-auto --help -Or for full command line help, type:: +.. hint:: The certbot-auto download is protected by HTTPS, which is pretty good, but if you'd like to + double check the integrity of the ``certbot-auto`` script, you can use these steps for verification before running it:: - ./letsencrypt-auto --help all + user@server:~$ wget -N https://dl.eff.org/certbot-auto.asc + user@server:~$ gpg2 --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 + user@server:~$ gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc certbot-auto -``letsencrypt-auto`` updates to the latest client release automatically. And -since ``letsencrypt-auto`` is a wrapper to ``letsencrypt``, it accepts exactly +And for full command line help, you can type:: + + ./certbot-auto --help all + +``certbot-auto`` updates to the latest client release automatically. And +since ``certbot-auto`` is a wrapper to ``certbot``, it accepts exactly the same command line flags and arguments. More details about this script and other installation methods can be found `in the User Guide -`_. +`_. How to run the client --------------------- -In many cases, you can just run ``letsencrypt-auto`` or ``letsencrypt``, and the +In many cases, you can just run ``certbot-auto`` or ``certbot``, and the client will guide you through the process of obtaining and installing certs interactively. @@ -56,7 +76,7 @@ For instance, if you want to obtain a cert for ``example.com``, ``www.example.com``, and ``other.example.net``, using the Apache plugin to both obtain and install the certs, you could do this:: - ./letsencrypt-auto --apache -d example.com -d www.example.com -d other.example.net + ./certbot-auto --apache -d example.com -d www.example.com -d other.example.net (The first time you run the command, it will make an account, and ask for an email and agreement to the Let's Encrypt Subscriber Agreement; you can @@ -65,7 +85,7 @@ automate those with ``--email`` and ``--agree-tos``) If you want to use a webserver that doesn't have full plugin support yet, you can still use "standalone" or "webroot" plugins to obtain a certificate:: - ./letsencrypt-auto certonly --standalone --email admin@example.com -d example.com -d www.example.com -d other.example.net + ./certbot-auto certonly --standalone --email admin@example.com -d example.com -d www.example.com -d other.example.net Understanding the client in more depth @@ -73,21 +93,21 @@ Understanding the client in more depth To understand what the client is doing in detail, it's important to understand the way it uses plugins. Please see the `explanation of -plugins `_ in +plugins `_ in the User Guide. Links ===== -Documentation: https://letsencrypt.readthedocs.org +Documentation: https://certbot.eff.org/docs -Software project: https://github.com/letsencrypt/letsencrypt +Software project: https://github.com/certbot/certbot -Notes for developers: https://letsencrypt.readthedocs.org/en/latest/contributing.html +Notes for developers: https://certbot.eff.org/docs/contributing.html Main Website: https://letsencrypt.org/ -IRC Channel: #letsencrypt on `Freenode`_ +IRC Channel: #letsencrypt on `Freenode`_ or #certbot on `OFTC`_ Community: https://community.letsencrypt.org @@ -103,12 +123,12 @@ email to client-dev+subscribe@letsencrypt.org) -.. |build-status| image:: https://travis-ci.org/letsencrypt/letsencrypt.svg?branch=master - :target: https://travis-ci.org/letsencrypt/letsencrypt +.. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master + :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://coveralls.io/repos/letsencrypt/letsencrypt/badge.svg?branch=master - :target: https://coveralls.io/r/letsencrypt/letsencrypt +.. |coverage| image:: https://coveralls.io/repos/certbot/certbot/badge.svg?branch=master + :target: https://coveralls.io/r/certbot/certbot :alt: Coverage status .. |docs| image:: https://readthedocs.org/projects/letsencrypt/badge/ @@ -151,10 +171,10 @@ Current Features - standalone (runs its own simple webserver to prove you control a domain) - webroot (adds files to webroot directories in order to prove control of domains and obtain certs) - - nginx/0.8.48+ (highly experimental, not included in letsencrypt-auto) + - nginx/0.8.48+ (highly experimental, not included in certbot-auto) * The private key is generated locally on your system. -* Can talk to the Let's Encrypt CA or optionally to other ACME +* Can talk to the Let's Encrypt CA or optionally to other ACME compliant services. * Can get domain-validated (DV) certificates. * Can revoke certificates. @@ -169,4 +189,6 @@ Current Features .. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt +.. _OFTC: https://webchat.oftc.net?channels=%23certbot .. _client-dev: https://groups.google.com/a/letsencrypt.org/forum/#!forum/client-dev +.. _certbot.eff.org: https://certbot.eff.org/ diff --git a/acme/.pylintrc b/acme/.pylintrc index d0d150631..06bb2a01f 100644 --- a/acme/.pylintrc +++ b/acme/.pylintrc @@ -21,12 +21,6 @@ persistent=yes # usually to register additional checkers. load-plugins=linter_plugin -# DEPRECATED -include-ids=no - -# DEPRECATED -symbols=no - # Use multiple processes to speed up Pylint. jobs=1 diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 280bc8308..c436cc631 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -500,7 +500,7 @@ class DNS(_TokenChallenge): """ return DNSResponse(validation=self.gen_validation( - self, account_key, **kwargs)) + account_key, **kwargs)) def validation_domain_name(self, name): """Domain name for TXT validation record. diff --git a/acme/acme/client.py b/acme/acme/client.py index 225245686..117ee6b7d 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -512,6 +512,10 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes self.verify_ssl = verify_ssl self._nonces = set() self.user_agent = user_agent + self.session = requests.Session() + + def __del__(self): + self.session.close() def _wrap_in_jws(self, obj, nonce): """Wrap `JSONDeSerializable` object in JWS. @@ -606,7 +610,7 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes kwargs['verify'] = self.verify_ssl kwargs.setdefault('headers', {}) kwargs['headers'].setdefault('User-Agent', self.user_agent) - response = requests.request(method, url, *args, **kwargs) + response = self.session.request(method, url, *args, **kwargs) logging.debug('Received %s. Headers: %s. Content: %r', response, response.headers, response.content) return response diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index c8e95a423..a526a0984 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -484,9 +484,11 @@ class ClientNetworkTest(unittest.TestCase): def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} - # pylint: disable=protected-access - self.assertRaises( - errors.ClientError, self.net._check_response, self.response) + with mock.patch('acme.client.messages.Error.from_json') as from_json: + from_json.side_effect = jose.DeserializationError + # pylint: disable=protected-access + self.assertRaises( + errors.ClientError, self.net._check_response, self.response) def test_check_response_not_ok_jobj_error(self): self.response.ok = False @@ -528,40 +530,49 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual( self.response, self.net._check_response(self.response)) - @mock.patch('acme.client.requests') - def test_send_request(self, mock_requests): - mock_requests.request.return_value = self.response + def test_send_request(self): + self.net.session = mock.MagicMock() + self.net.session.request.return_value = self.response # pylint: disable=protected-access self.assertEqual(self.response, self.net._send_request( - 'HEAD', 'url', 'foo', bar='baz')) - mock_requests.request.assert_called_once_with( - 'HEAD', 'url', 'foo', verify=mock.ANY, bar='baz', headers=mock.ANY) + 'HEAD', 'http://example.com/', 'foo', bar='baz')) + self.net.session.request.assert_called_once_with( + 'HEAD', 'http://example.com/', 'foo', + headers=mock.ANY, verify=mock.ANY, bar='baz') - @mock.patch('acme.client.requests') - def test_send_request_verify_ssl(self, mock_requests): + def test_send_request_verify_ssl(self): # pylint: disable=protected-access for verify in True, False: - mock_requests.request.reset_mock() - mock_requests.request.return_value = self.response + self.net.session = mock.MagicMock() + self.net.session.request.return_value = self.response self.net.verify_ssl = verify # pylint: disable=protected-access self.assertEqual( - self.response, self.net._send_request('GET', 'url')) - mock_requests.request.assert_called_once_with( - 'GET', 'url', verify=verify, headers=mock.ANY) + self.response, + self.net._send_request('GET', 'http://example.com/')) + self.net.session.request.assert_called_once_with( + 'GET', 'http://example.com/', verify=verify, headers=mock.ANY) - @mock.patch('acme.client.requests') - def test_send_request_user_agent(self, mock_requests): - mock_requests.request.return_value = self.response + def test_send_request_user_agent(self): + self.net.session = mock.MagicMock() # pylint: disable=protected-access - self.net._send_request('GET', 'url', headers={'bar': 'baz'}) - mock_requests.request.assert_called_once_with( - 'GET', 'url', verify=mock.ANY, + self.net._send_request('GET', 'http://example.com/', + headers={'bar': 'baz'}) + self.net.session.request.assert_called_once_with( + 'GET', 'http://example.com/', verify=mock.ANY, headers={'User-Agent': 'acme-python-test', 'bar': 'baz'}) - self.net._send_request('GET', 'url', headers={'User-Agent': 'foo2'}) - mock_requests.request.assert_called_with( - 'GET', 'url', verify=mock.ANY, headers={'User-Agent': 'foo2'}) + self.net._send_request('GET', 'http://example.com/', + headers={'User-Agent': 'foo2'}) + self.net.session.request.assert_called_with( + 'GET', 'http://example.com/', + verify=mock.ANY, headers={'User-Agent': 'foo2'}) + + def test_del(self): + sess = mock.MagicMock() + self.net.session = sess + del self.net + sess.close.assert_called_once_with() @mock.patch('acme.client.requests') def test_requests_error_passthrough(self, mock_requests): @@ -614,14 +625,16 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase): return self.checked_response def test_head(self): - self.assertEqual(self.response, self.net.head('url', 'foo', bar='baz')) + self.assertEqual(self.response, self.net.head( + 'http://example.com/', 'foo', bar='baz')) self.send_request.assert_called_once_with( - 'HEAD', 'url', 'foo', bar='baz') + 'HEAD', 'http://example.com/', 'foo', bar='baz') def test_get(self): self.assertEqual(self.checked_response, self.net.get( - 'url', content_type=self.content_type, bar='baz')) - self.send_request.assert_called_once_with('GET', 'url', bar='baz') + 'http://example.com/', content_type=self.content_type, bar='baz')) + self.send_request.assert_called_once_with( + 'GET', 'http://example.com/', bar='baz') def test_post(self): # pylint: disable=protected-access diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index 73f7f8f62..2b2133475 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -1,4 +1,5 @@ """Crypto utilities.""" +import binascii import contextlib import logging import re @@ -203,7 +204,7 @@ def gen_ss_cert(key, domains, not_before=None, """ assert domains, "Must provide one or more hostnames for the cert." cert = OpenSSL.crypto.X509() - cert.set_serial_number(1337) + cert.set_serial_number(int(binascii.hexlify(OpenSSL.rand.bytes(16)), 16)) cert.set_version(2) extensions = [ diff --git a/acme/acme/crypto_util_test.py b/acme/acme/crypto_util_test.py index 147cd5a2a..75a908d4f 100644 --- a/acme/acme/crypto_util_test.py +++ b/acme/acme/crypto_util_test.py @@ -8,6 +8,8 @@ import unittest import six from six.moves import socketserver # pylint: disable=import-error +import OpenSSL + from acme import errors from acme import jose from acme import test_util @@ -126,5 +128,23 @@ class PyOpenSSLCertOrReqSANTest(unittest.TestCase): self._get_idn_names()) +class RandomSnTest(unittest.TestCase): + """Test for random certificate serial numbers.""" + + def setUp(self): + self.cert_count = 5 + self.serial_num = [] + self.key = OpenSSL.crypto.PKey() + self.key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + + def test_sn_collisions(self): + from acme.crypto_util import gen_ss_cert + + for _ in range(self.cert_count): + cert = gen_ss_cert(self.key, ['dummy'], force_san=True) + self.serial_num.append(cert.get_serial_number()) + self.assertTrue(len(set(self.serial_num)) > 1) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 77d47c522..70894a808 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -49,7 +49,7 @@ class MissingNonce(NonceError): def __str__(self): return ('Server {0} response did not include a replay ' - 'nonce, headers: {1}'.format( + 'nonce, headers: {1} (This may be a service outage)'.format( self.response.request.method, self.response.headers)) diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 24a3b580c..56bcb1de2 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -37,9 +37,9 @@ class Error(jose.JSONObjectWithFields, errors.Error): ) ) - typ = jose.Field('type') + typ = jose.Field('type', omitempty=True, default='about:blank') title = jose.Field('title', omitempty=True) - detail = jose.Field('detail') + detail = jose.Field('detail', omitempty=True) @property def description(self): @@ -143,12 +143,6 @@ class Directory(jose.JSONDeSerializable): def __init__(self, jobj): canon_jobj = util.map_keys(jobj, self._canon_key) - if not set(canon_jobj).issubset( - set(self._REGISTERED_TYPES).union(['meta'])): - # TODO: acme-spec is not clear about this: 'It is a JSON - # dictionary, whose keys are the "resource" values listed - # in {{https-requests}}' - raise ValueError('Wrong directory fields') # TODO: check that everything is an absolute URL; acme-spec is # not clear on that self._jobj = canon_jobj @@ -171,10 +165,7 @@ class Directory(jose.JSONDeSerializable): @classmethod def from_json(cls, jobj): jobj['meta'] = cls.Meta.from_json(jobj.pop('meta', {})) - try: - return cls(jobj) - except ValueError as error: - raise jose.DeserializationError(str(error)) + return cls(jobj) class Resource(jose.JSONObjectWithFields): diff --git a/acme/acme/messages_test.py b/acme/acme/messages_test.py index b2b7febdc..36d0dd618 100644 --- a/acme/acme/messages_test.py +++ b/acme/acme/messages_test.py @@ -28,6 +28,14 @@ class ErrorTest(unittest.TestCase): self.error_custom = Error(typ='custom', detail='bar') self.jobj_cusom = {'type': 'custom', 'detail': 'bar'} + def test_default_typ(self): + from acme.messages import Error + self.assertEqual(Error().typ, 'about:blank') + + def test_from_json_empty(self): + from acme.messages import Error + self.assertEqual(Error(), Error.from_json('{}')) + def test_from_json_hashable(self): from acme.messages import Error hash(Error.from_json(self.error.to_json())) @@ -97,9 +105,9 @@ class DirectoryTest(unittest.TestCase): ), }) - def test_init_wrong_key_value_error(self): + def test_init_wrong_key_value_success(self): # pylint: disable=no-self-use from acme.messages import Directory - self.assertRaises(ValueError, Directory, {'foo': 'bar'}) + Directory({'foo': 'bar'}) def test_getitem(self): self.assertEqual('reg', self.dir['new-reg']) @@ -127,10 +135,9 @@ class DirectoryTest(unittest.TestCase): }, }) - def test_from_json_deserialization_error_on_wrong_key(self): + def test_from_json_deserialization_unknown_key_success(self): # pylint: disable=no-self-use from acme.messages import Directory - self.assertRaises( - jose.DeserializationError, Directory.from_json, {'foo': 'bar'}) + Directory.from_json({'foo': 'bar'}) class RegistrationTest(unittest.TestCase): diff --git a/acme/setup.py b/acme/setup.py index cbd3bfb87..ed133e128 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.6.0.dev0' +version = '0.9.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/certbot_apache/augeas_configurator.py b/certbot-apache/certbot_apache/augeas_configurator.py index 12753541c..6999120d6 100644 --- a/certbot-apache/certbot_apache/augeas_configurator.py +++ b/certbot-apache/certbot_apache/augeas_configurator.py @@ -1,7 +1,6 @@ """Class of Augeas Configurators.""" import logging -import augeas from certbot import errors from certbot import reverter @@ -29,12 +28,9 @@ class AugeasConfigurator(common.Plugin): def __init__(self, *args, **kwargs): super(AugeasConfigurator, self).__init__(*args, **kwargs) - self.aug = augeas.Augeas( - # specify a directory to load our preferred lens from - loadpath=constants.AUGEAS_LENS_DIR, - # Do not save backup (we do it ourselves), do not load - # anything by default - flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) + # Placeholder for augeas + self.aug = None + self.save_notes = "" # See if any temporary changes need to be recovered @@ -42,6 +38,16 @@ class AugeasConfigurator(common.Plugin): # because this will change the underlying configuration and potential # vhosts self.reverter = reverter.Reverter(self.config) + + def init_augeas(self): + """ Initialize the actual Augeas instance """ + import augeas + self.aug = augeas.Augeas( + # specify a directory to load our preferred lens from + loadpath=constants.AUGEAS_LENS_DIR, + # Do not save backup (we do it ourselves), do not load + # anything by default + flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) self.recovery_routine() def check_parsing_errors(self, lens): diff --git a/certbot-apache/certbot_apache/augeas_lens/httpd.aug b/certbot-apache/certbot_apache/augeas_lens/httpd.aug index 697d5de89..2729f4b60 100644 --- a/certbot-apache/certbot_apache/augeas_lens/httpd.aug +++ b/certbot-apache/certbot_apache/augeas_lens/httpd.aug @@ -45,21 +45,24 @@ autoload xfm let dels (s:string) = del s s (* deal with continuation lines *) -let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)/ " " -let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)/ "" +let sep_spc = del /([ \t]+|[ \t]*\\\\\r?\n[ \t]*)+/ " " +let sep_osp = del /([ \t]*|[ \t]*\\\\\r?\n[ \t]*)*/ "" let sep_eq = del /[ \t]*=[ \t]*/ "=" let nmtoken = /[a-zA-Z:_][a-zA-Z0-9:_.-]*/ let word = /[a-z][a-z0-9._-]*/i -let comment = Util.comment let eol = Util.doseol let empty = Util.empty_dos let indent = Util.indent +let comment_val_re = /([^ \t\r\n](.|\\\\\r?\n)*[^ \\\t\r\n]|[^ \t\r\n])/ +let comment = [ label "#comment" . del /[ \t]*#[ \t]*/ "# " + . store comment_val_re . eol ] + (* borrowed from shellvars.aug *) -let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'/ -let char_arg_sec = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'/ +let char_arg_dir = /([^\\ '"{\t\r\n]|[^ '"{\t\r\n]+[^\\ \t\r\n])|\\\\"|\\\\'|\\\\ / +let char_arg_sec = /([^\\ '"\t\r\n>]|[^ '"\t\r\n>]+[^\\ \t\r\n>])|\\\\"|\\\\'|\\\\ / let char_arg_wl = /([^\\ '"},\t\r\n]|[^ '"},\t\r\n]+[^\\ '"},\t\r\n])/ let cdot = /\\\\./ diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 26c3185be..89d602f5f 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -15,7 +15,7 @@ from acme import challenges from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util from certbot.plugins import common @@ -106,8 +106,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): add("handle-sites", default=constants.os_constant("handle_sites"), help="Let installer handle enabling sites for you." + "(Only Ubuntu/Debian currently)") - le_util.add_deprecated_argument(add, argument_name="ctl", nargs=1) - le_util.add_deprecated_argument( + util.add_deprecated_argument(add, argument_name="ctl", nargs=1) + util.add_deprecated_argument( add, argument_name="init-script", nargs=1) def __init__(self, *args, **kwargs): @@ -124,13 +124,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.assoc = dict() # Outstanding challenges self._chall_out = set() + # Maps enhancements to vhosts we've enabled the enhancement for + self._enhanced_vhosts = defaultdict(set) # These will be set in the prepare function self.parser = None self.version = version self.vhosts = None self._enhance_func = {"redirect": self._enable_redirect, - "ensure-http-header": self._set_http_header} + "ensure-http-header": self._set_http_header, + "staple-ocsp": self._enable_ocsp_stapling} @property def mod_ssl_conf(self): @@ -147,8 +150,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :raises .errors.PluginError: If there is any other error """ + # Perform the actual Augeas initialization to be able to react + try: + self.init_augeas() + except ImportError: + raise errors.NoInstallationError("Problem in Augeas installation") + # Verify Apache is installed - if not le_util.exe_exists(constants.os_constant("restart_cmd")[0]): + if not util.exe_exists(constants.os_constant("restart_cmd")[0]): raise errors.NoInstallationError # Make sure configuration is valid @@ -318,9 +327,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): vhost = display_ops.select_vhost(target_name, self.vhosts) if vhost is None: logger.error( - "No vhost exists with servername or alias of: %s. " - "No vhost was selected. Please specify servernames " - "in the Apache config", target_name) + "No vhost exists with servername or alias of: %s " + "(or it's in a file with multiple vhosts, which Certbot " + "can't parse yet). " + "No vhost was selected. Please specify ServerName or ServerAlias " + "in the Apache config, or split vhosts into separate files.", + target_name) raise errors.PluginError("No vhost selected") elif temp: return vhost @@ -593,8 +605,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :type addr: :class:`~certbot_apache.obj.Addr` """ - loc = parser.get_aug_path(self.parser.loc["name"]) + loc = parser.get_aug_path(self.parser.loc["name"]) if addr.get_port() == "443": path = self.parser.add_dir_to_ifmodssl( loc, "NameVirtualHost", [str(addr)]) @@ -616,50 +628,93 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ + # If nonstandard port, add service definition for matching + if port != "443": + port_service = "%s %s" % (port, "https") + else: + port_service = port + self.prepare_https_modules(temp) # Check for Listen # Note: This could be made to also look for ip:443 combo listens = [self.parser.get_arg(x).split()[0] for x in self.parser.find_dir("Listen")] + # In case no Listens are set (which really is a broken apache config) if not listens: listens = ["80"] - if port in listens: + + # Listen already in place + if self._has_port_already(listens, port): return + + listen_dirs = set(listens) + for listen in listens: # For any listen statement, check if the machine also listens on # Port 443. If not, add such a listen statement. if len(listen.split(":")) == 1: # Its listening to all interfaces - if port not in listens: - if port == "443": - args = [port] - else: - # Non-standard ports should specify https protocol - args = [port, "https"] - self.parser.add_dir_to_ifmodssl( - parser.get_aug_path( - self.parser.loc["listen"]), "Listen", args) - self.save_notes += "Added Listen %s directive to %s\n" % ( - port, self.parser.loc["listen"]) - listens.append(port) + if port not in listen_dirs and port_service not in listen_dirs: + listen_dirs.add(port_service) else: # The Listen statement specifies an ip _, ip = listen[::-1].split(":", 1) ip = ip[::-1] - if "%s:%s" % (ip, port) not in listens: - if port == "443": - args = ["%s:%s" % (ip, port)] - else: - # Non-standard ports should specify https protocol - args = ["%s:%s" % (ip, port), "https"] - self.parser.add_dir_to_ifmodssl( - parser.get_aug_path( - self.parser.loc["listen"]), "Listen", args) - self.save_notes += ("Added Listen %s:%s directive to " - "%s\n") % (ip, port, - self.parser.loc["listen"]) - listens.append("%s:%s" % (ip, port)) + if "%s:%s" % (ip, port_service) not in listen_dirs and ( + "%s:%s" % (ip, port_service) not in listen_dirs): + listen_dirs.add("%s:%s" % (ip, port_service)) + self._add_listens(listen_dirs, listens, port) + + def _add_listens(self, listens, listens_orig, port): + """Helper method for prepare_server_https to figure out which new + listen statements need adding + + :param set listens: Set of all needed Listen statements + :param list listens_orig: List of existing listen statements + :param string port: Port number we're adding + """ + + # Add service definition for non-standard ports + if port != "443": + port_service = "%s %s" % (port, "https") + else: + port_service = port + + new_listens = listens.difference(listens_orig) + + if port in new_listens or port_service in new_listens: + # We have wildcard, skip the rest + self.parser.add_dir_to_ifmodssl( + parser.get_aug_path(self.parser.loc["listen"]), + "Listen", port_service.split(" ")) + self.save_notes += "Added Listen %s directive to %s\n" % ( + port_service, self.parser.loc["listen"]) + else: + for listen in new_listens: + self.parser.add_dir_to_ifmodssl( + parser.get_aug_path(self.parser.loc["listen"]), + "Listen", listen.split(" ")) + self.save_notes += ("Added Listen %s directive to " + "%s\n") % (listen, + self.parser.loc["listen"]) + + def _has_port_already(self, listens, port): + """Helper method for prepare_server_https to find out if user + already has an active Listen statement for the port we need + + :param list listens: List of listen variables + :param string port: Port in question + """ + + if port in listens: + return True + # Check if Apache is already listening on a specific IP + for listen in listens: + if len(listen.split(":")) > 1: + # Ugly but takes care of protocol def, eg: 1.1.1.1:443 https + if listen.split(":")[-1].split(" ")[0] == port: + return True def prepare_https_modules(self, temp): """Helper method for prepare_server_https, taking care of enabling @@ -944,7 +999,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ###################################################################### def supported_enhancements(self): # pylint: disable=no-self-use """Returns currently supported enhancements.""" - return ["redirect", "ensure-http-header"] + return ["redirect", "ensure-http-header", "staple-ocsp"] def enhance(self, domain, enhancement, options=None): """Enhance configuration. @@ -971,6 +1026,68 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): logger.warn("Failed %s for %s", enhancement, domain) raise + def _enable_ocsp_stapling(self, ssl_vhost, unused_options): + """Enables OCSP Stapling + + In OCSP, each client (e.g. browser) would have to query the + OCSP Responder to validate that the site certificate was not revoked. + + Enabling OCSP Stapling, would allow the web-server to query the OCSP + Responder, and staple its response to the offered certificate during + TLS. i.e. clients would not have to query the OCSP responder. + + OCSP Stapling enablement on Apache implicitly depends on + SSLCertificateChainFile being set by other code. + + .. note:: This function saves the configuration + + :param ssl_vhost: Destination of traffic, an ssl enabled vhost + :type ssl_vhost: :class:`~letsencrypt_apache.obj.VirtualHost` + + :param unused_options: Not currently used + :type unused_options: Not Available + + :returns: Success, general_vhost (HTTP vhost) + :rtype: (bool, :class:`~letsencrypt_apache.obj.VirtualHost`) + + """ + min_apache_ver = (2, 3, 3) + if self.get_version() < min_apache_ver: + raise errors.PluginError( + "Unable to set OCSP directives.\n" + "Apache version is below 2.3.3.") + + if "socache_shmcb_module" not in self.parser.modules: + self.enable_mod("socache_shmcb") + + # Check if there's an existing SSLUseStapling directive on. + use_stapling_aug_path = self.parser.find_dir("SSLUseStapling", + "on", start=ssl_vhost.path) + if not use_stapling_aug_path: + self.parser.add_dir(ssl_vhost.path, "SSLUseStapling", "on") + + ssl_vhost_aug_path = parser.get_aug_path(ssl_vhost.filep) + + # Check if there's an existing SSLStaplingCache directive. + stapling_cache_aug_path = self.parser.find_dir('SSLStaplingCache', + None, ssl_vhost_aug_path) + + # We'll simply delete the directive, so that we'll have a + # consistent OCSP cache path. + if stapling_cache_aug_path: + self.aug.remove( + re.sub(r"/\w*$", "", stapling_cache_aug_path[0])) + + self.parser.add_dir_to_ifmodssl(ssl_vhost_aug_path, + "SSLStaplingCache", + ["shmcb:/var/run/apache2/stapling_cache(128000)"]) + + msg = "OCSP Stapling was enabled on SSL Vhost: %s.\n"%( + ssl_vhost.filep) + self.save_notes += msg + self.save() + logger.info(msg) + def _set_http_header(self, ssl_vhost, header_substring): """Enables header that is identified by header_substring on ssl_vhost. @@ -1058,9 +1175,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param unused_options: Not currently used :type unused_options: Not Available - :returns: Success, general_vhost (HTTP vhost) - :rtype: (bool, :class:`~certbot_apache.obj.VirtualHost`) - :raises .errors.PluginError: If no viable HTTP host can be created or used for the redirect. @@ -1083,6 +1197,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "redirection") self._create_redirect_vhost(ssl_vhost) else: + if general_vh in self._enhanced_vhosts["redirect"]: + logger.debug("Already enabled redirect for this vhost") + return + # Check if Certbot redirection already exists self._verify_no_certbot_redirect(general_vh) @@ -1118,6 +1236,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): (general_vh.filep, ssl_vhost.filep)) self.save() + self._enhanced_vhosts["redirect"].add(general_vh) logger.info("Redirecting vhost in %s to ssl vhost in %s", general_vh.filep, ssl_vhost.filep) @@ -1177,10 +1296,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :type vhost: :class:`~certbot_apache.obj.VirtualHost` """ - rewrite_engine_path = self.parser.find_dir("RewriteEngine", "on", + rewrite_engine_path_list = self.parser.find_dir("RewriteEngine", "on", start=vhost.path) - if rewrite_engine_path: - return self.parser.get_arg(rewrite_engine_path[0]) + if rewrite_engine_path_list: + for re_path in rewrite_engine_path_list: + # A RewriteEngine directive may also be included in per + # directory .htaccess files. We only care about the VirtualHost. + if 'VirtualHost' in re_path: + return self.parser.get_arg(re_path) return False def _create_redirect_vhost(self, ssl_vhost): @@ -1202,6 +1325,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Make a new vhost data structure and add it to the lists new_vhost = self._create_vhost(parser.get_aug_path(redirect_filepath)) self.vhosts.append(new_vhost) + self._enhanced_vhosts["redirect"].add(new_vhost) # Finally create documentation for the change self.save_notes += ("Created a port 80 vhost, %s, for redirection to " @@ -1449,14 +1573,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Generate reversal command. # Try to be safe here... check that we can probably reverse before # applying enmod command - if not le_util.exe_exists(self.conf("dismod")): + if not util.exe_exists(self.conf("dismod")): raise errors.MisconfigurationError( "Unable to find a2dismod, please make sure a2enmod and " "a2dismod are configured correctly for certbot.") self.reverter.register_undo_command( temp, [self.conf("dismod"), mod_name]) - le_util.run_script([self.conf("enmod"), mod_name]) + util.run_script([self.conf("enmod"), mod_name]) def restart(self): """Runs a config test and reloads the Apache server. @@ -1475,7 +1599,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script(constants.os_constant("restart_cmd")) + util.run_script(constants.os_constant("restart_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1486,7 +1610,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - le_util.run_script(constants.os_constant("conftest_cmd")) + util.run_script(constants.os_constant("conftest_cmd")) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -1502,8 +1626,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ try: - stdout, _ = le_util.run_script( - constants.os_constant("version_cmd")) + stdout, _ = util.run_script(constants.os_constant("version_cmd")) except errors.SubprocessError: raise errors.PluginError( "Unable to run %s -v" % diff --git a/certbot-apache/certbot_apache/constants.py b/certbot-apache/certbot_apache/constants.py index f3226572c..9252814c4 100644 --- a/certbot-apache/certbot_apache/constants.py +++ b/certbot-apache/certbot_apache/constants.py @@ -1,6 +1,6 @@ """Apache plugin constants.""" import pkg_resources -from certbot import le_util +from certbot import util CLI_DEFAULTS_DEBIAN = dict( @@ -78,6 +78,9 @@ CLI_DEFAULTS = { "centos linux": CLI_DEFAULTS_CENTOS, "fedora": CLI_DEFAULTS_CENTOS, "red hat enterprise linux server": CLI_DEFAULTS_CENTOS, + "rhel": CLI_DEFAULTS_CENTOS, + "amazon": CLI_DEFAULTS_CENTOS, + "gentoo": CLI_DEFAULTS_GENTOO, "gentoo base system": CLI_DEFAULTS_GENTOO, "darwin": CLI_DEFAULTS_DARWIN, } @@ -116,7 +119,7 @@ def os_constant(key): :param key: name of cli constant :return: value of constant for active os """ - os_info = le_util.get_os_info() + os_info = util.get_os_info() try: constants = CLI_DEFAULTS[os_info[0].lower()] except KeyError: diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index c9359e7d3..24aabaed4 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -86,11 +86,12 @@ def _vhost_menu(domain, vhosts): "like to choose?\n(note: conf files with multiple " "vhosts are not yet supported)".format(domain, os.linesep), choices, help_label="More Info", ok_label="Select") - except errors.MissingCommandlineFlag as e: - msg = ("Failed to run Apache plugin non-interactively{1}{0}{1}" - "(The best solution is to add ServerName or ServerAlias " - "entries to the VirtualHost directives of your apache " - "configuration files.)".format(e, os.linesep)) + except errors.MissingCommandlineFlag: + msg = ("Encountered vhost ambiguity but unable to ask for user guidance in " + "non-interactive mode. Currently Certbot needs each vhost to be " + "in its own conf file, and may need vhosts to be explicitly " + "labelled with ServerName or ServerAlias directories.") + logger.warn(msg) raise errors.MissingCommandlineFlag(msg) return code, tag diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index b88b22428..c71443e92 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -6,6 +6,7 @@ from certbot.plugins import common class Addr(common.Addr): """Represents an Apache address.""" + def __eq__(self, other): """This is defined as equalivalent within Apache. @@ -21,6 +22,9 @@ class Addr(common.Addr): def __ne__(self, other): return not self.__eq__(other) + def __repr__(self): + return "certbot_apache.obj.Addr(" + repr(self.tup) + ")" + def _addr_less_specific(self, addr): """Returns if addr.get_addr() is more specific than self.get_addr().""" # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/comment-continuations-2050.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/comment-continuations-2050.conf new file mode 100644 index 000000000..48b344d8a --- /dev/null +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/comment-continuations-2050.conf @@ -0,0 +1,428 @@ +# --------------------------------------------------------------- +# Core ModSecurity Rule Set ver.2.2.6 +# Copyright (C) 2006-2012 Trustwave All rights reserved. +# +# The OWASP ModSecurity Core Rule Set is distributed under +# Apache Software License (ASL) version 2 +# Please see the enclosed LICENCE file for full details. +# --------------------------------------------------------------- + + +# +# -- [[ Recommended Base Configuration ]] ------------------------------------------------- +# +# The configuration directives/settings in this file are used to control +# the OWASP ModSecurity CRS. These settings do **NOT** configure the main +# ModSecurity settings such as: +# +# - SecRuleEngine +# - SecRequestBodyAccess +# - SecAuditEngine +# - SecDebugLog +# +# You should use the modsecurity.conf-recommended file that comes with the +# ModSecurity source code archive. +# +# Ref: http://mod-security.svn.sourceforge.net/viewvc/mod-security/m2/trunk/modsecurity.conf-recommended +# + + +# +# -- [[ Rule Version ]] ------------------------------------------------------------------- +# +# Rule version data is added to the "Producer" line of Section H of the Audit log: +# +# - Producer: ModSecurity for Apache/2.7.0-rc1 (http://www.modsecurity.org/); OWASP_CRS/2.2.4. +# +# Ref: https://sourceforge.net/apps/mediawiki/mod-security/index.php?title=Reference_Manual#SecComponentSignature +# +#SecComponentSignature "OWASP_CRS/2.2.6" + + +# +# -- [[ Modes of Operation: Self-Contained vs. Collaborative Detection ]] ----------------- +# +# Each detection rule uses the "block" action which will inherit the SecDefaultAction +# specified below. Your settings here will determine which mode of operation you use. +# +# -- [[ Self-Contained Mode ]] -- +# Rules inherit the "deny" disruptive action. The first rule that matches will block. +# +# -- [[ Collaborative Detection Mode ]] -- +# This is a "delayed blocking" mode of operation where each matching rule will inherit +# the "pass" action and will only contribute to anomaly scores. Transactional blocking +# can be applied +# +# -- [[ Alert Logging Control ]] -- +# You have three options - +# +# - To log to both the Apache error_log and ModSecurity audit_log file use: "log" +# - To log *only* to the ModSecurity audit_log file use: "nolog,auditlog" +# - To log *only* to the Apache error_log file use: "log,noauditlog" +# +# Ref: http://blog.spiderlabs.com/2010/11/advanced-topic-of-the-week-traditional-vs-anomaly-scoring-detection-modes.html +# Ref: https://sourceforge.net/apps/mediawiki/mod-security/index.php?title=Reference_Manual#SecDefaultAction +# +#SecDefaultAction "phase:1,deny,log" + + +# +# -- [[ Collaborative Detection Severity Levels ]] ---------------------------------------- +# +# These are the default scoring points for each severity level. You may +# adjust these to you liking. These settings will be used in macro expansion +# in the rules to increment the anomaly scores when rules match. +# +# These are the default Severity ratings (with anomaly scores) of the individual rules - +# +# - 2: Critical - Anomaly Score of 5. +# Is the highest severity level possible without correlation. It is +# normally generated by the web attack rules (40 level files). +# - 3: Error - Anomaly Score of 4. +# Is generated mostly from outbound leakage rules (50 level files). +# - 4: Warning - Anomaly Score of 3. +# Is generated by malicious client rules (35 level files). +# - 5: Notice - Anomaly Score of 2. +# Is generated by the Protocol policy and anomaly files. +# +#SecAction \ + "id:'900001', \ + phase:1, \ + t:none, \ + setvar:tx.critical_anomaly_score=5, \ + setvar:tx.error_anomaly_score=4, \ + setvar:tx.warning_anomaly_score=3, \ + setvar:tx.notice_anomaly_score=2, \ + nolog, \ + pass" + + +# +# -- [[ Collaborative Detection Scoring Threshold Levels ]] ------------------------------ +# +# These variables are used in macro expansion in the 49 inbound blocking and 59 +# outbound blocking files. +# +# **MUST HAVE** ModSecurity v2.5.12 or higher to use macro expansion in numeric +# operators. If you have an earlier version, edit the 49/59 files directly to +# set the appropriate anomaly score levels. +# +# You should set the score to the proper threshold you would prefer. If set to "5" +# it will work similarly to previous Mod CRS rules and will create an event in the error_log +# file if there are any rules that match. If you would like to lessen the number of events +# generated in the error_log file, you should increase the anomaly score threshold to +# something like "20". This would only generate an event in the error_log file if +# there are multiple lower severity rule matches or if any 1 higher severity item matches. +# +#SecAction \ + "id:'900002', \ + phase:1, \ + t:none, \ + setvar:tx.inbound_anomaly_score_level=5, \ + nolog, \ + pass" + + +#SecAction \ + "id:'900003', \ + phase:1, \ + t:none, \ + setvar:tx.outbound_anomaly_score_level=4, \ + nolog, \ + pass" + + +# +# -- [[ Collaborative Detection Blocking ]] ----------------------------------------------- +# +# This is a collaborative detection mode where each rule will increment an overall +# anomaly score for the transaction. The scores are then evaluated in the following files: +# +# Inbound anomaly score - checked in the modsecurity_crs_49_inbound_blocking.conf file +# Outbound anomaly score - checked in the modsecurity_crs_59_outbound_blocking.conf file +# +# If you want to use anomaly scoring mode, then uncomment this line. +# +#SecAction \ + "id:'900004', \ + phase:1, \ + t:none, \ + setvar:tx.anomaly_score_blocking=on, \ + nolog, \ + pass" + + +# +# -- [[ GeoIP Database ]] ----------------------------------------------------------------- +# +# There are some rulesets that need to inspect the GEO data of the REMOTE_ADDR data. +# +# You must first download the MaxMind GeoIP Lite City DB - +# +# http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +# +# You then need to define the proper path for the SecGeoLookupDb directive +# +# Ref: http://blog.spiderlabs.com/2010/10/detecting-malice-with-modsecurity-geolocation-data.html +# Ref: http://blog.spiderlabs.com/2010/11/detecting-malice-with-modsecurity-ip-forensics.html +# +#SecGeoLookupDb /opt/modsecurity/lib/GeoLiteCity.dat + +# +# -- [[ Regression Testing Mode ]] -------------------------------------------------------- +# +# If you are going to run the regression testing mode, you should uncomment the +# following rule. It will enable DetectionOnly mode for the SecRuleEngine and +# will enable Response Header tagging so that the client testing script can see +# which rule IDs have matched. +# +# You must specify the your source IP address where you will be running the tests +# from. +# +#SecRule REMOTE_ADDR "@ipMatch 192.168.1.100" \ + "id:'900005', \ + phase:1, \ + t:none, \ + ctl:ruleEngine=DetectionOnly, \ + setvar:tx.regression_testing=1, \ + nolog, \ + pass" + + +# +# -- [[ HTTP Policy Settings ]] ---------------------------------------------------------- +# +# Set the following policy settings here and they will be propagated to the 23 rules +# file (modsecurity_common_23_request_limits.conf) by using macro expansion. +# If you run into false positives, you can adjust the settings here. +# +# Only the max number of args is uncommented by default as there are a high rate +# of false positives. Uncomment the items you wish to set. +# +# +# -- Maximum number of arguments in request limited +#SecAction \ + "id:'900006', \ + phase:1, \ + t:none, \ + setvar:tx.max_num_args=255, \ + nolog, \ + pass" + +# +# -- Limit argument name length +#SecAction \ + "id:'900007', \ + phase:1, \ + t:none, \ + setvar:tx.arg_name_length=100, \ + nolog, \ + pass" + +# +# -- Limit value name length +#SecAction \ + "id:'900008', \ + phase:1, \ + t:none, \ + setvar:tx.arg_length=400, \ + nolog, \ + pass" + +# +# -- Limit arguments total length +#SecAction \ + "id:'900009', \ + phase:1, \ + t:none, \ + setvar:tx.total_arg_length=64000, \ + nolog, \ + pass" + +# +# -- Individual file size is limited +#SecAction \ + "id:'900010', \ + phase:1, \ + t:none, \ + setvar:tx.max_file_size=1048576, \ + nolog, \ + pass" + +# +# -- Combined file size is limited +#SecAction \ + "id:'900011', \ + phase:1, \ + t:none, \ + setvar:tx.combined_file_sizes=1048576, \ + nolog, \ + pass" + + +# +# Set the following policy settings here and they will be propagated to the 30 rules +# file (modsecurity_crs_30_http_policy.conf) by using macro expansion. +# If you run into false positves, you can adjust the settings here. +# +#SecAction \ + "id:'900012', \ + phase:1, \ + t:none, \ + setvar:'tx.allowed_methods=GET HEAD POST OPTIONS', \ + setvar:'tx.allowed_request_content_type=application/x-www-form-urlencoded|multipart/form-data|text/xml|application/xml|application/x-amf', \ + setvar:'tx.allowed_http_versions=HTTP/0.9 HTTP/1.0 HTTP/1.1', \ + setvar:'tx.restricted_extensions=.asa/ .asax/ .ascx/ .axd/ .backup/ .bak/ .bat/ .cdx/ .cer/ .cfg/ .cmd/ .com/ .config/ .conf/ .cs/ .csproj/ .csr/ .dat/ .db/ .dbf/ .dll/ .dos/ .htr/ .htw/ .ida/ .idc/ .idq/ .inc/ .ini/ .key/ .licx/ .lnk/ .log/ .mdb/ .old/ .pass/ .pdb/ .pol/ .printer/ .pwd/ .resources/ .resx/ .sql/ .sys/ .vb/ .vbs/ .vbproj/ .vsdisco/ .webinfo/ .xsd/ .xsx/', \ + setvar:'tx.restricted_headers=/Proxy-Connection/ /Lock-Token/ /Content-Range/ /Translate/ /via/ /if/', \ + nolog, \ + pass" + + +# +# -- [[ Content Security Policy (CSP) Settings ]] ----------------------------------------- +# +# The purpose of these settings is to send CSP response headers to +# Mozilla FireFox users so that you can enforce how dynamic content +# is used. CSP usage helps to prevent XSS attacks against your users. +# +# Reference Link: +# +# https://developer.mozilla.org/en/Security/CSP +# +# Uncomment this SecAction line if you want use CSP enforcement. +# You need to set the appropriate directives and settings for your site/domain and +# and activate the CSP file in the experimental_rules directory. +# +# Ref: http://blog.spiderlabs.com/2011/04/modsecurity-advanced-topic-of-the-week-integrating-content-security-policy-csp.html +# +#SecAction \ + "id:'900013', \ + phase:1, \ + t:none, \ + setvar:tx.csp_report_only=1, \ + setvar:tx.csp_report_uri=/csp_violation_report, \ + setenv:'csp_policy=allow \'self\'; img-src *.yoursite.com; media-src *.yoursite.com; style-src *.yoursite.com; frame-ancestors *.yoursite.com; script-src *.yoursite.com; report-uri %{tx.csp_report_uri}', \ + nolog, \ + pass" + + +# +# -- [[ Brute Force Protection ]] --------------------------------------------------------- +# +# If you are using the Brute Force Protection rule set, then uncomment the following +# lines and set the following variables: +# - Protected URLs: resources to protect (e.g. login pages) - set to your login page +# - Burst Time Slice Interval: time interval window to monitor for bursts +# - Request Threshold: request # threshold to trigger a burst +# - Block Period: temporary block timeout +# +#SecAction \ + "id:'900014', \ + phase:1, \ + t:none, \ + setvar:'tx.brute_force_protected_urls=/login.jsp /partner_login.php', \ + setvar:'tx.brute_force_burst_time_slice=60', \ + setvar:'tx.brute_force_counter_threshold=10', \ + setvar:'tx.brute_force_block_timeout=300', \ + nolog, \ + pass" + + +# +# -- [[ DoS Protection ]] ---------------------------------------------------------------- +# +# If you are using the DoS Protection rule set, then uncomment the following +# lines and set the following variables: +# - Burst Time Slice Interval: time interval window to monitor for bursts +# - Request Threshold: request # threshold to trigger a burst +# - Block Period: temporary block timeout +# +#SecAction \ + "id:'900015', \ + phase:1, \ + t:none, \ + setvar:'tx.dos_burst_time_slice=60', \ + setvar:'tx.dos_counter_threshold=100', \ + setvar:'tx.dos_block_timeout=600', \ + nolog, \ + pass" + + +# +# -- [[ Check UTF enconding ]] ----------------------------------------------------------- +# +# We only want to apply this check if UTF-8 encoding is actually used by the site, otherwise +# it will result in false positives. +# +# Uncomment this line if your site uses UTF8 encoding +#SecAction \ + "id:'900016', \ + phase:1, \ + t:none, \ + setvar:tx.crs_validate_utf8_encoding=1, \ + nolog, \ + pass" + + +# +# -- [[ Enable XML Body Parsing ]] ------------------------------------------------------- +# +# The rules in this file will trigger the XML parser upon an XML request +# +# Initiate XML Processor in case of xml content-type +# +#SecRule REQUEST_HEADERS:Content-Type "text/xml" \ + "id:'900017', \ + phase:1, \ + t:none,t:lowercase, \ + nolog, \ + pass, \ + chain" + #SecRule REQBODY_PROCESSOR "!@streq XML" \ + "ctl:requestBodyProcessor=XML" + + +# +# -- [[ Global and IP Collections ]] ----------------------------------------------------- +# +# Create both Global and IP collections for rules to use +# There are some CRS rules that assume that these two collections +# have already been initiated. +# +#SecRule REQUEST_HEADERS:User-Agent "^(.*)$" \ + "id:'900018', \ + phase:1, \ + t:none,t:sha1,t:hexEncode, \ + setvar:tx.ua_hash=%{matched_var}, \ + nolog, \ + pass" + + +#SecRule REQUEST_HEADERS:x-forwarded-for "^\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b" \ + "id:'900019', \ + phase:1, \ + t:none, \ + capture, \ + setvar:tx.real_ip=%{tx.1}, \ + nolog, \ + pass" + + +#SecRule &TX:REAL_IP "!@eq 0" \ + "id:'900020', \ + phase:1, \ + t:none, \ + initcol:global=global, \ + initcol:ip=%{tx.real_ip}_%{tx.ua_hash}, \ + nolog, \ + pass" + + +#SecRule &TX:REAL_IP "@eq 0" \ + "id:'900021', \ + phase:1, \ + t:none, \ + initcol:global=global, \ + initcol:ip=%{remote_addr}_%{tx.ua_hash}, \ + nolog, \ + pass" diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/escaped-space-arguments-2735.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/escaped-space-arguments-2735.conf new file mode 100644 index 000000000..1ea53dfab --- /dev/null +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/escaped-space-arguments-2735.conf @@ -0,0 +1,2 @@ +RewriteCond %{HTTP:Content-Disposition} \.php [NC] +RewriteCond %{THE_REQUEST} ^[A-Z]{3,9}\ /.+/trackback/?\ HTTP/ [NC] diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/passing/section-empty-continuations-2731.conf b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/section-empty-continuations-2731.conf new file mode 100644 index 000000000..3f2f96965 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/passing/section-empty-continuations-2731.conf @@ -0,0 +1,247 @@ +#ATTENTION! +# +#DO NOT MODIFY THIS FILE BECAUSE IT WAS GENERATED AUTOMATICALLY, +#SO ALL YOUR CHANGES WILL BE LOST THE NEXT TIME THE FILE IS GENERATED. + +NameVirtualHost 192.168.100.218:80 +NameVirtualHost 10.128.178.192:80 + +NameVirtualHost 192.168.100.218:443 +NameVirtualHost 10.128.178.192:443 + + +ServerName "254020-web1.example.com" +ServerAdmin "name@example.com" + +DocumentRoot "/tmp" + + + LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" plesklog + + + TraceEnable off + +ServerTokens ProductOnly + + + AllowOverride "All" + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + +php_admin_flag engine off + + + +php_admin_flag engine off + + + + + + AllowOverride All + Options SymLinksIfOwnerMatch + Order allow,deny + Allow from all + + php_admin_flag engine off + + + php_admin_flag engine off + + + + + Header add X-Powered-By PleskLin + + + + JkWorkersFile "/etc/httpd/conf/workers.properties" + JkLogFile /var/log/httpd/mod_jk.log + JkLogLevel info + + +#Include "/etc/httpd/conf/plesk.conf.d/ip_default/*.conf" + + + + ServerName "default" + UseCanonicalName Off + DocumentRoot "/tmp" + ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin" + + + + SSLEngine off + + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + +php_admin_flag engine on + + + +php_admin_flag engine on + + + + + + + + + + ServerName "default-192_168_100_218" + UseCanonicalName Off + DocumentRoot "/tmp" + ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin" + + + SSLEngine on + SSLVerifyClient none + #SSLCertificateFile "/usr/local/psa/var/certificates/cert-9MgutN" + + #SSLCACertificateFile "/usr/local/psa/var/certificates/cert-s6Wx3P" + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + +php_admin_flag engine on + + + +php_admin_flag engine on + + + + + + + ServerName "default-10_128_178_192" + UseCanonicalName Off + DocumentRoot "/tmp" + ScriptAlias /cgi-bin/ "/var/www/vhosts/default/cgi-bin" + + + SSLEngine on + SSLVerifyClient none + #SSLCertificateFile "/usr/local/psa/var/certificates/certxfb6025" + + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + +php_admin_flag engine on + + + +php_admin_flag engine on + + + + + + + + + + + DocumentRoot "/tmp" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + + SSLEngine off + + + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + DocumentRoot "/tmp" + ServerName lists + ServerAlias lists.* + UseCanonicalName Off + + ScriptAlias "/mailman/" "/usr/lib/mailman/cgi-bin/" + + Alias "/icons/" "/var/www/icons/" + Alias "/pipermail/" "/var/lib/mailman/archives/public/" + + SSLEngine on + SSLVerifyClient none + #SSLCertificateFile "/usr/local/psa/var/certificates/certxfb6025" + + + + Options FollowSymLinks + Order allow,deny + Allow from all + + + + + + + RPAFproxy_ips 192.168.100.218 10.128.178.192 + + + RPAFproxy_ips 192.168.100.218 10.128.178.192 + diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index f2f78c8f9..9a034c3e0 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -15,6 +15,7 @@ from certbot import errors from certbot.tests import acme_util from certbot_apache import configurator +from certbot_apache import parser from certbot_apache import obj from certbot_apache.tests import util @@ -48,14 +49,24 @@ class MultipleVhostsTest(util.ApacheTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - @mock.patch("certbot_apache.configurator.le_util.exe_exists") + @mock.patch("certbot_apache.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): mock_exe_exists.return_value = False self.assertRaises( errors.NoInstallationError, self.config.prepare) + @mock.patch("certbot_apache.augeas_configurator.AugeasConfigurator.init_augeas") + def test_prepare_no_augeas(self, mock_init_augeas): + """ Test augeas initialization ImportError """ + def side_effect_error(): + """ Side effect error for the test """ + raise ImportError + mock_init_augeas.side_effect = side_effect_error + self.assertRaises( + errors.NoInstallationError, self.config.prepare) + @mock.patch("certbot_apache.parser.ApacheParser") - @mock.patch("certbot_apache.configurator.le_util.exe_exists") + @mock.patch("certbot_apache.configurator.util.exe_exists") def test_prepare_version(self, mock_exe_exists, _): mock_exe_exists.return_value = True self.config.version = None @@ -66,7 +77,7 @@ class MultipleVhostsTest(util.ApacheTest): errors.NotSupportedError, self.config.prepare) @mock.patch("certbot_apache.parser.ApacheParser") - @mock.patch("certbot_apache.configurator.le_util.exe_exists") + @mock.patch("certbot_apache.configurator.util.exe_exists") def test_prepare_old_aug(self, mock_exe_exists, _): mock_exe_exists.return_value = True self.config.config_test = mock.Mock() @@ -85,7 +96,8 @@ class MultipleVhostsTest(util.ApacheTest): mock_getutility.notification = mock.MagicMock(return_value=True) names = self.config.get_all_names() self.assertEqual(names, set( - ["certbot.demo", "encryption-example.demo", "ip-172-30-0-17", "*.blue.purple.com"])) + ["certbot.demo", "ocspvhost.com", "encryption-example.demo", + "ip-172-30-0-17", "*.blue.purple.com"])) @mock.patch("zope.component.getUtility") @mock.patch("certbot_apache.configurator.socket.gethostbyaddr") @@ -100,14 +112,24 @@ class MultipleVhostsTest(util.ApacheTest): obj.Addr(("zombo.com",)), obj.Addr(("192.168.1.2"))]), True, False) + self.config.vhosts.append(vhost) names = self.config.get_all_names() - self.assertEqual(len(names), 6) + self.assertEqual(len(names), 7) self.assertTrue("zombo.com" in names) self.assertTrue("google.com" in names) self.assertTrue("certbot.demo" in names) + def test_bad_servername_alias(self): + ssl_vh1 = obj.VirtualHost( + "fp1", "ap1", set([obj.Addr(("*", "443"))]), + True, False) + # pylint: disable=protected-access + self.config._add_servernames(ssl_vh1) + self.assertTrue( + self.config._add_servername_alias("oy_vey", ssl_vh1) is None) + def test_add_servernames_alias(self): self.config.parser.add_dir( self.vh_truth[2].path, "ServerAlias", ["*.le.co"]) @@ -124,7 +146,7 @@ class MultipleVhostsTest(util.ApacheTest): """ vhs = self.config.get_virtual_hosts() - self.assertEqual(len(vhs), 7) + self.assertEqual(len(vhs), 8) found = 0 for vhost in vhs: @@ -135,7 +157,7 @@ class MultipleVhostsTest(util.ApacheTest): else: raise Exception("Missed: %s" % vhost) # pragma: no cover - self.assertEqual(found, 7) + self.assertEqual(found, 8) # Handle case of non-debian layout get_virtual_hosts with mock.patch( @@ -143,7 +165,7 @@ class MultipleVhostsTest(util.ApacheTest): ) as mock_conf: mock_conf.return_value = False vhs = self.config.get_virtual_hosts() - self.assertEqual(len(vhs), 7) + self.assertEqual(len(vhs), 8) @mock.patch("certbot_apache.display_ops.select_vhost") def test_choose_vhost_none_avail(self, mock_select): @@ -224,16 +246,18 @@ class MultipleVhostsTest(util.ApacheTest): # Assume only the two default vhosts. self.config.vhosts = [ vh for vh in self.config.vhosts - if vh.name not in ["certbot.demo", "encryption-example.demo"] + if vh.name not in ["certbot.demo", + "encryption-example.demo", + "ocspvhost.com"] and "*.blue.purple.com" not in vh.aliases ] - self.assertEqual( - self.config._find_best_vhost("example.demo"), self.vh_truth[2]) + self.config._find_best_vhost("encryption-example.demo"), + self.vh_truth[2]) def test_non_default_vhosts(self): # pylint: disable=protected-access - self.assertEqual(len(self.config._non_default_vhosts()), 5) + self.assertEqual(len(self.config._non_default_vhosts()), 6) def test_is_site_enabled(self): """Test if site is enabled. @@ -254,8 +278,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.is_site_enabled, "irrelevant") - @mock.patch("certbot.le_util.run_script") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") @mock.patch("certbot_apache.parser.subprocess.Popen") def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script): mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "") @@ -273,7 +297,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.NotSupportedError, self.config.enable_mod, "ssl") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.exe_exists") def test_enable_mod_no_disable(self, mock_exe_exists): mock_exe_exists.return_value = False self.assertRaises( @@ -473,13 +497,8 @@ class MultipleVhostsTest(util.ApacheTest): # Test Listen statements with specific ip listeed self.config.prepare_server_https("443") - # Should only be 2 here, as the third interface - # already listens to the correct port - self.assertEqual(mock_add_dir.call_count, 2) - - # Check argument to new Listen statements - self.assertEqual(mock_add_dir.call_args_list[0][0][2], ["1.2.3.4:443"]) - self.assertEqual(mock_add_dir.call_args_list[1][0][2], ["[::1]:443"]) + # Should be 0 as one interface already listens to 443 + self.assertEqual(mock_add_dir.call_count, 0) # Reset return lists and inputs mock_add_dir.reset_mock() @@ -495,6 +514,28 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_args_list[2][0][2], ["1.1.1.1:8080", "https"]) + # mock_get.side_effect = ["1.2.3.4:80", "[::1]:80"] + # mock_find.return_value = ["test1", "test2", "test3"] + # self.config.parser.get_arg = mock_get + # self.config.prepare_server_https("8080", temp=True) + # self.assertEqual(self.listens, 0) + + def test_prepare_server_https_needed_listen(self): + mock_find = mock.Mock() + mock_find.return_value = ["test1", "test2"] + mock_get = mock.Mock() + mock_get.side_effect = ["1.2.3.4:8080", "80"] + mock_add_dir = mock.Mock() + mock_enable = mock.Mock() + + self.config.parser.find_dir = mock_find + self.config.parser.get_arg = mock_get + self.config.parser.add_dir_to_ifmodssl = mock_add_dir + self.config.enable_mod = mock_enable + + self.config.prepare_server_https("443") + self.assertEqual(mock_add_dir.call_count, 1) + def test_prepare_server_https_mixed_listen(self): mock_find = mock.Mock() @@ -539,7 +580,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]), self.config.is_name_vhost(ssl_vhost)) - self.assertEqual(len(self.config.vhosts), 8) + self.assertEqual(len(self.config.vhosts), 9) def test_clean_vhost_ssl(self): # pylint: disable=protected-access @@ -681,7 +722,7 @@ class MultipleVhostsTest(util.ApacheTest): self.config.cleanup([achall1, achall2]) self.assertTrue(mock_restart.called) - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_get_version(self, mock_script): mock_script.return_value = ( "Server Version: Apache/2.4.2 (Debian)", "") @@ -703,21 +744,21 @@ class MultipleVhostsTest(util.ApacheTest): mock_script.side_effect = errors.SubprocessError("Can't find program") self.assertRaises(errors.PluginError, self.config.get_version) - @mock.patch("certbot_apache.configurator.le_util.run_script") + @mock.patch("certbot_apache.configurator.util.run_script") def test_restart(self, _): self.config.restart() - @mock.patch("certbot_apache.configurator.le_util.run_script") + @mock.patch("certbot_apache.configurator.util.run_script") def test_restart_bad_process(self, mock_run_script): mock_run_script.side_effect = [None, errors.SubprocessError] self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_config_test(self, _): self.config.config_test() - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_config_test_bad_process(self, mock_run_script): mock_run_script.side_effect = errors.SubprocessError @@ -726,16 +767,15 @@ class MultipleVhostsTest(util.ApacheTest): def test_get_all_certs_keys(self): c_k = self.config.get_all_certs_keys() - - self.assertEqual(len(c_k), 2) + self.assertEqual(len(c_k), 3) cert, key, path = next(iter(c_k)) self.assertTrue("cert" in cert) self.assertTrue("key" in key) - self.assertTrue("default-ssl" in path) + self.assertTrue("default-ssl" in path or "ocsp-ssl" in path) def test_get_all_certs_keys_malformed_conf(self): self.config.parser.find_dir = mock.Mock( - side_effect=[["path"], [], ["path"], []]) + side_effect=[["path"], [], ["path"], [], ["path"], []]) c_k = self.config.get_all_certs_keys() self.assertFalse(c_k) @@ -756,15 +796,20 @@ class MultipleVhostsTest(util.ApacheTest): def test_supported_enhancements(self): self.assertTrue(isinstance(self.config.supported_enhancements(), list)) - @mock.patch("certbot.le_util.exe_exists") - def test_enhance_unknown_vhost(self, mock_exe): + @mock.patch("certbot_apache.configurator.ApacheConfigurator._get_http_vhost") + @mock.patch("certbot_apache.display_ops.select_vhost") + @mock.patch("certbot.util.exe_exists") + def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get): self.config.parser.modules.add("rewrite_module") mock_exe.return_value = True - ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443"))]), + ssl_vh1 = obj.VirtualHost( + "fp1", "ap1", set([obj.Addr(("*", "443"))]), True, False) - ssl_vh.name = "satoshi.com" - self.config.vhosts.append(ssl_vh) + ssl_vh1.name = "satoshi.com" + self.config.vhosts.append(ssl_vh1) + mock_sel_vhost.return_value = None + mock_get.return_value = None + self.assertRaises( errors.PluginError, self.config.enhance, "satoshi.com", "redirect") @@ -774,8 +819,87 @@ class MultipleVhostsTest(util.ApacheTest): errors.PluginError, self.config.enhance, "certbot.demo", "unknown_enhancement") - @mock.patch("certbot.le_util.run_script") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") + def test_ocsp_stapling(self, mock_exe, mock_run_script): + self.config.parser.update_runtime_variables = mock.Mock() + self.config.parser.modules.add("mod_ssl.c") + self.config.get_version = mock.Mock(return_value=(2, 4, 7)) + mock_exe.return_value = True + + # This will create an ssl vhost for certbot.demo + self.config.enhance("certbot.demo", "staple-ocsp") + + self.assertTrue("socache_shmcb_module" in self.config.parser.modules) + self.assertTrue(mock_run_script.called) + + # Get the ssl vhost for certbot.demo + ssl_vhost = self.config.assoc["certbot.demo"] + + ssl_use_stapling_aug_path = self.config.parser.find_dir( + "SSLUseStapling", "on", ssl_vhost.path) + + self.assertEqual(len(ssl_use_stapling_aug_path), 1) + + ssl_vhost_aug_path = parser.get_aug_path(ssl_vhost.filep) + stapling_cache_aug_path = self.config.parser.find_dir('SSLStaplingCache', + "shmcb:/var/run/apache2/stapling_cache(128000)", + ssl_vhost_aug_path) + + self.assertEqual(len(stapling_cache_aug_path), 1) + + @mock.patch("certbot.util.exe_exists") + def test_ocsp_stapling_twice(self, mock_exe): + self.config.parser.update_runtime_variables = mock.Mock() + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") + self.config.get_version = mock.Mock(return_value=(2, 4, 7)) + mock_exe.return_value = True + + # Checking the case with already enabled ocsp stapling configuration + self.config.enhance("ocspvhost.com", "staple-ocsp") + + # Get the ssl vhost for letsencrypt.demo + ssl_vhost = self.config.assoc["ocspvhost.com"] + + ssl_use_stapling_aug_path = self.config.parser.find_dir( + "SSLUseStapling", "on", ssl_vhost.path) + + self.assertEqual(len(ssl_use_stapling_aug_path), 1) + + ssl_vhost_aug_path = parser.get_aug_path(ssl_vhost.filep) + stapling_cache_aug_path = self.config.parser.find_dir('SSLStaplingCache', + "shmcb:/var/run/apache2/stapling_cache(128000)", + ssl_vhost_aug_path) + + self.assertEqual(len(stapling_cache_aug_path), 1) + + + @mock.patch("certbot.util.exe_exists") + def test_ocsp_unsupported_apache_version(self, mock_exe): + mock_exe.return_value = True + self.config.parser.update_runtime_variables = mock.Mock() + self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules.add("socache_shmcb_module") + self.config.get_version = mock.Mock(return_value=(2, 2, 0)) + + self.assertRaises(errors.PluginError, + self.config.enhance, "certbot.demo", "staple-ocsp") + + + def test_get_http_vhost_third_filter(self): + ssl_vh = obj.VirtualHost( + "fp", "ap", set([obj.Addr(("*", "443"))]), + True, False) + ssl_vh.name = "satoshi.com" + self.config.vhosts.append(ssl_vh) + + # pylint: disable=protected-access + http_vh = self.config._get_http_vhost(ssl_vh) + self.assertTrue(http_vh.ssl == False) + + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") def test_http_header_hsts(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() self.config.parser.modules.add("mod_ssl.c") @@ -812,8 +936,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enhance, "encryption-example.demo", "ensure-http-header", "Strict-Transport-Security") - @mock.patch("certbot.le_util.run_script") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") def test_http_header_uir(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() self.config.parser.modules.add("mod_ssl.c") @@ -850,8 +974,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enhance, "encryption-example.demo", "ensure-http-header", "Upgrade-Insecure-Requests") - @mock.patch("certbot.le_util.run_script") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") def test_redirect_well_formed_http(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True @@ -894,12 +1018,12 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.assertTrue(self.config._is_rewrite_engine_on(self.vh_truth[3])) - @mock.patch("certbot.le_util.run_script") - @mock.patch("certbot.le_util.exe_exists") + @mock.patch("certbot.util.run_script") + @mock.patch("certbot.util.exe_exists") def test_redirect_with_existing_rewrite(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True - self.config.get_version = mock.Mock(return_value=(2, 2)) + self.config.get_version = mock.Mock(return_value=(2, 2, 0)) # Create a preexisting rewrite rule self.config.parser.add_dir( @@ -938,15 +1062,31 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.PluginError, self.config._enable_redirect, ssl_vh, "") - def test_redirect_twice(self): + def test_redirect_two_domains_one_vhost(self): # Skip the enable mod self.config.parser.modules.add("rewrite_module") self.config.get_version = mock.Mock(return_value=(2, 3, 9)) - self.config.enhance("encryption-example.demo", "redirect") + self.config.enhance("red.blue.purple.com", "redirect") + verify_no_redirect = ("certbot_apache.configurator." + "ApacheConfigurator._verify_no_certbot_redirect") + with mock.patch(verify_no_redirect) as mock_verify: + self.config.enhance("green.blue.purple.com", "redirect") + self.assertFalse(mock_verify.called) + + def test_redirect_from_previous_run(self): + # Skip the enable mod + self.config.parser.modules.add("rewrite_module") + self.config.get_version = mock.Mock(return_value=(2, 3, 9)) + + self.config.enhance("red.blue.purple.com", "redirect") + # Clear state about enabling redirect on this run + # pylint: disable=protected-access + self.config._enhanced_vhosts["redirect"].clear() + self.assertRaises( errors.PluginEnhancementAlreadyPresent, - self.config.enhance, "encryption-example.demo", "redirect") + self.config.enhance, "green.blue.purple.com", "redirect") def test_create_own_redirect(self): self.config.parser.modules.add("rewrite_module") @@ -957,7 +1097,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") - self.assertEqual(len(self.config.vhosts), 8) + self.assertEqual(len(self.config.vhosts), 9) def test_create_own_redirect_for_old_apache_version(self): self.config.parser.modules.add("rewrite_module") @@ -968,7 +1108,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") - self.assertEqual(len(self.config.vhosts), 8) + self.assertEqual(len(self.config.vhosts), 9) def test_sift_line(self): # pylint: disable=protected-access diff --git a/certbot-apache/certbot_apache/tests/constants_test.py b/certbot-apache/certbot_apache/tests/constants_test.py index d970c96be..c040030df 100644 --- a/certbot-apache/certbot_apache/tests/constants_test.py +++ b/certbot-apache/certbot_apache/tests/constants_test.py @@ -8,19 +8,19 @@ from certbot_apache import constants class ConstantsTest(unittest.TestCase): - @mock.patch("certbot.le_util.get_os_info") + @mock.patch("certbot.util.get_os_info") def test_get_debian_value(self, os_info): os_info.return_value = ('Debian', '', '') self.assertEqual(constants.os_constant("vhost_root"), "/etc/apache2/sites-available") - @mock.patch("certbot.le_util.get_os_info") + @mock.patch("certbot.util.get_os_info") def test_get_centos_value(self, os_info): os_info.return_value = ('CentOS Linux', '', '') self.assertEqual(constants.os_constant("vhost_root"), "/etc/httpd/conf.d") - @mock.patch("certbot.le_util.get_os_info") + @mock.patch("certbot.util.get_os_info") def test_get_default_value(self, os_info): os_info.return_value = ('Nonexistent Linux', '', '') self.assertEqual(constants.os_constant("vhost_root"), diff --git a/certbot-apache/certbot_apache/tests/display_ops_test.py b/certbot-apache/certbot_apache/tests/display_ops_test.py index fd1e52fde..585661c7f 100644 --- a/certbot-apache/certbot_apache/tests/display_ops_test.py +++ b/certbot-apache/certbot_apache/tests/display_ops_test.py @@ -38,7 +38,7 @@ class SelectVhostTest(unittest.TestCase): try: self._call(self.vhosts) except errors.MissingCommandlineFlag as e: - self.assertTrue("VirtualHost directives" in e.message) + self.assertTrue("vhost ambiguity" in e.message) @mock.patch("certbot_apache.display_ops.zope.component.getUtility") def test_more_info_cancel(self, mock_util): diff --git a/certbot-apache/certbot_apache/tests/obj_test.py b/certbot-apache/certbot_apache/tests/obj_test.py index 4c3d331be..10dba18bc 100644 --- a/certbot-apache/certbot_apache/tests/obj_test.py +++ b/certbot-apache/certbot_apache/tests/obj_test.py @@ -22,6 +22,9 @@ class VirtualHostTest(unittest.TestCase): self.vhost2 = VirtualHost( "fp", "vhp", set([self.addr2]), False, False, "localhost") + def test_repr(self): + self.assertEqual(repr(self.addr2), "certbot_apache.obj.Addr(('127.0.0.1', '443'))") + def test_eq(self): self.assertTrue(self.vhost1b == self.vhost1) self.assertFalse(self.vhost1 == self.vhost2) diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/ocsp-ssl.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/ocsp-ssl.conf new file mode 100644 index 000000000..631cf16c8 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-available/ocsp-ssl.conf @@ -0,0 +1,36 @@ + +SSLStaplingCache shmcb:/var/run/apache2/stapling_cache(128000) + + # The ServerName directive sets the request scheme, hostname and port that + # the server uses to identify itself. This is used when creating + # redirection URLs. In the context of virtual hosts, the ServerName + # specifies what hostname must appear in the request's Host: header to + # match this virtual host. For the default virtual host (this file) this + # value is not decisive as it is used as a last resort host regardless. + # However, you must set it for any further virtual host explicitly. + ServerName ocspvhost.com + + ServerAdmin webmaster@dumpbits.com + DocumentRoot /var/www/html + + # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, + # error, crit, alert, emerg. + # It is also possible to configure the loglevel for particular + # modules, e.g. + #LogLevel info ssl:warn + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + # For most configuration files from conf-available/, which are + # enabled or disabled at a global level, it is possible to + # include a line for only one particular virtual host. For example the + # following line enables the CGI configuration for this host only + # after it has been globally disabled with "a2disconf". + #Include conf-available/serve-cgi-bin.conf +SSLCertificateFile /etc/apache2/certs/certbot-cert_5.pem +SSLCertificateKeyFile /etc/apache2/ssl/key-certbot_15.pem +SSLUseStapling on + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet + diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/ocsp-ssl.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/ocsp-ssl.conf new file mode 120000 index 000000000..b25ee0482 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/apache2/sites-enabled/ocsp-ssl.conf @@ -0,0 +1 @@ +../sites-available/ocsp-ssl.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/sites b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/sites index 06bf6a2ae..ab518ee5b 100644 --- a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/sites +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multiple_vhosts/sites @@ -1,2 +1,3 @@ sites-available/certbot.conf, certbot.demo sites-available/encryption-example.conf, encryption-example.demo +sites-available/ocsp-ssl.conf, ocspvhost.com diff --git a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py index aa6a2a09c..5e369e3db 100644 --- a/certbot-apache/certbot_apache/tests/tls_sni_01_test.py +++ b/certbot-apache/certbot_apache/tests/tls_sni_01_test.py @@ -38,8 +38,8 @@ class TlsSniPerformTest(util.ApacheTest): resp = self.sni.perform() self.assertEqual(len(resp), 0) - @mock.patch("certbot.le_util.exe_exists") - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.exe_exists") + @mock.patch("certbot.util.run_script") def test_perform1(self, _, mock_exists): mock_register = mock.Mock() self.sni.configurator.reverter.register_undo_command = mock_register diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 9fb5dcdfa..050876687 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -95,8 +95,8 @@ def get_apache_configurator( in_progress_dir=os.path.join(backups, "IN_PROGRESS"), work_dir=work_dir) - with mock.patch("certbot_apache.configurator.le_util.run_script"): - with mock.patch("certbot_apache.configurator.le_util." + with mock.patch("certbot_apache.configurator.util.run_script"): + with mock.patch("certbot_apache.configurator.util." "exe_exists") as mock_exe_exists: mock_exe_exists.return_value = True with mock.patch("certbot_apache.parser.ApacheParser." @@ -156,8 +156,12 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "wildcard.conf"), os.path.join(aug_pre, "wildcard.conf/VirtualHost"), set([obj.Addr.fromstring("*:80")]), False, False, - "ip-172-30-0-17", aliases=["*.blue.purple.com"]) - ] + "ip-172-30-0-17", aliases=["*.blue.purple.com"]), + obj.VirtualHost( + os.path.join(prefix, "ocsp-ssl.conf"), + os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"), + set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + "ocspvhost.com")] return vh_truth return None # pragma: no cover diff --git a/certbot-apache/certbot_apache/tls_sni_01.py b/certbot-apache/certbot_apache/tls_sni_01.py index f14f7be0f..06d9e9ebd 100644 --- a/certbot-apache/certbot_apache/tls_sni_01.py +++ b/certbot-apache/certbot_apache/tls_sni_01.py @@ -129,6 +129,7 @@ class ApacheTlsSni01(common.TLSSNI01): # because it's a new vhost that's not configured yet (GH #677), # or perhaps because there were multiple sections # in the config file (GH #1042). See also GH #2600. + logger.warn("Falling back to default vhost %s...", default_addr) addrs.add(default_addr) return addrs diff --git a/certbot-apache/docs/conf.py b/certbot-apache/docs/conf.py index 2f996c7f4..d2fe15581 100644 --- a/certbot-apache/docs/conf.py +++ b/certbot-apache/docs/conf.py @@ -314,5 +314,5 @@ texinfo_documents = [ intersphinx_mapping = { 'python': ('https://docs.python.org/', None), 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), - 'certbot': ('https://letsencrypt.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), } diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 7358c7041..e3dbe4563 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.6.0.dev0' +version = '0.9.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto new file mode 100755 index 000000000..80f39cf59 --- /dev/null +++ b/certbot-auto @@ -0,0 +1,1122 @@ +#!/bin/sh +# +# Download and run the latest release version of the Certbot client. +# +# NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING +# +# IF YOU WANT TO EDIT IT LOCALLY, *ALWAYS* RUN YOUR COPY WITH THE +# "--no-self-upgrade" FLAG +# +# IF YOU WANT TO SEND PULL REQUESTS, THE REAL SOURCE FOR THIS FILE IS +# letsencrypt-auto-source/letsencrypt-auto.template AND +# letsencrypt-auto-source/pieces/bootstrappers/* + +set -e # Work even if somebody does "sh thisscript.sh". + +# Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, +# if you want to change where the virtual environment will be installed +XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} +VENV_NAME="letsencrypt" +VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} +VENV_BIN="$VENV_PATH/bin" +LE_AUTO_VERSION="0.8.1" +BASENAME=$(basename $0) +USAGE="Usage: $BASENAME [OPTIONS] +A self-updating wrapper script for the Certbot ACME client. When run, updates +to both this script and certbot will be downloaded and installed. After +ensuring you have the latest versions installed, certbot will be invoked with +all arguments you have provided. + +Help for certbot itself cannot be provided until it is installed. + + --debug attempt experimental installation + -h, --help print this help + -n, --non-interactive, --noninteractive run without asking for user input + --no-self-upgrade do not download updates + --os-packages-only install OS dependencies and exit + -v, --verbose provide more output + +All arguments are accepted and forwarded to the Certbot client when run." + +for arg in "$@" ; do + case "$arg" in + --debug) + DEBUG=1;; + --os-packages-only) + OS_PACKAGES_ONLY=1;; + --no-self-upgrade) + # Do not upgrade this script (also prevents client upgrades, because each + # copy of the script pins a hash of the python client) + NO_SELF_UPGRADE=1;; + --help) + HELP=1;; + --noninteractive|--non-interactive) + ASSUME_YES=1;; + --verbose) + VERBOSE=1;; + -[!-]*) + while getopts ":hnv" short_arg $arg; do + case "$short_arg" in + h) + HELP=1;; + n) + ASSUME_YES=1;; + v) + VERBOSE=1;; + esac + done;; + esac +done + +if [ $BASENAME = "letsencrypt-auto" ]; then + # letsencrypt-auto does not respect --help or --yes for backwards compatibility + ASSUME_YES=1 + HELP=0 +fi + +# certbot-auto needs root access to bootstrap OS dependencies, and +# certbot itself needs root access for almost all modes of operation +# The "normal" case is that sudo is used for the steps that need root, but +# this script *can* be run as root (not recommended), or fall back to using +# `su` +SUDO_ENV="" +export CERTBOT_AUTO="$0" +if test "`id -u`" -ne "0" ; then + if command -v sudo 1>/dev/null 2>&1; then + SUDO=sudo + SUDO_ENV="CERTBOT_AUTO=$0" + else + echo \"sudo\" is not available, will use \"su\" for installation steps... + # Because the parameters in `su -c` has to be a string, + # we need properly escape it + su_sudo() { + args="" + # This `while` loop iterates over all parameters given to this function. + # For each parameter, all `'` will be replace by `'"'"'`, and the escaped string + # will be wrapped in a pair of `'`, then appended to `$args` string + # For example, `echo "It's only 1\$\!"` will be escaped to: + # 'echo' 'It'"'"'s only 1$!' + # │ │└┼┘│ + # │ │ │ └── `'s only 1$!'` the literal string + # │ │ └── `\"'\"` is a single quote (as a string) + # │ └── `'It'`, to be concatenated with the strings following it + # └── `echo` wrapped in a pair of `'`, it's totally fine for the shell command itself + while [ $# -ne 0 ]; do + args="$args'$(printf "%s" "$1" | sed -e "s/'/'\"'\"'/g")' " + shift + done + su root -c "$args" + } + SUDO=su_sudo + fi +else + SUDO= +fi + +ExperimentalBootstrap() { + # Arguments: Platform name, bootstrap function name + if [ "$DEBUG" = 1 ]; then + if [ "$2" != "" ]; then + echo "Bootstrapping dependencies via $1..." + $2 + fi + else + echo "WARNING: $1 support is very experimental at present..." + echo "if you would like to work on improving it, please ensure you have backups" + echo "and then run this script again with the --debug flag!" + exit 1 + fi +} + +DeterminePythonVersion() { + for LE_PYTHON in "$LE_PYTHON" python2.7 python27 python2 python; do + # Break (while keeping the LE_PYTHON value) if found. + command -v "$LE_PYTHON" > /dev/null && break + done + if [ "$?" != "0" ]; then + echo "Cannot find any Pythons; please install one!" + exit 1 + fi + export LE_PYTHON + + PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` + if [ "$PYVER" -lt 26 ]; then + echo "You have an ancient version of Python entombed in your operating system..." + echo "This isn't going to work; you'll need at least version 2.6." + exit 1 + fi +} + +BootstrapDebCommon() { + # Current version tested with: + # + # - Ubuntu + # - 14.04 (x64) + # - 15.04 (x64) + # - Debian + # - 7.9 "wheezy" (x64) + # - sid (2015-10-21) (x64) + + # Past versions tested with: + # + # - Debian 8.0 "jessie" (x64) + # - Raspbian 7.8 (armhf) + + # Believed not to work: + # + # - Debian 6.0.10 "squeeze" (x64) + + $SUDO apt-get update || echo apt-get update hit problems but continuing anyway... + + # virtualenv binary can be found in different packages depending on + # distro version (#346) + + virtualenv= + if apt-cache show virtualenv > /dev/null 2>&1 && ! apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then + virtualenv="virtualenv" + fi + + if apt-cache show python-virtualenv > /dev/null 2>&1; then + virtualenv="$virtualenv python-virtualenv" + fi + + augeas_pkg="libaugeas0 augeas-lenses" + AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + + AddBackportRepo() { + # ARGS: + BACKPORT_NAME="$1" + BACKPORT_SOURCELINE="$2" + echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then + # This can theoretically error if sources.list.d is empty, but in that case we don't care. + if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + add_backports=1 + else + read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response + case $response in + [yY][eE][sS]|[yY]|"") + add_backports=1;; + *) + add_backports=0;; + esac + fi + if [ "$add_backports" = 1 ]; then + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update + fi + fi + fi + if [ "$add_backports" != 0 ]; then + $SUDO apt-get install $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + fi + } + + + if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then + if lsb_release -a | grep -q wheezy ; then + AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" + elif lsb_release -a | grep -q precise ; then + # XXX add ARM case + AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" + else + echo "No libaugeas0 version is available that's new enough to run the" + echo "Certbot apache plugin..." + fi + # XXX add a case for ubuntu PPAs + fi + + $SUDO apt-get install $YES_FLAG --no-install-recommends \ + python \ + python-dev \ + $virtualenv \ + gcc \ + dialog \ + $augeas_pkg \ + libssl-dev \ + libffi-dev \ + ca-certificates \ + + + + if ! command -v virtualenv > /dev/null ; then + echo Failed to install a working \"virtualenv\" command, exiting + exit 1 + fi +} + +BootstrapRpmCommon() { + # Tested with: + # - Fedora 20, 21, 22, 23 (x64) + # - Centos 7 (x64: on DigitalOcean droplet) + # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) + + if type dnf 2>/dev/null + then + tool=dnf + elif type yum 2>/dev/null + then + tool=yum + + else + echo "Neither yum nor dnf found. Aborting bootstrap!" + exit 1 + fi + + pkgs=" + gcc + dialog + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + + # Some distros and older versions of current distros use a "python27" + # instead of "python" naming convention. Try both conventions. + if $SUDO $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python + python-devel + python-virtualenv + python-tools + python-pip + " + else + pkgs="$pkgs + python27 + python27-devel + python27-virtualenv + python27-tools + python27-pip + " + fi + + if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then + pkgs="$pkgs + mod_ssl + " + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + + if ! $SUDO $tool install $yes_flag $pkgs; then + echo "Could not install OS dependencies. Aborting bootstrap!" + exit 1 + fi +} + +BootstrapSuseCommon() { + # SLE12 don't have python-virtualenv + + if [ "$ASSUME_YES" = 1 ]; then + zypper_flags="-nq" + install_flags="-l" + fi + + $SUDO zypper $zypper_flags in $install_flags \ + python \ + python-devel \ + python-virtualenv \ + gcc \ + dialog \ + augeas-lenses \ + libopenssl-devel \ + libffi-devel \ + ca-certificates +} + +BootstrapArchCommon() { + # Tested with: + # - ArchLinux (x86_64) + # + # "python-virtualenv" is Python3, but "python2-virtualenv" provides + # only "virtualenv2" binary, not "virtualenv" necessary in + # ./tools/_venv_common.sh + + deps=" + python2 + python-virtualenv + gcc + dialog + augeas + openssl + libffi + ca-certificates + pkg-config + " + + # pacman -T exits with 127 if there are missing dependencies + missing=$($SUDO pacman -T $deps) || true + + if [ "$ASSUME_YES" = 1 ]; then + noconfirm="--noconfirm" + fi + + if [ "$missing" ]; then + $SUDO pacman -S --needed $missing $noconfirm + fi +} + +BootstrapGentooCommon() { + PACKAGES=" + dev-lang/python:2.7 + dev-python/virtualenv + dev-util/dialog + app-admin/augeas + dev-libs/openssl + dev-libs/libffi + app-misc/ca-certificates + virtual/pkgconfig" + + case "$PACKAGE_MANAGER" in + (paludis) + $SUDO cave resolve --preserve-world --keep-targets if-possible $PACKAGES -x + ;; + (pkgcore) + $SUDO pmerge --noreplace --oneshot $PACKAGES + ;; + (portage|*) + $SUDO emerge --noreplace --oneshot $PACKAGES + ;; + esac +} + +BootstrapFreeBsd() { + $SUDO pkg install -Ay \ + python \ + py27-virtualenv \ + augeas \ + libffi +} + +BootstrapMac() { + if hash brew 2>/dev/null; then + echo "Using Homebrew to install dependencies..." + pkgman=brew + pkgcmd="brew install" + elif hash port 2>/dev/null; then + echo "Using MacPorts to install dependencies..." + pkgman=port + pkgcmd="$SUDO port install" + else + echo "No Homebrew/MacPorts; installing Homebrew..." + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + pkgman=brew + pkgcmd="brew install" + fi + + $pkgcmd augeas + $pkgcmd dialog + if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ + -o "$(which python)" = "/usr/bin/python" ]; then + # We want to avoid using the system Python because it requires root to use pip. + # python.org, MacPorts or HomeBrew Python installations should all be OK. + echo "Installing python..." + $pkgcmd python + fi + + # Workaround for _dlopen not finding augeas on OS X + if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then + echo "Applying augeas workaround" + $SUDO mkdir -p /usr/local/lib/ + $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ + fi + + if ! hash pip 2>/dev/null; then + echo "pip not installed" + echo "Installing pip..." + curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python + fi + + if ! hash virtualenv 2>/dev/null; then + echo "virtualenv not installed." + echo "Installing with pip..." + pip install virtualenv + fi +} + +BootstrapSmartOS() { + pkgin update + pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' +} + +BootstrapMageiaCommon() { + if ! $SUDO urpmi --force \ + python \ + libpython-devel \ + python-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + + if ! $SUDO urpmi --force \ + git \ + gcc \ + cdialog \ + python-augeas \ + libopenssl-devel \ + libffi-devel \ + rootcerts + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} + + +# Install required OS packages: +Bootstrap() { + if [ -f /etc/debian_version ]; then + echo "Bootstrapping dependencies for Debian-based OSes..." + BootstrapDebCommon + elif [ -f /etc/mageia-release ] ; then + # Mageia has both /etc/mageia-release and /etc/redhat-release + ExperimentalBootstrap "Mageia" BootstrapMageiaCommon + elif [ -f /etc/redhat-release ]; then + echo "Bootstrapping dependencies for RedHat-based OSes..." + BootstrapRpmCommon + elif [ -f /etc/os-release ] && `grep -q openSUSE /etc/os-release` ; then + echo "Bootstrapping dependencies for openSUSE-based OSes..." + BootstrapSuseCommon + elif [ -f /etc/arch-release ]; then + if [ "$DEBUG" = 1 ]; then + echo "Bootstrapping dependencies for Archlinux..." + BootstrapArchCommon + else + echo "Please use pacman to install letsencrypt packages:" + echo "# pacman -S certbot certbot-apache" + echo + echo "If you would like to use the virtualenv way, please run the script again with the" + echo "--debug flag." + exit 1 + fi + elif [ -f /etc/manjaro-release ]; then + ExperimentalBootstrap "Manjaro Linux" BootstrapArchCommon + elif [ -f /etc/gentoo-release ]; then + ExperimentalBootstrap "Gentoo" BootstrapGentooCommon + elif uname | grep -iq FreeBSD ; then + ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd + elif uname | grep -iq Darwin ; then + ExperimentalBootstrap "Mac OS X" BootstrapMac + elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then + ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then + ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS + else + echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" + echo + echo "You will need to bootstrap, configure virtualenv, and run pip install manually." + echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + echo "for more info." + exit 1 + fi +} + +TempDir() { + mktemp -d 2>/dev/null || mktemp -d -t 'le' # Linux || OS X +} + + + +if [ "$1" = "--le-auto-phase2" ]; then + # Phase 2: Create venv, install LE, and run. + + shift 1 # the --le-auto-phase2 arg + if [ -f "$VENV_BIN/letsencrypt" ]; then + # --version output ran through grep due to python-cryptography DeprecationWarnings + # grep for both certbot and letsencrypt until certbot and shim packages have been released + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) + else + INSTALLED_VERSION="none" + fi + if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then + echo "Creating virtual environment..." + DeterminePythonVersion + rm -rf "$VENV_PATH" + if [ "$VERBOSE" = 1 ]; then + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" + else + virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null + fi + + echo "Installing Python packages..." + TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT + # There is no $ interpolation due to quotes on starting heredoc delimiter. + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" +# This is the flattened list of packages certbot-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e certbot-apache`, and +# then use `hashin` or a more secure method to gather the hashes. + +argparse==1.4.0 \ + --hash=sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314 \ + --hash=sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4 + +# This comes before cffi because cffi will otherwise install an unchecked +# version via setup_requires. +pycparser==2.14 \ + --hash=sha256:7959b4a74abdc27b312fed1c21e6caf9309ce0b29ea86b591fd2e99ecdf27f73 + +cffi==1.4.2 \ + --hash=sha256:53c1c9ddb30431513eb7f3cdef0a3e06b0f1252188aaa7744af0f5a4cd45dbaf \ + --hash=sha256:a568f49dfca12a8d9f370187257efc58a38109e1eee714d928561d7a018a64f8 \ + --hash=sha256:809c6ca8cfbcaeebfbd432b4576001b40d38ff2463773cb57577d75e1a020bc3 \ + --hash=sha256:86cdca2cd9cba41422230390df17dfeaa9f344a911e3975c8be9da57b35548e9 \ + --hash=sha256:24b13db84aec385ca23c7b8ded83ef8bb4177bc181d14758f9f975be5d020d86 \ + --hash=sha256:969aeffd7c0e097f6be1efd682c156ae226591a0793a94b6c2d5e4293f4c8d4e \ + --hash=sha256:000f358d4b0fa249feaab9c1ce7d5b2fe7e02e7bdf6806c26418505fc685e268 \ + --hash=sha256:a9d86f460bbd8358a2d513ad779e3f3fc878e3b93a00b5002faebf616ffe6b9c \ + --hash=sha256:3127b3ab33eb23ccac071f9a0802748e5cf7c5cbcd02482bb063e35b41dbb0b0 \ + --hash=sha256:e2b2d42236469a40224d39e7b6c60575f388b2f423f354c7ee90a5b7f58c8065 \ + --hash=sha256:8c2dccafee89b1b424b0bec6ad2dd9622c949d2024e929f5da1ed801eac75f1d \ + --hash=sha256:a4de7a4d11aed488bab4fb14f4988587a829bece5a20433f780d6e33b08083cb \ + --hash=sha256:5ca8fe30425265a49274e4b0213a1bc98f4b13449ae5e96f984771e5d83e58c1 \ + --hash=sha256:a4fd38802f59e714eba81a024f62db710b27dbe27a7ea12e911537327aa84d30 \ + --hash=sha256:86cd6912bbc83e9405d4a73cd7f4b4ee8353652d2dbc7c820106ed5b4d1bab3a \ + --hash=sha256:8f1d177d364ea35900415ae24ca3e471be3d5334ed0419294068c49f45913998 +ConfigArgParse==0.10.0 \ + --hash=sha256:3b50a83dd58149dfcee98cb6565265d10b53e9c0a2bca7eeef7fb5f5524890a7 +configobj==5.0.6 \ + --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 +cryptography==1.2.3 \ + --hash=sha256:031938f73a5c5eb3e809e18ff7caeb6865351871417be6050cb8c86a9a202b9a \ + --hash=sha256:a179a38d50f8d68b491d7a313db78f8cabe290842cecddddc7b34d408e59db0a \ + --hash=sha256:906c88b2aadcf99cfabb24098263d1bf65ab0c8688acde10dae1f09d865920f1 \ + --hash=sha256:6e706c5c6088770b1d1b634e959e21963e315b0255f5f4777125ad3d54082977 \ + --hash=sha256:f5ebf8e31c48f8707921dca0e994de77813a9c9b9bf03c119c5ddf97bdcffe73 \ + --hash=sha256:c7b89e42288cc7fbee3812e99ef5c744f22452e11d6822f6807afc6d6b3be83e \ + --hash=sha256:8408d29865947109d8b68f1837a7cde1aa4dc86e0f79ca3ba58c0c44e443d6a5 \ + --hash=sha256:c7e76cf3c3d925dd31fa238cfb806cffba718c0f08707d77a538768477969956 \ + --hash=sha256:7d8de35380f31702758b7753bb5c40723832c73006dedb2f9099bf61a37f7287 \ + --hash=sha256:5edbee71fae5469ee83fe0a37866b9398c8ce3a46325c24fcedfbf097bb48a19 \ + --hash=sha256:594edafe4801c13bdc1cc305e7704a90c19617e95936f6ab457ee4ffe000ba50 \ + --hash=sha256:b7fdb16a0a7f481be42da744bfe1ea2163025de21f90f2c688a316f3c354da9c \ + --hash=sha256:207b8bf0fe0907336df38b733b487521cf9e138189aba9234ad54fe545dd0db8 \ + --hash=sha256:509a2f05386270cf783993c90d49ffefb3dd62aee45bf1ea8ce3d2cde7271c21 \ + --hash=sha256:ac69b65dd1af0179ede40c9f15788c88f73e628ea6c0519de3838e279bb388c6 \ + --hash=sha256:8df6fad6c6ae12fd7004ea29357f0a2b4d3774eaeca7656530d08d2d90cd41aa \ + --hash=sha256:0b8b96dd81cc1533a04f30382c0fe21c1972e189f794d0c4261a18cec08fd9b5 \ + --hash=sha256:cae8fca1883f23c50ea78d89de6fe4fefdb4cea83177760f47177559414ded93 \ + --hash=sha256:1a471ca576a9cdce1b1cd9f3a22b1d09ee44d46862037557de17919c0db44425 \ + --hash=sha256:8ec4e8e3d453b3a1b63b5f57737a434dcf1ee4a2f26f6ff7c5a37c3f679104d2 \ + --hash=sha256:8eb11c77dd8e73f48df6b2f7a7e16173fe0fe8fdfe266232832e88477e08454e +enum34==1.1.2 \ + --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ + --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 +funcsigs==0.4 \ + --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ + --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 +idna==2.0 \ + --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \ + --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b +ipaddress==1.0.16 \ + --hash=sha256:935712800ce4760701d89ad677666cd52691fd2f6f0b340c8b4239a3c17988a5 \ + --hash=sha256:5a3182b322a706525c46282ca6f064d27a02cffbd449f9f47416f1dc96aa71b0 +linecache2==1.0.0 \ + --hash=sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef \ + --hash=sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c +ndg-httpsclient==0.4.0 \ + --hash=sha256:e8c155fdebd9c4bcb0810b4ed01ae1987554b1ee034dd7532d7b8fdae38a6274 +ordereddict==1.1 \ + --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f +parsedatetime==2.1 \ + --hash=sha256:ce9d422165cf6e963905cd5f74f274ebf7cc98c941916169178ef93f0e557838 \ + --hash=sha256:17c578775520c99131634e09cfca5a05ea9e1bd2a05cd06967ebece10df7af2d +pbr==1.8.1 \ + --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ + --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 +psutil==3.3.0 \ + --hash=sha256:584f0b29fcc5d523b433cb8918b2fc74d67e30ee0b44a95baf031528f424619f \ + --hash=sha256:28ca0b6e9d99aa8dc286e8747a4471362b69812a25291de29b6a8d70a1545a0d \ + --hash=sha256:167ad5fff52a672c4ddc1c1a0b25146d6813ebb08a9aab0a3ac45f8a5b669c3b \ + --hash=sha256:e6dea6173a988727bb223d3497349ad5cdef5c0b282eff2d83e5f9065c53f85f \ + --hash=sha256:2af5e0a4aad66049955d0734aa4e3dc8caa17a9eaf8b4c1a27a5f1ee6e40f6fc \ + --hash=sha256:d9884dc0dc2e55e2448e495778dc9899c1c8bf37aeb2f434c1bea74af93c2683 \ + --hash=sha256:e27c2fe6dfcc8738be3d2c5a022f785eb72971057e1a9e1e34fba73bce8a71a6 \ + --hash=sha256:65afd6fecc8f3aed09ee4be63583bc8eb472f06ceaa4fe24c4d1d5a1a3c0e13f \ + --hash=sha256:ba1c558fbfcdf94515c2394b1155c1dc56e2bc2a9c17d30349827c9ed8a67e46 \ + --hash=sha256:ba95ea0022dcb64d36f0c1335c0605fae35bdf3e0fea8d92f5d0f6456a35e55b \ + --hash=sha256:421b6591d16b509aaa8d8c15821d66bb94cb4a8dc4385cad5c51b85d4a096d85 \ + --hash=sha256:326b305cbdb6f94dafbfe2c26b11da88b0ab07b8a07f8188ab9d75ff0c6e841a \ + --hash=sha256:9aede5b2b6fe46b3748ea8e5214443890d1634027bef3d33b7dad16556830278 \ + --hash=sha256:73bed1db894d1aa9c3c7e611d302cdeab7ae8a0dc0eeaf76727878db1ac5cd87 \ + --hash=sha256:935b5dd6d558af512f42501a7c08f41d7aff139af1bb3959daa3abb859234d6c \ + --hash=sha256:4ca0111cf157dcc0f2f69a323c5b5478718d68d45fc9435d84be0ec0f186215b \ + --hash=sha256:b6f13c95398a3fcf0226c4dcfa448560ba5865259cd96ec2810658651e932189 \ + --hash=sha256:ee6be30d1635bbdea4c4325d507dc8a0dbbde7e1c198bd62ddb9f43198b9e214 \ + --hash=sha256:dfa786858c268d7fbbe1b6175e001ec02738d7cfae0a7ce77bf9b651af676729 \ + --hash=sha256:aa77f9de72af9c16cc288cd4a24cf58824388f57d7a81e400c4616457629870e \ + --hash=sha256:f500093357d04da8140d87932cac2e54ef592a54ca8a743abb2850f60c2c22eb +pyasn1==0.1.9 \ + --hash=sha256:61f9d99e3cef65feb1bfe3a2eef7a93eb93819d345bf54bcd42f4e63d5204dae \ + --hash=sha256:1802a6dd32045e472a419db1441aecab469d33e0d2749e192abdec52101724af \ + --hash=sha256:35025cd9422c96504912f04e2f15fe79390a8597b430c2ca5d0534cf9309ffa0 \ + --hash=sha256:2f96ed5a0c329ca16230b326ca12b7461ec8f65e0be3e4f997516f36bf82a345 \ + --hash=sha256:28fee44217991cfad9e6a0b9f7e3f26041e21ebc96629e94e585ccd05d49fa65 \ + --hash=sha256:326e7a854a17fab07691204747695f8f692d674588a355c441fb14f660bf4e68 \ + --hash=sha256:cda5a90485709ca6795c86056c3e5fe7266028b05e53f1d527fdf93a6365a6b8 \ + --hash=sha256:0cb2a14742b543fdd68f931a14ce3829186ed2b1b2267a06787388c96b2dd9be \ + --hash=sha256:5191ff6b9126d2c039dd87f8ff025bed274baf07fa78afa46f556b1ad7265d6e \ + --hash=sha256:8323e03637b2d072cc7041300bac6ec448c3c28950ab40376036788e9a1af629 \ + --hash=sha256:853cacd96d1f701ddd67aa03ecc05f51890135b7262e922710112f12a2ed2a7f +pyOpenSSL==0.15.1 \ + --hash=sha256:88e45e6bb25dfed272a1ef2e728461d44b634c2cd689e989b6e56a349c5a3ae5 \ + --hash=sha256:f0a26070d6db0881de8bcc7846934b7c3c930d8f9c79d45883ee48984bc0d672 +pyRFC3339==1.0 \ + --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ + --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 +python-augeas==0.5.0 \ + --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 +python2-pythondialog==3.3.0 \ + --hash=sha256:04e93f24995c43dd90f338d5d865ca72ce3fb5a5358d4daa4965571db35fc3ec \ + --hash=sha256:3e6f593fead98f8a526bc3e306933533236e33729f552f52896ea504f55313fa +pytz==2015.7 \ + --hash=sha256:3abe6a6d3fc2fbbe4c60144211f45da2edbe3182a6f6511af6bbba0598b1f992 \ + --hash=sha256:939ef9c1e1224d980405689a97ffcf7828c56d1517b31d73464356c1f2b7769e \ + --hash=sha256:ead4aefa7007249e05e51b01095719d5a8dd95760089f5730aac5698b1932918 \ + --hash=sha256:3cca0df08bd0ed98432390494ce3ded003f5e661aa460be7a734bffe35983605 \ + --hash=sha256:3ede470d3d17ba3c07638dfa0d10452bc1b6e5ad326127a65ba77e6aaeb11bec \ + --hash=sha256:68c47964f7186eec306b13629627722b9079cd4447ed9e5ecaecd4eac84ca734 \ + --hash=sha256:dd5d3991950aae40a6c81de1578942e73d629808cefc51d12cd157980e6cfc18 \ + --hash=sha256:a77c52062c07eb7c7b30545dbc73e32995b7e117eea750317b5cb5c7a4618f14 \ + --hash=sha256:81af9aec4bc960a9a0127c488f18772dae4634689233f06f65443e7b11ebeb51 \ + --hash=sha256:e079b1dadc5c06246cc1bb6fe1b23a50b1d1173f2edd5104efd40bb73a28f406 \ + --hash=sha256:fbd26746772c24cb93c8b97cbdad5cb9e46c86bbdb1b9d8a743ee00e2fb1fc5d \ + --hash=sha256:99266ef30a37e43932deec2b7ca73e83c8dbc3b9ff703ec73eca6b1dae6befea \ + --hash=sha256:8b6ce1c993909783bc96e0b4f34ea223bff7a4df2c90bdb9c4e0f1ac928689e3 +requests==2.9.1 \ + --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \ + --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f +six==1.10.0 \ + --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ + --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a +traceback2==1.4.0 \ + --hash=sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23 \ + --hash=sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030 +unittest2==1.1.0 \ + --hash=sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8 \ + --hash=sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579 +zope.component==4.2.2 \ + --hash=sha256:282c112b55dd8e3c869a3571f86767c150ab1284a9ace2bdec226c592acaf81a +zope.event==4.1.0 \ + --hash=sha256:dc7a59a2fd91730d3793131a5d261b29e93ec4e2a97f1bc487ce8defee2fe786 +zope.interface==4.1.3 \ + --hash=sha256:f07b631f7a601cd8cbd3332d54f43142c7088a83299f859356f08d1d4d4259b3 \ + --hash=sha256:de5cca083b9439d8002fb76bbe6b4998c5a5a721fab25b84298967f002df4c94 \ + --hash=sha256:6788416f7ea7f5b8a97be94825377aa25e8bdc73463e07baaf9858b29e737077 \ + --hash=sha256:6f3230f7254518201e5a3708cbb2de98c848304f06e3ded8bfb39e5825cba2e1 \ + --hash=sha256:5fa575a5240f04200c3088427d0d4b7b737f6e9018818a51d8d0f927a6a2517a \ + --hash=sha256:522194ad6a545735edd75c8a83f48d65d1af064e432a7d320d64f56bafc12e99 \ + --hash=sha256:e8c7b2d40943f71c99148c97f66caa7f5134147f57423f8db5b4825099ce9a09 \ + --hash=sha256:279024f0208601c3caa907c53876e37ad88625f7eaf1cb3842dbe360b2287017 \ + --hash=sha256:2e221a9eec7ccc58889a278ea13dcfed5ef939d80b07819a9a8b3cb1c681484f \ + --hash=sha256:69118965410ec86d44dc6b9017ee3ddbd582e0c0abeef62b3a19dbf6c8ad132b \ + --hash=sha256:d04df8686ec864d0cade8cf199f7f83aecd416109a20834d568f8310ded12dea \ + --hash=sha256:e75a947e15ee97e7e71e02ea302feb2fc62d3a2bb4668bf9dfbed43a506ac7e7 \ + --hash=sha256:4e45d22fb883222a5ab9f282a116fec5ee2e8d1a568ccff6a2d75bbd0eb6bcfc \ + --hash=sha256:bce9339bb3c7a55e0803b63d21c5839e8e479bc85c4adf42ae415b72f94facb2 \ + --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ + --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ + --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 +mock==1.0.1 \ + --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ + --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 + +# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. + +acme==0.8.1 \ + --hash=sha256:ccd7883772efbf933f91713b8241455993834f3620c8fbd459d9ed5e50bbaaca \ + --hash=sha256:d3ea4acf280bf6253ad7d641cb0970f230a19805acfed809e7a8ddcf62157d9f +certbot==0.8.1 \ + --hash=sha256:89805d9f70249ae859ec4d7a99c00b4bb7083ca90cd12d4d202b76dfc284f7c5 \ + --hash=sha256:6ca8df3d310ced6687d38aac17c0fb8c1b2ec7a3bea156a254e4cc2a1c132771 +certbot-apache==0.8.1 \ + --hash=sha256:c9e3fdc15e65589c2e39eb0e6b1f61f0c0a1db3c17b00bb337f0ff636cc61cb3 \ + --hash=sha256:0faf2879884d3b7a58b071902fba37d4b8b58a50e2c3b8ac262c0a74134045ed + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" +#!/usr/bin/env python +"""A small script that can act as a trust root for installing pip 8 + +Embed this in your project, and your VCS checkout is all you have to trust. In +a post-peep era, this lets you claw your way to a hash-checking version of pip, +with which you can install the rest of your dependencies safely. All it assumes +is Python 2.6 or better and *some* version of pip already installed. If +anything goes wrong, it will exit with a non-zero status code. + +""" +# This is here so embedded copies are MIT-compliant: +# Copyright (c) 2016 Erik Rose +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +from __future__ import print_function +from hashlib import sha256 +from os.path import join +from pipes import quote +from shutil import rmtree +try: + from subprocess import check_output +except ImportError: + from subprocess import CalledProcessError, PIPE, Popen + + def check_output(*popenargs, **kwargs): + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be ' + 'overridden.') + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) + return output +from sys import exit, version_info +from tempfile import mkdtemp +try: + from urllib2 import build_opener, HTTPHandler, HTTPSHandler +except ImportError: + from urllib.request import build_opener, HTTPHandler, HTTPSHandler +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # 3.4 + + +__version__ = 1, 1, 1 + + +# wheel has a conditional dependency on argparse: +maybe_argparse = ( + [('https://pypi.python.org/packages/source/a/argparse/' + 'argparse-1.4.0.tar.gz', + '62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4')] + if version_info < (2, 7, 0) else []) + + +PACKAGES = maybe_argparse + [ + # Pip has no dependencies, as it vendors everything: + ('https://pypi.python.org/packages/source/p/pip/pip-8.0.3.tar.gz', + '30f98b66f3fe1069c529a491597d34a1c224a68640c82caf2ade5f88aa1405e8'), + # This version of setuptools has only optional dependencies: + ('https://pypi.python.org/packages/source/s/setuptools/' + 'setuptools-20.2.2.tar.gz', + '24fcfc15364a9fe09a220f37d2dcedc849795e3de3e4b393ee988e66a9cbd85a'), + ('https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz', + '1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648') +] + + +class HashError(Exception): + def __str__(self): + url, path, actual, expected = self.args + return ('{url} did not match the expected hash {expected}. Instead, ' + 'it was {actual}. The file (left at {path}) may have been ' + 'tampered with.'.format(**locals())) + + +def hashed_download(url, temp, digest): + """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``, + and return its path.""" + # Based on pip 1.4.1's URLOpener but with cert verification removed. Python + # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert + # authenticity has only privacy (not arbitrary code execution) + # implications, since we're checking hashes. + def opener(): + opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in opener.handlers: + if isinstance(handler, HTTPHandler): + opener.handlers.remove(handler) + return opener + + def read_chunks(response, chunk_size): + while True: + chunk = response.read(chunk_size) + if not chunk: + break + yield chunk + + response = opener().open(url) + path = join(temp, urlparse(url).path.split('/')[-1]) + actual_hash = sha256() + with open(path, 'wb') as file: + for chunk in read_chunks(response, 4096): + file.write(chunk) + actual_hash.update(chunk) + + actual_digest = actual_hash.hexdigest() + if actual_digest != digest: + raise HashError(url, path, actual_digest, digest) + return path + + +def main(): + temp = mkdtemp(prefix='pipstrap-') + try: + downloads = [hashed_download(url, temp, digest) + for url, digest in PACKAGES] + check_output('pip install --no-index --no-deps -U ' + + ' '.join(quote(d) for d in downloads), + shell=True) + except HashError as exc: + print(exc) + except Exception: + rmtree(temp) + raise + else: + rmtree(temp) + return 0 + return 1 + + +if __name__ == '__main__': + exit(main()) + +UNLIKELY_EOF + # ------------------------------------------------------------------------- + # Set PATH so pipstrap upgrades the right (v)env: + PATH="$VENV_BIN:$PATH" "$VENV_BIN/python" "$TEMP_DIR/pipstrap.py" + set +e + PIP_OUT=`"$VENV_BIN/pip" install --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` + PIP_STATUS=$? + set -e + if [ "$PIP_STATUS" != 0 ]; then + # Report error. (Otherwise, be quiet.) + echo "Had a problem while installing Python packages:" + echo "$PIP_OUT" + rm -rf "$VENV_PATH" + exit 1 + fi + echo "Installation succeeded." + fi + if [ -n "$SUDO" ]; then + # SUDO is su wrapper or sudo + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" + fi + if [ -z "$SUDO_ENV" ] ; then + # SUDO is su wrapper / noop + $SUDO "$VENV_BIN/letsencrypt" "$@" + else + # sudo + $SUDO "$SUDO_ENV" "$VENV_BIN/letsencrypt" "$@" + fi + +else + # Phase 1: Upgrade certbot-auto if neceesary, then self-invoke. + # + # Each phase checks the version of only the thing it is responsible for + # upgrading. Phase 1 checks the version of the latest release of + # certbot-auto (which is always the same as that of the certbot + # package). Phase 2 checks the version of the locally installed certbot. + + if [ ! -f "$VENV_BIN/letsencrypt" ]; then + if [ "$HELP" = 1 ]; then + echo "$USAGE" + exit 0 + fi + # If it looks like we've never bootstrapped before, bootstrap: + Bootstrap + fi + if [ "$OS_PACKAGES_ONLY" = 1 ]; then + echo "OS packages installed." + exit 0 + fi + + if [ "$NO_SELF_UPGRADE" != 1 ]; then + TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT + # --------------------------------------------------------------------------- + cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" +"""Do downloading and JSON parsing without additional dependencies. :: + + # Print latest released version of LE to stdout: + python fetch.py --latest-version + + # Download letsencrypt-auto script from git tag v1.2.3 into the folder I'm + # in, and make sure its signature verifies: + python fetch.py --le-auto-script v1.2.3 + +On failure, return non-zero. + +""" +from distutils.version import LooseVersion +from json import loads +from os import devnull, environ +from os.path import dirname, join +import re +from subprocess import check_call, CalledProcessError +from sys import argv, exit +from urllib2 import build_opener, HTTPHandler, HTTPSHandler, HTTPError + +PUBLIC_KEY = environ.get('LE_AUTO_PUBLIC_KEY', """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6MR8W/galdxnpGqBsYbq +OzQb2eyW15YFjDDEMI0ZOzt8f504obNs920lDnpPD2/KqgsfjOgw2K7xWDJIj/18 +xUvWPk3LDkrnokNiRkA3KOx3W6fHycKL+zID7zy+xZYBuh2fLyQtWV1VGQ45iNRp +9+Zo7rH86cdfgkdnWTlNSHyTLW9NbXvyv/E12bppPcEvgCTAQXgnDVJ0/sqmeiij +n9tTFh03aM+R2V/21h8aTraAS24qiPCz6gkmYGC8yr6mglcnNoYbsLNYZ69zF1XH +cXPduCPdPdfLlzVlKK1/U7hkA28eG3BIAMh6uJYBRJTpiGgaGdPd7YekUB8S6cy+ +CQIDAQAB +-----END PUBLIC KEY----- +""") + +class ExpectedError(Exception): + """A novice-readable exception that also carries the original exception for + debugging""" + + +class HttpsGetter(object): + def __init__(self): + """Build an HTTPS opener.""" + # Based on pip 1.4.1's URLOpener + # This verifies certs on only Python >=2.7.9. + self._opener = build_opener(HTTPSHandler()) + # Strip out HTTPHandler to prevent MITM spoof: + for handler in self._opener.handlers: + if isinstance(handler, HTTPHandler): + self._opener.handlers.remove(handler) + + def get(self, url): + """Return the document contents pointed to by an HTTPS URL. + + If something goes wrong (404, timeout, etc.), raise ExpectedError. + + """ + try: + return self._opener.open(url).read() + except (HTTPError, IOError) as exc: + raise ExpectedError("Couldn't download %s." % url, exc) + + +def write(contents, dir, filename): + """Write something to a file in a certain directory.""" + with open(join(dir, filename), 'w') as file: + file.write(contents) + + +def latest_stable_version(get): + """Return the latest stable release of letsencrypt.""" + metadata = loads(get( + environ.get('LE_AUTO_JSON_URL', + 'https://pypi.python.org/pypi/certbot/json'))) + # metadata['info']['version'] actually returns the latest of any kind of + # release release, contrary to https://wiki.python.org/moin/PyPIJSON. + # The regex is a sufficient regex for picking out prereleases for most + # packages, LE included. + return str(max(LooseVersion(r) for r + in metadata['releases'].iterkeys() + if re.match('^[0-9.]+$', r))) + + +def verified_new_le_auto(get, tag, temp_dir): + """Return the path to a verified, up-to-date letsencrypt-auto script. + + If the download's signature does not verify or something else goes wrong + with the verification process, raise ExpectedError. + + """ + le_auto_dir = environ.get( + 'LE_AUTO_DIR_TEMPLATE', + 'https://raw.githubusercontent.com/certbot/certbot/%s/' + 'letsencrypt-auto-source/') % tag + write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') + write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') + write(PUBLIC_KEY, temp_dir, 'public_key.pem') + try: + with open(devnull, 'w') as dev_null: + check_call(['openssl', 'dgst', '-sha256', '-verify', + join(temp_dir, 'public_key.pem'), + '-signature', + join(temp_dir, 'letsencrypt-auto.sig'), + join(temp_dir, 'letsencrypt-auto')], + stdout=dev_null, + stderr=dev_null) + except CalledProcessError as exc: + raise ExpectedError("Couldn't verify signature of downloaded " + "certbot-auto.", exc) + + +def main(): + get = HttpsGetter().get + flag = argv[1] + try: + if flag == '--latest-version': + print latest_stable_version(get) + elif flag == '--le-auto-script': + tag = argv[2] + verified_new_le_auto(get, tag, dirname(argv[0])) + except ExpectedError as exc: + print exc.args[0], exc.args[1] + return 1 + else: + return 0 + + +if __name__ == '__main__': + exit(main()) + +UNLIKELY_EOF + # --------------------------------------------------------------------------- + DeterminePythonVersion + if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + echo "WARNING: unable to check for updates." + elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + + # Now we drop into Python so we don't have to install even more + # dependencies (curl, etc.), for better flow control, and for the option of + # future Windows compatibility. + "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" + + # Install new copy of certbot-auto. + # TODO: Deal with quotes in pathnames. + echo "Replacing certbot-auto..." + # Clone permissions with cp. chmod and chown don't have a --reference + # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: + $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" + $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" + # Using mv rather than cp leaves the old file descriptor pointing to the + # original copy so the shell can continue to read it unmolested. mv across + # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the + # cp is unlikely to fail (esp. under sudo) if the rm doesn't. + $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" + fi # A newer version is available. + fi # Self-upgrading is allowed. + + "$0" --le-auto-phase2 "$@" +fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh deleted file mode 100755 index ca96e216f..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2dismod.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for disabling modules in the -# httpd docker image. First argument is the server_root and the second is the -# module to be disabled. - -apache_confdir=$1 -module=$2 - -sed -i "/.*"$module".*/d" "$apache_confdir/test.conf" -enabled_conf="$apache_confdir/mods-enabled/"$module".conf" -if [ -e "$enabled_conf" ] -then - rm $enabled_conf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh deleted file mode 100755 index 4da9288a2..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/a2enmod.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# An extremely simplified version of `a2enmod` for enabling modules in the -# httpd docker image. First argument is the Apache ServerRoot which should be -# an absolute path. The second is the module to be enabled, such as `ssl`. - -confdir=$1 -module=$2 - -echo "LoadModule ${module}_module " \ - "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf" -availbase="/mods-available/${module}.conf" -availconf=$confdir$availbase -enabldir="$confdir/mods-enabled" -enablconf="$enabldir/${module}.conf" -if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ] -then - ln -s "..$availbase" $enablconf -fi diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py deleted file mode 100644 index 927c329ef..000000000 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/apache24.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Proxies ApacheConfigurator for Apache 2.4 tests""" - -import zope.interface - -from certbot_compatibility_test import errors -from certbot_compatibility_test import interfaces -from certbot_compatibility_test.configurators.apache import common as apache_common - - -# The docker image doesn't actually have the watchdog module, but unless the -# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and -# therefore the config won't be loaded), I believe this isn't a problem -# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html -STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog"]) - - -SHARED_MODULES = { - "log_config", "logio", "version", "unixd", "access_compat", "actions", - "alias", "allowmethods", "auth_basic", "auth_digest", "auth_form", - "authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file", - "authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm", - "authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex", - "buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs", - "dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter", - "file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness", - "lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap", - "log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp", - "proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi", - "proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit", - "remoteip", "reqtimeout", "request", "rewrite", "sed", "session", - "session_cookie", "session_crypto", "session_dbd", "setenvif", - "slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb", - "speling", "ssl", "status", "substitute", "unique_id", "userdir", - "vhost_alias"} - - -@zope.interface.implementer(interfaces.IConfiguratorProxy) -class Proxy(apache_common.Proxy): - """Wraps the ApacheConfigurator for Apache 2.4 tests""" - - def __init__(self, args): - """Initializes the plugin with the given command line args""" - super(Proxy, self).__init__(args) - # Running init isn't ideal, but the Docker container needs to survive - # Apache restarts - self.start_docker("bradmw/apache2.4", "init") - - def preprocess_config(self, server_root): - """Prepares the configuration for use in the Docker""" - super(Proxy, self).preprocess_config(server_root) - if self.version[1] != 4: - raise errors.Error("Apache version not 2.4") - - with open(self.test_conf, "a") as f: - for module in self.modules: - if module not in STATIC_MODULES: - if module in SHARED_MODULES: - f.write( - "LoadModule {0}_module /usr/local/apache2/modules/" - "mod_{0}.so\n".format(module)) - else: - raise errors.Error( - "Unsupported module {0}".format(module)) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index f57e0512d..ed3d9d67a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -1,6 +1,7 @@ """Provides a common base for Apache proxies""" import re import os +import shutil import subprocess import mock @@ -9,6 +10,7 @@ import zope.interface from certbot import configuration from certbot import errors as le_errors from certbot_apache import configurator +from certbot_apache import constants from certbot_compatibility_test import errors from certbot_compatibility_test import interfaces from certbot_compatibility_test import util @@ -29,58 +31,14 @@ class Proxy(configurators_common.Proxy): super(Proxy, self).__init__(args) self.le_config.apache_le_vhost_ext = "-le-ssl.conf" - self._setup_mock() - self.modules = self.server_root = self.test_conf = self.version = None self._apache_configurator = self._all_names = self._test_names = None - - def _setup_mock(self): - """Replaces specific modules with mock.MagicMock""" - mock_subprocess = mock.MagicMock() - mock_subprocess.check_call = self.check_call - mock_subprocess.Popen = self.popen - - mock.patch( - "certbot_apache.configurator.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.parser.subprocess", - mock_subprocess).start() - mock.patch( - "certbot.le_util.subprocess", - mock_subprocess).start() - mock.patch( - "certbot_apache.configurator.le_util.exe_exists", - _is_apache_command).start() - patch = mock.patch( "certbot_apache.configurator.display_ops.select_vhost") mock_display = patch.start() mock_display.side_effect = le_errors.PluginError( "Unable to determine vhost") - def check_call(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.check_call is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).check_call(command, *args, **kwargs) - else: - return subprocess.check_call(command, *args, **kwargs) - - def popen(self, command, *args, **kwargs): - """If command is an Apache command, command is executed in the - running docker image. Otherwise, subprocess.Popen is used. - - """ - if _is_apache_command(command): - command = _modify_command(command) - return super(Proxy, self).popen(command, *args, **kwargs) - else: - return subprocess.Popen(command, *args, **kwargs) - def __getattr__(self, name): """Wraps the Apache Configurator methods""" method = getattr(self._apache_configurator, name, None) @@ -91,29 +49,20 @@ class Proxy(configurators_common.Proxy): def load_config(self): """Loads the next configuration for the plugin to test""" - if hasattr(self.le_config, "apache_init_script"): - try: - self.check_call([self.le_config.apache_init_script, "stop"]) - except errors.Error: - raise errors.Error( - "Failed to stop previous apache config from running") config = super(Proxy, self).load_config() - self.modules = _get_modules(config) - self.version = _get_version(config) self._all_names, self._test_names = _get_names(config) server_root = _get_server_root(config) - with open(os.path.join(config, "config_file")) as f: - config_file = os.path.join(server_root, f.readline().rstrip()) - self.test_conf = _create_test_conf(server_root, config_file) + # with open(os.path.join(config, "config_file")) as f: + # config_file = os.path.join(server_root, f.readline().rstrip()) + shutil.rmtree("/etc/apache2") + shutil.copytree(server_root, "/etc/apache2", symlinks=True) - self.preprocess_config(server_root) - self._prepare_configurator(server_root, config_file) + self._prepare_configurator() try: - self.check_call("apachectl -d {0} -f {1} -k start".format( - server_root, config_file)) + subprocess.check_call("apachectl -k start".split()) except errors.Error: raise errors.Error( "Apache failed to load {0} before tests started".format( @@ -121,34 +70,13 @@ class Proxy(configurators_common.Proxy): return config - def preprocess_config(self, server_root): - # pylint: disable=anomalous-backslash-in-string, no-self-use - """Prepares the configuration for use in the Docker""" - - find = subprocess.Popen( - ["find", server_root, "-type", "f"], - stdout=subprocess.PIPE) - subprocess.check_call([ - "xargs", "sed", "-e", "s/DocumentRoot.*/DocumentRoot " - "\/usr\/local\/apache2\/htdocs/I", - "-e", "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I", - "-e", "s/TypesConfig.*/TypesConfig " - "\/usr\/local\/apache2\/conf\/mime.types/I", - "-e", "s/LoadModule/#LoadModule/I", - "-e", "s/SSLCertificateFile.*/SSLCertificateFile " - "\/usr\/local\/apache2\/conf\/empty_cert.pem/I", - "-e", "s/SSLCertificateKeyFile.*/SSLCertificateKeyFile " - "\/usr\/local\/apache2\/conf\/rsa1024_key2.pem/I", - "-i"], stdin=find.stdout) - - def _prepare_configurator(self, server_root, config_file): + def _prepare_configurator(self): """Prepares the Apache plugin for testing""" - self.le_config.apache_server_root = server_root - self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format( - server_root, config_file) - self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root) - self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root) - self.le_config.apache_init_script = self.le_config.apache_ctl + " -k" + for k in constants.CLI_DEFAULTS_DEBIAN.keys(): + setattr(self.le_config, "apache_" + k, constants.os_constant(k)) + + # An alias + self.le_config.apache_handle_modules = self.le_config.apache_handle_mods self._apache_configurator = configurator.ApacheConfigurator( config=configuration.NamespaceConfig(self.le_config), @@ -183,39 +111,6 @@ class Proxy(configurators_common.Proxy): domain, cert_path, key_path, chain_path, fullchain_path) -def _is_apache_command(command): - """Returns true if command is an Apache command""" - if isinstance(command, list): - command = command[0] - - for apache_command in APACHE_COMMANDS: - if command.startswith(apache_command): - return True - - return False - - -def _modify_command(command): - """Modifies command so configtest works inside the docker image""" - if isinstance(command, list): - for i in xrange(len(command)): - if command[i] == "configtest": - command[i] = "-t" - else: - command = command.replace("configtest", "-t") - - return command - - -def _create_test_conf(server_root, apache_config): - """Creates a test config file and adds it to the Apache config""" - test_conf = os.path.join(server_root, "test.conf") - open(test_conf, "w").close() - subprocess.check_call( - ["sed", "-i", "1iInclude test.conf", apache_config]) - return test_conf - - def _get_server_root(config): """Returns the server root directory in config""" subdirs = [ @@ -223,7 +118,7 @@ def _get_server_root(config): if os.path.isdir(os.path.join(config, name))] if len(subdirs) != 1: - errors.Error("Malformed configuration directiory {0}".format(config)) + errors.Error("Malformed configuration directory {0}".format(config)) return os.path.join(config, subdirs[0].rstrip()) @@ -251,34 +146,3 @@ def _get_names(config): words[1].find(".") != -1): all_names.add(words[1]) return all_names, non_ip_names - - -def _get_modules(config): - """Returns the list of modules found in module_list""" - modules = [] - with open(os.path.join(config, "modules")) as f: - for line in f: - # Modules list is indented, everything else is headers/footers - if line[0].isspace(): - words = line.split() - # Modules redundantly end in "_module" which we can discard - modules.append(words[0][:-7]) - - return modules - - -def _get_version(config): - """Return version of Apache Server. - - Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from - the Apache plugin. - - """ - with open(os.path.join(config, "version")) as f: - # Should be on first line of input - matches = APACHE_VERSION_REGEX.findall(f.readline()) - - if len(matches) != 1: - raise errors.Error("Unable to find Apache version") - - return tuple([int(i) for i in matches[0].split(".")]) diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py index 4657883a3..03128cc86 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/common.py @@ -4,10 +4,7 @@ import os import shutil import tempfile -import docker - from certbot import constants -from certbot_compatibility_test import errors from certbot_compatibility_test import util @@ -18,20 +15,9 @@ class Proxy(object): # pylint: disable=too-many-instance-attributes """A common base for compatibility test configurators""" - _NOT_ADDED_ARGS = True - @classmethod def add_parser_arguments(cls, parser): """Adds command line arguments needed by the plugin""" - if Proxy._NOT_ADDED_ARGS: - group = parser.add_argument_group("docker") - group.add_argument( - "--docker-url", default="unix://var/run/docker.sock", - help="URL of the docker server") - group.add_argument( - "--no-remove", action="store_true", - help="do not delete container on program exit") - Proxy._NOT_ADDED_ARGS = False def __init__(self, args): """Initializes the plugin with the given command line args""" @@ -43,10 +29,8 @@ class Proxy(object): for config in os.listdir(config_dir)] self.args = args - self._docker_client = docker.Client( - base_url=self.args.docker_url, version="auto") - self.http_port, self.https_port = util.get_two_free_ports() - self._container_id = None + self.http_port = 80 + self.https_port = 443 def has_more_configs(self): """Returns true if there are more configs to test""" @@ -54,9 +38,6 @@ class Proxy(object): def cleanup_from_tests(self): """Performs any necessary cleanup from running plugin tests""" - self._docker_client.stop(self._container_id, 0) - if not self.args.no_remove: - self._docker_client.remove_container(self._container_id) def load_config(self): """Returns the next config directory to be tested""" @@ -65,67 +46,6 @@ class Proxy(object): os.makedirs(backup) return self._configs.pop() - def start_docker(self, image_name, command): - """Creates and runs a Docker container with the specified image""" - logger.warning("Pulling Docker image. This may take a minute.") - for line in self._docker_client.pull(image_name, stream=True): - logger.debug(line) - - host_config = docker.utils.create_host_config( - binds={self._temp_dir: {"bind": self._temp_dir, "mode": "rw"}}, - port_bindings={ - 80: ("127.0.0.1", self.http_port), - 443: ("127.0.0.1", self.https_port)},) - container = self._docker_client.create_container( - image_name, command, ports=[80, 443], volumes=self._temp_dir, - host_config=host_config) - if container["Warnings"]: - logger.warning(container["Warnings"]) - self._container_id = container["Id"] - self._docker_client.start(self._container_id) - - def check_call(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to check_call but executes the command in the - running docker image - - """ - if self.popen(command).returncode: - raise errors.Error( - "{0} exited with a nonzero value".format(command)) - - def popen(self, command, *args, **kwargs): - # pylint: disable=unused-argument - """Simulates a call to Popen but executes the command in the - running docker image - - """ - class SimplePopen(object): - # pylint: disable=too-few-public-methods - """Simplified Popen object""" - def __init__(self, returncode, output): - self.returncode = returncode - self._stdout = output - self._stderr = output - - def communicate(self): - """Returns stdout and stderr""" - return self._stdout, self._stderr - - if isinstance(command, list): - command = " ".join(command) - - returncode, output = self.execute_in_docker(command) - return SimplePopen(returncode, output) - - def execute_in_docker(self, command): - """Executes command inside the running docker image""" - logger.debug("Executing '%s'", command) - exec_id = self._docker_client.exec_create(self._container_id, command) - output = self._docker_client.exec_start(exec_id) - returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"] - return returncode, output - def copy_certs_and_keys(self, cert_path, key_path, chain_path=None): """Copies certs and keys into the temporary directory""" cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys") diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index 6823dfdab..38abffb18 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -7,6 +7,7 @@ import os import shutil import tempfile import time +import sys import OpenSSL @@ -21,17 +22,17 @@ from certbot_compatibility_test import errors from certbot_compatibility_test import util from certbot_compatibility_test import validator -from certbot_compatibility_test.configurators.apache import apache24 +from certbot_compatibility_test.configurators.apache import common DESCRIPTION = """ -Tests Certbot plugins against different server configuratons. It is -assumed that Docker is already installed. If no test types is specified, all +Tests Certbot plugins against different server configurations. It is +assumed that Docker is already installed. If no test type is specified, all tests that the plugin supports are performed. """ -PLUGINS = {"apache": apache24.Proxy} +PLUGINS = {"apache": common.Proxy} logger = logging.getLogger(__name__) @@ -61,8 +62,8 @@ def test_authenticator(plugin, config, temp_dir): "Plugin failed to complete %s for %s in %s", type(achalls[i]), achalls[i].domain, config) success = False - elif isinstance(responses[i], challenges.TLSSNI01): - verify = functools.partial(responses[i].simple_verify, achalls[i], + elif isinstance(responses[i], challenges.TLSSNI01Response): + verify = functools.partial(responses[i].simple_verify, achalls[i].chall, achalls[i].domain, util.JWK.public_key(), host="127.0.0.1", @@ -142,7 +143,8 @@ def test_deploy_cert(plugin, temp_dir, domains): for domain in domains: try: - plugin.deploy_cert(domain, cert_path, util.KEY_PATH) + plugin.deploy_cert(domain, cert_path, util.KEY_PATH, cert_path) + plugin.save() # Needed by the Apache plugin except le_errors.Error as error: logger.error("Plugin failed to deploy ceritificate for %s:", domain) logger.exception(error) @@ -177,6 +179,7 @@ def test_enhancements(plugin, domains): for domain in domains: try: plugin.enhance(domain, "redirect") + plugin.save() # Needed by the Apache plugin except le_errors.PluginError as error: # Don't immediately fail because a redirect may already be enabled logger.warning("Plugin failed to enable redirect for %s:", domain) @@ -341,7 +344,7 @@ def main(): temp_dir = tempfile.mkdtemp() plugin = PLUGINS[args.plugin](args) try: - plugin.execute_in_docker("mkdir -p /var/log/apache2") + overall_success = True while plugin.has_more_configs(): success = True @@ -360,10 +363,18 @@ def main(): if success: logger.info("All tests on %s succeeded", config) else: + overall_success = False logger.error("Tests on %s failed", config) finally: plugin.cleanup_from_tests() + if overall_success: + logger.warn("All compatibility tests succeeded") + sys.exit(0) + else: + logger.warn("One or more compatibility tests failed") + sys.exit(1) + if __name__ == "__main__": main() diff --git a/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz b/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz index 7323acf74..9b819d0c7 100644 Binary files a/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz and b/certbot-compatibility-test/certbot_compatibility_test/testdata/configs.tar.gz differ diff --git a/certbot-compatibility-test/certbot_compatibility_test/util.py b/certbot-compatibility-test/certbot_compatibility_test/util.py index cbce4fb56..af951aa6a 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/util.py +++ b/certbot-compatibility-test/certbot_compatibility_test/util.py @@ -1,11 +1,9 @@ """Utility functions for Certbot plugin tests.""" import argparse import copy -import contextlib import os import re import shutil -import socket import tarfile from acme import jose @@ -52,13 +50,3 @@ def extract_configs(configs, parent_dir): raise errors.Error("Unknown configurations file type") return config_dir - - -def get_two_free_ports(): - """Returns two free ports to use for the tests""" - with contextlib.closing(socket.socket()) as sock1: - with contextlib.closing(socket.socket()) as sock2: - sock1.bind(("", 0)) - sock2.bind(("", 0)) - - return sock1.getsockname()[1], sock2.getsockname()[1] diff --git a/certbot-compatibility-test/docs/conf.py b/certbot-compatibility-test/docs/conf.py index 1ef69ab2d..f89f4b368 100644 --- a/certbot-compatibility-test/docs/conf.py +++ b/certbot-compatibility-test/docs/conf.py @@ -311,7 +311,7 @@ texinfo_documents = [ intersphinx_mapping = { 'python': ('https://docs.python.org/', None), 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), - 'certbot': ('https://letsencrypt.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), 'certbot-apache': ( 'https://letsencrypt-apache.readthedocs.org/en/latest/', None), 'certbot-nginx': ( diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index c62a10f89..fb56be65f 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,12 +4,11 @@ from setuptools import setup from setuptools import find_packages -version = '0.6.0.dev0' +version = '0.9.0.dev0' install_requires = [ - 'certbot=={0}'.format(version), - 'certbot-apache=={0}'.format(version), - 'docker-py', + 'certbot', + 'certbot-apache', 'requests', 'zope.interface', ] diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index e402d5c79..d11b6bb14 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -17,7 +17,7 @@ from certbot import constants as core_constants from certbot import crypto_util from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util from certbot import reverter from certbot.plugins import common @@ -111,7 +111,7 @@ class NginxConfigurator(common.Plugin): :raises .errors.MisconfigurationError: If Nginx is misconfigured """ # Verify Nginx is installed - if not le_util.exe_exists(self.conf('ctl')): + if not util.exe_exists(self.conf('ctl')): raise errors.NoInstallationError # Make sure configuration is valid @@ -152,17 +152,17 @@ class NginxConfigurator(common.Plugin): "install a cert.") vhost = self.choose_vhost(domain) - cert_directives = [['ssl_certificate', fullchain_path], - ['ssl_certificate_key', key_path]] + cert_directives = [['\n', 'ssl_certificate', ' ', fullchain_path], + ['\n', 'ssl_certificate_key', ' ', key_path]] # OCSP stapling was introduced in Nginx 1.3.7. If we have that version # or greater, add config settings for it. stapling_directives = [] if self.version >= (1, 3, 7): stapling_directives = [ - ['ssl_trusted_certificate', chain_path], - ['ssl_stapling', 'on'], - ['ssl_stapling_verify', 'on']] + ['\n', 'ssl_trusted_certificate', ' ', chain_path], + ['\n', 'ssl_stapling', ' ', 'on'], + ['\n', 'ssl_stapling_verify', ' ', 'on'], ['\n']] if len(stapling_directives) != 0 and not chain_path: raise errors.PluginError( @@ -225,7 +225,7 @@ class NginxConfigurator(common.Plugin): if not matches: # No matches. Create a new vhost with this name in nginx.conf. filep = self.parser.loc["root"] - new_block = [['server'], [['server_name', target_name]]] + new_block = [['server'], [['\n', 'server_name', ' ', target_name]]] self.parser.add_http_directives(filep, new_block) vhost = obj.VirtualHost(filep, set([]), False, True, set([target_name]), list(new_block[1])) @@ -318,7 +318,7 @@ class NginxConfigurator(common.Plugin): cert = acme_crypto_util.gen_ss_cert(key, domains=[socket.gethostname()]) cert_pem = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, cert) - cert_file, cert_path = le_util.unique_file(os.path.join(tmp_dir, "cert.pem")) + cert_file, cert_path = util.unique_file(os.path.join(tmp_dir, "cert.pem")) with cert_file: cert_file.write(cert_pem) return cert_path, le_key.file @@ -337,10 +337,10 @@ class NginxConfigurator(common.Plugin): """ snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() - ssl_block = [['listen', '{0} ssl'.format(self.config.tls_sni_01_port)], - ['ssl_certificate', snakeoil_cert], - ['ssl_certificate_key', snakeoil_key], - ['include', self.parser.loc["ssl_options"]]] + ssl_block = [['\n', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)], + ['\n', 'ssl_certificate', ' ', snakeoil_cert], + ['\n', 'ssl_certificate_key', ' ', snakeoil_key], + ['\n', 'include', ' ', self.parser.loc["ssl_options"]]] self.parser.add_server_directives( vhost.filep, vhost.names, ssl_block, replace=False) vhost.ssl = True @@ -426,7 +426,7 @@ class NginxConfigurator(common.Plugin): """ try: - le_util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"]) + util.run_script([self.conf('ctl'), "-c", self.nginx_conf, "-t"]) except errors.SubprocessError as err: raise errors.MisconfigurationError(str(err)) @@ -439,11 +439,11 @@ class NginxConfigurator(common.Plugin): """ uid = os.geteuid() - le_util.make_or_verify_dir( + util.make_or_verify_dir( self.config.work_dir, core_constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( + util.make_or_verify_dir( self.config.backup_dir, core_constants.CONFIG_DIRS_MODE, uid) - le_util.make_or_verify_dir( + util.make_or_verify_dir( self.config.config_dir, core_constants.CONFIG_DIRS_MODE, uid) def get_version(self): @@ -689,11 +689,6 @@ def nginx_restart(nginx_ctl, nginx_conf="/etc/nginx.conf"): def temp_install(options_ssl): """Temporary install for convenience.""" - # WARNING: THIS IS A POTENTIAL SECURITY VULNERABILITY - # THIS SHOULD BE HANDLED BY THE PACKAGE MANAGER - # AND TAKEN OUT BEFORE RELEASE, INSTEAD - # SHOWING A NICE ERROR MESSAGE ABOUT THE PROBLEM. - # Check to make sure options-ssl.conf is installed if not os.path.isfile(options_ssl): shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) diff --git a/certbot-nginx/certbot_nginx/nginxparser.py b/certbot-nginx/certbot_nginx/nginxparser.py index cef0756d7..d6c352296 100644 --- a/certbot-nginx/certbot_nginx/nginxparser.py +++ b/certbot-nginx/certbot_nginx/nginxparser.py @@ -1,23 +1,30 @@ """Very low-level nginx config parser based on pyparsing.""" +# Forked from https://github.com/fatiherikli/nginxparser (MIT Licensed) +import copy +import logging import string from pyparsing import ( - Literal, White, Word, alphanums, CharsNotIn, Forward, Group, + Literal, White, Word, alphanums, CharsNotIn, Combine, Forward, Group, Optional, OneOrMore, Regex, ZeroOrMore) from pyparsing import stringEnd from pyparsing import restOfLine +logger = logging.getLogger(__name__) class RawNginxParser(object): # pylint: disable=expression-not-assigned """A class that parses nginx configuration with pyparsing.""" # constants + space = Optional(White()) + nonspace = Regex(r"\S+") left_bracket = Literal("{").suppress() - right_bracket = Literal("}").suppress() + right_bracket = space.leaveWhitespace() + Literal("}").suppress() semicolon = Literal(";").suppress() - space = White().suppress() - key = Word(alphanums + "_/") + key = Word(alphanums + "_/+-.") + dollar_var = Combine(Literal('$') + nonspace) + condition = Regex(r"\(.+\)") # Matches anything that is not a special character AND any chars in single # or double quotes value = Regex(r"((\".*\")?(\'.*\')?[^\{\};,]?)+") @@ -26,19 +33,25 @@ class RawNginxParser(object): modifier = Literal("=") | Literal("~*") | Literal("~") | Literal("^~") # rules - comment = Literal('#') + restOfLine() - assignment = (key + Optional(space + value, default=None) + semicolon) - location_statement = Optional(space + modifier) + Optional(space + location) - if_statement = Literal("if") + space + Regex(r"\(.+\)") + space + comment = space + Literal('#') + restOfLine() + + assignment = space + key + Optional(space + value, default=None) + semicolon + location_statement = space + Optional(modifier) + Optional(space + location + space) + if_statement = space + Literal("if") + space + condition + space + map_statement = space + Literal("map") + space + nonspace + space + dollar_var + space block = Forward() block << Group( - (Group(key + location_statement) ^ Group(if_statement)) + + # key could for instance be "server" or "http", or "location" (in which case + # location_statement needs to have a non-empty location) + (Group(space + key + location_statement) ^ Group(if_statement) ^ + Group(map_statement)).leaveWhitespace() + left_bracket + - Group(ZeroOrMore(Group(comment | assignment) | block)) + + Group(ZeroOrMore(Group(comment | assignment) | block) + space).leaveWhitespace() + right_bracket) - script = OneOrMore(Group(comment | assignment) ^ block) + stringEnd + script = OneOrMore(Group(comment | assignment) ^ block) + space + stringEnd + script.parseWithTabs() def __init__(self, source): self.source = source @@ -51,42 +64,47 @@ class RawNginxParser(object): """Returns the parsed tree as a list.""" return self.parse().asList() - class RawNginxDumper(object): # pylint: disable=too-few-public-methods """A class that dumps nginx configuration from the provided tree.""" - def __init__(self, blocks, indentation=4): + def __init__(self, blocks): self.blocks = blocks - self.indentation = indentation - def __iter__(self, blocks=None, current_indent=0, spacer=' '): + def __iter__(self, blocks=None): """Iterates the dumped nginx content.""" blocks = blocks or self.blocks - for key, values in blocks: - indentation = spacer * current_indent + for b0 in blocks: + if isinstance(b0, str): + yield b0 + continue + b = copy.deepcopy(b0) + if spacey(b[0]): + yield b.pop(0) # indentation + if not b: + continue + key, values = b.pop(0), b.pop(0) + if isinstance(key, list): - if current_indent: - yield '' - yield indentation + spacer.join(key) + ' {' - + yield "".join(key) + '{' for parameter in values: - dumped = self.__iter__([parameter], current_indent + self.indentation) - for line in dumped: + for line in self.__iter__([parameter]): # negate "for b0 in blocks" yield line - - yield indentation + '}' + yield '}' else: - if key == '#': - yield spacer * current_indent + key + values - else: - if values is None: - yield spacer * current_indent + key + ';' - else: - yield spacer * current_indent + key + spacer + values + ';' + if isinstance(key, str) and key.strip() == '#': # comment + yield key + values + else: # assignment + gap = "" + # Sometimes the parser has stuck some gap whitespace in here; + # if so rotate it into gap + if values and spacey(values): + gap = values + values = b.pop(0) + yield key + gap + values + ';' def __str__(self): """Return the parsed block as a string.""" - return '\n'.join(self) + '\n' + return ''.join(self) # Shortcut functions to respect Python's serialization interface @@ -100,7 +118,7 @@ def loads(source): :rtype: list """ - return RawNginxParser(source).as_list() + return UnspacedList(RawNginxParser(source).as_list()) def load(_file): @@ -114,24 +132,143 @@ def load(_file): return loads(_file.read()) -def dumps(blocks, indentation=4): +def dumps(blocks): """Dump to a string. - :param list block: The parsed tree + :param UnspacedList block: The parsed tree :param int indentation: The number of spaces to indent :rtype: str """ - return str(RawNginxDumper(blocks, indentation)) + return str(RawNginxDumper(blocks.spaced)) -def dump(blocks, _file, indentation=4): +def dump(blocks, _file): """Dump to a file. - :param list block: The parsed tree + :param UnspacedList block: The parsed tree :param file _file: The file to dump to :param int indentation: The number of spaces to indent :rtype: NoneType """ - return _file.write(dumps(blocks, indentation)) + return _file.write(dumps(blocks)) + + +spacey = lambda x: (isinstance(x, str) and x.isspace()) or x == '' + +class UnspacedList(list): + """Wrap a list [of lists], making any whitespace entries magically invisible""" + + def __init__(self, list_source): + # ensure our argument is not a generator, and duplicate any sublists + self.spaced = copy.deepcopy(list(list_source)) + self.dirty = False + + # Turn self into a version of the source list that has spaces removed + # and all sub-lists also UnspacedList()ed + list.__init__(self, list_source) + for i, entry in reversed(list(enumerate(self))): + if isinstance(entry, list): + sublist = UnspacedList(entry) + list.__setitem__(self, i, sublist) + self.spaced[i] = sublist.spaced + elif spacey(entry): + # don't delete comments + if "#" not in self[:i]: + list.__delitem__(self, i) + + def _coerce(self, inbound): + """ + Coerce some inbound object to be appropriately usable in this object + + :param inbound: string or None or list or UnspacedList + :returns: (coerced UnspacedList or string or None, spaced equivalent) + :rtype: tuple + + """ + if not isinstance(inbound, list): # str or None + return (inbound, inbound) + else: + if not hasattr(inbound, "spaced"): + inbound = UnspacedList(inbound) + return (inbound, inbound.spaced) + + + def insert(self, i, x): + item, spaced_item = self._coerce(x) + self.spaced.insert(self._spaced_position(i), spaced_item) + list.insert(self, i, item) + self.dirty = True + + def append(self, x): + item, spaced_item = self._coerce(x) + self.spaced.append(spaced_item) + list.append(self, item) + self.dirty = True + + def extend(self, x): + item, spaced_item = self._coerce(x) + self.spaced.extend(spaced_item) + list.extend(self, item) + self.dirty = True + + def __add__(self, other): + l = copy.deepcopy(self) + l.extend(other) + l.dirty = True + return l + + def pop(self, _i=None): + raise NotImplementedError("UnspacedList.pop() not yet implemented") + def remove(self, _): + raise NotImplementedError("UnspacedList.remove() not yet implemented") + def reverse(self): + raise NotImplementedError("UnspacedList.reverse() not yet implemented") + def sort(self, _cmp=None, _key=None, _Rev=None): + raise NotImplementedError("UnspacedList.sort() not yet implemented") + def __setslice__(self, _i, _j, _newslice): + raise NotImplementedError("Slice operations on UnspacedLists not yet implemented") + + def __setitem__(self, i, value): + if isinstance(i, slice): + raise NotImplementedError("Slice operations on UnspacedLists not yet implemented") + item, spaced_item = self._coerce(value) + self.spaced.__setitem__(self._spaced_position(i), spaced_item) + list.__setitem__(self, i, item) + self.dirty = True + + def __delitem__(self, i): + self.spaced.__delitem__(self._spaced_position(i)) + list.__delitem__(self, i) + self.dirty = True + + def __deepcopy__(self, memo): + l = UnspacedList(self[:]) + l.spaced = copy.deepcopy(self.spaced, memo=memo) + l.dirty = self.dirty + return l + + def is_dirty(self): + """Recurse through the parse tree to figure out if any sublists are dirty""" + if self.dirty: + return True + return any((isinstance(x, list) and x.is_dirty() for x in self)) + + def _spaced_position(self, idx): + "Convert from indexes in the unspaced list to positions in the spaced one" + pos = spaces = 0 + # Normalize indexes like list[-1] etc, and save the result + if idx < 0: + idx = len(self) + idx + if not 0 <= idx < len(self): + raise IndexError("list index out of range") + idx0 = idx + # Count the number of spaces in the spaced list before idx in the unspaced one + while idx != -1: + if spacey(self.spaced[pos]): + spaces += 1 + else: + idx -= 1 + pos += 1 + return idx0 + spaces diff --git a/certbot-nginx/certbot_nginx/obj.py b/certbot-nginx/certbot_nginx/obj.py index 0d1151f39..f5ac88f6c 100644 --- a/certbot-nginx/certbot_nginx/obj.py +++ b/certbot-nginx/certbot_nginx/obj.py @@ -85,6 +85,9 @@ class Addr(common.Addr): return parts + def __repr__(self): + return "Addr(" + self.__str__() + ")" + def __eq__(self, other): if isinstance(other, self.__class__): return (self.tup == other.tup and @@ -126,6 +129,9 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods "enabled: %s" % (self.filep, addr_str, self.names, self.ssl, self.enabled)) + def __repr__(self): + return "VirtualHost(" + self.__str__().replace("\n", ", ") + ")\n" + def __eq__(self, other): if isinstance(other, self.__class__): return (self.filep == other.filep and diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 2f08c15d3..e7992810a 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -1,4 +1,5 @@ """NginxParser is a member object of the NginxConfigurator class.""" +import copy import glob import logging import os @@ -17,7 +18,7 @@ logger = logging.getLogger(__name__) class NginxParser(object): """Class handles the fine details of parsing the Nginx Configuration. - :ivar str root: Normalized abosulte path to the server root + :ivar str root: Normalized absolute path to the server root directory. Without trailing slash. :ivar dict parsed: Mapping of file paths to parsed trees @@ -113,6 +114,7 @@ class NginxParser(object): for filename in servers: for server in servers[filename]: # Parse the server block into a VirtualHost object + parsed_server = parse_server(server) vhost = obj.VirtualHost(filename, parsed_server['addrs'], @@ -132,7 +134,7 @@ class NginxParser(object): :rtype: list """ - result = list(block) # Copy the list to keep self.parsed idempotent + result = copy.deepcopy(block) # Copy the list to keep self.parsed idempotent for directive in block: if _is_include_directive(directive): included_files = glob.glob( @@ -153,7 +155,9 @@ class NginxParser(object): :rtype: list """ - files = glob.glob(filepath) + files = glob.glob(filepath) # nginx on unix calls glob(3) for this + # XXX Windows nginx uses FindFirstFile, and + # should have a narrower call here trees = [] for item in files: if item in self.parsed and not override: @@ -201,21 +205,27 @@ class NginxParser(object): raise errors.NoInstallationError( "Could not find configuration root") - def filedump(self, ext='tmp'): + def filedump(self, ext='tmp', lazy=True): """Dumps parsed configurations into files. :param str ext: The file extension to use for the dumped files. If empty, this overrides the existing conf files. + :param bool lazy: Only write files that have been modified """ + # Best-effort atomicity is enforced above us by reverter.py for filename in self.parsed: tree = self.parsed[filename] if ext: filename = filename + os.path.extsep + ext try: - logger.debug('Dumping to %s:\n%s', filename, nginxparser.dumps(tree)) + if lazy and not tree.is_dirty(): + continue + out = nginxparser.dumps(tree) + logger.debug('Writing nginx conf tree to %s:\n%s', filename, out) with open(filename, 'w') as _file: - nginxparser.dump(tree, _file) + _file.write(out) + except IOError: logger.error("Could not open file for writing: %s", filename) @@ -463,6 +473,8 @@ def parse_server(server): 'names': set()} for directive in server: + if not directive: + continue if directive[0] == 'listen': addr = obj.Addr.fromstring(directive[1]) parsed_server['addrs'].add(addr) @@ -503,6 +515,11 @@ def _add_directive(block, directive, replace): See _add_directives for more documentation. """ + directive = nginxparser.UnspacedList(directive) + if len(directive) == 0: + # whitespace + block.append(directive) + return location = -1 # Find the index of a config line where the name of the directive matches # the name of the directive we want to add. diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index b36802939..7529bc10b 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -30,7 +30,7 @@ class NginxConfiguratorTest(util.NginxTest): shutil.rmtree(self.config_dir) shutil.rmtree(self.work_dir) - @mock.patch("certbot_nginx.configurator.le_util.exe_exists") + @mock.patch("certbot_nginx.configurator.util.exe_exists") def test_prepare_no_install(self, mock_exe_exists): mock_exe_exists.return_value = False self.assertRaises( @@ -40,7 +40,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEquals((1, 6, 2), self.config.version) self.assertEquals(5, len(self.config.parser.parsed)) - @mock.patch("certbot_nginx.configurator.le_util.exe_exists") + @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") def test_prepare_initializes_version(self, mock_popen, mock_exe_exists): mock_popen().communicate.return_value = ( @@ -83,7 +83,7 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') self.config.parser.add_server_directives( filep, set(['.example.com', 'example.*']), - [['listen', '5001 ssl']], + [['listen', ' ', '5001 ssl']], replace=False) self.config.save() @@ -265,7 +265,8 @@ class NginxConfiguratorTest(util.NginxTest): @mock.patch("certbot_nginx.configurator.tls_sni_01.NginxTlsSni01.perform") @mock.patch("certbot_nginx.configurator.NginxConfigurator.restart") - def test_perform(self, mock_restart, mock_perform): + @mock.patch("certbot_nginx.configurator.NginxConfigurator.revert_challenge_config") + def test_perform_and_cleanup(self, mock_revert, mock_restart, mock_perform): # Only tests functionality specific to configurator.perform # Note: As more challenges are offered this will have to be expanded achall1 = achallenges.KeyAuthorizationAnnotatedChallenge( @@ -291,7 +292,11 @@ class NginxConfiguratorTest(util.NginxTest): self.assertEqual(mock_perform.call_count, 1) self.assertEqual(responses, expected) - self.assertEqual(mock_restart.call_count, 1) + + self.config.cleanup([achall1, achall2]) + self.assertEqual(0, self.config._chall_out) # pylint: disable=protected-access + self.assertEqual(mock_revert.call_count, 1) + self.assertEqual(mock_restart.call_count, 2) @mock.patch("certbot_nginx.configurator.subprocess.Popen") def test_get_version(self, mock_popen): @@ -362,11 +367,11 @@ class NginxConfiguratorTest(util.NginxTest): mock_popen.side_effect = OSError("Can't find program") self.assertRaises(errors.MisconfigurationError, self.config.restart) - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_config_test(self, _): self.config.config_test() - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_config_test_bad_process(self, mock_run_script): mock_run_script.side_effect = errors.SubprocessError self.assertRaises(errors.MisconfigurationError, self.config.config_test) diff --git a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py index 80e82c903..a0a0736a5 100644 --- a/certbot-nginx/certbot_nginx/tests/nginxparser_test.py +++ b/certbot-nginx/certbot_nginx/tests/nginxparser_test.py @@ -1,11 +1,13 @@ """Test for certbot_nginx.nginxparser.""" +import copy import operator +import os import unittest from pyparsing import ParseException from certbot_nginx.nginxparser import ( - RawNginxParser, loads, load, dumps, dump) + RawNginxParser, loads, load, dumps, dump, UnspacedList) from certbot_nginx.tests import util @@ -17,39 +19,39 @@ class TestRawNginxParser(unittest.TestCase): def test_assignments(self): parsed = RawNginxParser.assignment.parseString('root /test;').asList() - self.assertEqual(parsed, ['root', '/test']) - parsed = RawNginxParser.assignment.parseString('root /test;' - 'foo bar;').asList() - self.assertEqual(parsed, ['root', '/test'], ['foo', 'bar']) + self.assertEqual(parsed, ['root', ' ', '/test']) + parsed = RawNginxParser.assignment.parseString('root /test;foo bar;').asList() + self.assertEqual(parsed, ['root', ' ', '/test'], ['foo', ' ', 'bar']) def test_blocks(self): parsed = RawNginxParser.block.parseString('foo {}').asList() - self.assertEqual(parsed, [[['foo'], []]]) + self.assertEqual(parsed, [[['foo', ' '], []]]) parsed = RawNginxParser.block.parseString('location /foo{}').asList() - self.assertEqual(parsed, [[['location', '/foo'], []]]) - parsed = RawNginxParser.block.parseString('foo { bar foo; }').asList() - self.assertEqual(parsed, [[['foo'], [['bar', 'foo']]]]) + self.assertEqual(parsed, [[['location', ' ', '/foo'], []]]) + parsed = RawNginxParser.block.parseString('foo { bar foo ; }').asList() + self.assertEqual(parsed, [[['foo', ' '], [[' ', 'bar', ' ', 'foo '], ' ']]]) def test_nested_blocks(self): parsed = RawNginxParser.block.parseString('foo { bar {} }').asList() block, content = FIRST(parsed) - self.assertEqual(FIRST(content), [['bar'], []]) + self.assertEqual(FIRST(content), [[' ', 'bar', ' '], []]) self.assertEqual(FIRST(block), 'foo') def test_dump_as_string(self): - dumped = dumps([ - ['user', 'www-data'], - [['server'], [ - ['listen', '80'], - ['server_name', 'foo.com'], - ['root', '/home/ubuntu/sites/foo/'], - [['location', '/status'], [ - ['check_status', None], - [['types'], [['image/jpeg', 'jpg']]], + dumped = dumps(UnspacedList([ + ['user', ' ', 'www-data'], + [['\n', 'server', ' '], [ + ['\n ', 'listen', ' ', '80'], + ['\n ', 'server_name', ' ', 'foo.com'], + ['\n ', 'root', ' ', '/home/ubuntu/sites/foo/'], + [['\n\n ', 'location', ' ', '/status', ' '], [ + ['\n ', 'check_status', ''], + [['\n\n ', 'types', ' '], + [['\n ', 'image/jpeg', ' ', 'jpg']]], ]] - ]]]) + ]]])) - self.assertEqual(dumped, + self.assertEqual(dumped.split('\n'), 'user www-data;\n' 'server {\n' ' listen 80;\n' @@ -60,10 +62,7 @@ class TestRawNginxParser(unittest.TestCase): ' check_status;\n' '\n' ' types {\n' - ' image/jpeg jpg;\n' - ' }\n' - ' }\n' - '}\n') + ' image/jpeg jpg;}}}'.split('\n')) def test_parse_from_file(self): with open(util.get_data_filename('foo.conf')) as handle: @@ -116,24 +115,29 @@ class TestRawNginxParser(unittest.TestCase): def test_dump_as_file(self): with open(util.get_data_filename('nginx.conf')) as handle: - parsed = util.filter_comments(load(handle)) - parsed[-1][-1].append([['server'], - [['listen', '443 ssl'], - ['server_name', 'localhost'], - ['ssl_certificate', 'cert.pem'], - ['ssl_certificate_key', 'cert.key'], - ['ssl_session_cache', 'shared:SSL:1m'], - ['ssl_session_timeout', '5m'], - ['ssl_ciphers', 'HIGH:!aNULL:!MD5'], - [['location', '/'], - [['root', 'html'], - ['index', 'index.html index.htm']]]]]) + parsed = load(handle) + parsed[-1][-1].append(UnspacedList([['server'], + [['listen', ' ', '443 ssl'], + ['server_name', ' ', 'localhost'], + ['ssl_certificate', ' ', 'cert.pem'], + ['ssl_certificate_key', ' ', 'cert.key'], + ['ssl_session_cache', ' ', 'shared:SSL:1m'], + ['ssl_session_timeout', ' ', '5m'], + ['ssl_ciphers', ' ', 'HIGH:!aNULL:!MD5'], + [['location', ' ', '/'], + [['root', ' ', 'html'], + ['index', ' ', 'index.html index.htm']]]]])) with open(util.get_data_filename('nginx.new.conf'), 'w') as handle: dump(parsed, handle) with open(util.get_data_filename('nginx.new.conf')) as handle: - parsed_new = util.filter_comments(load(handle)) - self.assertEquals(parsed, parsed_new) + parsed_new = load(handle) + try: + self.maxDiff = None + self.assertEquals(parsed[0], parsed_new[0]) + self.assertEquals(parsed[1:], parsed_new[1:]) + finally: + os.unlink(util.get_data_filename('nginx.new.conf')) def test_comments(self): with open(util.get_data_filename('minimalistic_comments.conf')) as handle: @@ -145,20 +149,23 @@ class TestRawNginxParser(unittest.TestCase): with open(util.get_data_filename('minimalistic_comments.new.conf')) as handle: parsed_new = load(handle) - self.assertEquals(parsed, parsed_new) + try: + self.assertEquals(parsed, parsed_new) - self.assertEqual(parsed_new, [ - ['#', " Use bar.conf when it's a full moon!"], - ['include', 'foo.conf'], - ['#', ' Kilroy was here'], - ['check_status', None], - [['server'], - [['#', ''], - ['#', " Don't forget to open up your firewall!"], - ['#', ''], - ['listen', '1234'], - ['#', ' listen 80;']]], - ]) + self.assertEqual(parsed_new, [ + ['#', " Use bar.conf when it's a full moon!"], + ['include', 'foo.conf'], + ['#', ' Kilroy was here'], + ['check_status'], + [['server'], + [['#', ''], + ['#', " Don't forget to open up your firewall!"], + ['#', ''], + ['listen', '1234'], + ['#', ' listen 80;']]], + ]) + finally: + os.unlink(util.get_data_filename('minimalistic_comments.new.conf')) def test_issue_518(self): parsed = loads('if ($http_accept ~* "webp") { set $webp "true"; }') @@ -168,5 +175,73 @@ class TestRawNginxParser(unittest.TestCase): [['set', '$webp "true"']]] ]) +class TestUnspacedList(unittest.TestCase): + """Test the UnspacedList data structure""" + def setUp(self): + self.a = ["\n ", "things", " ", "quirk"] + self.b = ["y", " "] + self.l = self.a[:] + self.l2 = self.b[:] + self.ul = UnspacedList(self.l) + self.ul2 = UnspacedList(self.l2) + + def test_construction(self): + self.assertEqual(self.ul, ["things", "quirk"]) + self.assertEqual(self.ul2, ["y"]) + + def test_append(self): + ul3 = copy.deepcopy(self.ul) + ul3.append("wise") + self.assertEqual(ul3, ["things", "quirk", "wise"]) + self.assertEqual(ul3.spaced, self.a + ["wise"]) + + def test_add(self): + ul3 = self.ul + self.ul2 + self.assertEqual(ul3, ["things", "quirk", "y"]) + self.assertEqual(ul3.spaced, self.a + self.b) + self.assertEqual(self.ul.spaced, self.a) + ul3 = self.ul + self.l2 + self.assertEqual(ul3, ["things", "quirk", "y"]) + self.assertEqual(ul3.spaced, self.a + self.b) + + def test_extend(self): + ul3 = copy.deepcopy(self.ul) + ul3.extend(self.ul2) + self.assertEqual(ul3, ["things", "quirk", "y"]) + self.assertEqual(ul3.spaced, self.a + self.b) + self.assertEqual(self.ul.spaced, self.a) + + def test_set(self): + ul3 = copy.deepcopy(self.ul) + ul3[0] = "zither" + l = ["\n ", "zather", "zest"] + ul3[1] = UnspacedList(l) + self.assertEqual(ul3, ["zither", ["zather", "zest"]]) + self.assertEqual(ul3.spaced, [self.a[0], "zither", " ", l]) + + def test_get(self): + self.assertRaises(IndexError, self.ul2.__getitem__, 2) + self.assertRaises(IndexError, self.ul2.__getitem__, -3) + + def test_rawlists(self): + ul3 = copy.deepcopy(self.ul) + ul3.insert(0, "some") + ul3.append("why") + ul3.extend(["did", "whether"]) + del ul3[2] + self.assertEqual(ul3, ["some", "things", "why", "did", "whether"]) + + def test_is_dirty(self): + self.assertEqual(False, self.ul2.is_dirty()) + ul3 = UnspacedList([]) + ul3.append(self.ul) + self.assertEqual(False, self.ul.is_dirty()) + self.assertEqual(True, ul3.is_dirty()) + ul4 = UnspacedList([[1], [2, 3, 4]]) + self.assertEqual(False, ul4.is_dirty()) + ul4[1][2] = 5 + self.assertEqual(True, ul4.is_dirty()) + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 8ac995dfc..464d717ed 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -66,7 +66,7 @@ class NginxParserTest(util.NginxTest): def test_filedump(self): nparser = parser.NginxParser(self.config_path, self.ssl_options) - nparser.filedump('test') + nparser.filedump('test', lazy=False) # pylint: disable=protected-access parsed = nparser._parse_files(nparser.abs_path( 'sites-enabled/example.com.test')) @@ -126,7 +126,7 @@ class NginxParserTest(util.NginxTest): nparser.add_server_directives(nparser.abs_path('nginx.conf'), set(['localhost', r'~^(www\.)?(example|bar)\.']), - [['foo', 'bar'], ['ssl_certificate', + [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ', '/etc/ssl/cert.pem']], replace=False) ssl_re = re.compile(r'\n\s+ssl_certificate /etc/ssl/cert.pem') diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/minimalistic_comments.new.conf b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/minimalistic_comments.new.conf deleted file mode 100644 index d1b7be91e..000000000 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/minimalistic_comments.new.conf +++ /dev/null @@ -1,11 +0,0 @@ -# Use bar.conf when it's a full moon! -include foo.conf; -# Kilroy was here -check_status; -server { - # - # Don't forget to open up your firewall! - # - listen 1234; - # listen 80; -} diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/nginx.new.conf b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/nginx.new.conf deleted file mode 100644 index 59c1c968f..000000000 --- a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/nginx.new.conf +++ /dev/null @@ -1,83 +0,0 @@ -user nobody; -worker_processes 1; -error_log logs/error.log; -error_log logs/error.log notice; -error_log logs/error.log info; -pid logs/nginx.pid; -events { - worker_connections 1024; -} -include foo.conf; -http { - include mime.types; - include sites-enabled/*; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log logs/access.log main; - sendfile on; - tcp_nopush on; - keepalive_timeout 0; - gzip on; - - server { - listen 8080; - server_name localhost; - server_name ~^(www\.)?(example|bar)\.; - charset koi8-r; - access_log logs/host.access.log main; - - location / { - root html; - index index.html index.htm; - } - error_page 404 /404.html; - error_page 500 502 503 504 /50x.html; - - location = /50x.html { - root html; - } - - location ~ \.php$ { - proxy_pass http://127.0.0.1; - } - - location ~ \.php$ { - root html; - fastcgi_pass 127.0.0.1:9000; - fastcgi_index index.php; - fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; - } - - location ~ /\.ht { - deny all; - } - } - - server { - listen 8000; - listen somename:8080; - include server.conf; - - location / { - root html; - index index.html index.htm; - } - } - - server { - listen 443 ssl; - server_name localhost; - ssl_certificate cert.pem; - ssl_certificate_key cert.key; - ssl_session_cache shared:SSL:1m; - ssl_session_timeout 5m; - ssl_ciphers HIGH:!aNULL:!MD5; - - location / { - root html; - index index.html index.htm; - } - } -} diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 3264d6ed3..a92caf788 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -80,7 +80,7 @@ class TlsSniPerformTest(util.NginxTest): mock_setup_cert.assert_called_once_with(self.achalls[0]) self.assertEqual([response], responses) - self.assertEqual(mock_save.call_count, 2) + self.assertEqual(mock_save.call_count, 1) # Make sure challenge config is included in main config http = self.sni.configurator.parser.parsed[ diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 3c4731700..866e5a9c7 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -1,4 +1,5 @@ """Common utilities for certbot_nginx.""" +import copy import os import pkg_resources import unittest @@ -51,7 +52,7 @@ def get_nginx_configurator( with mock.patch("certbot_nginx.configurator.NginxConfigurator." "config_test"): - with mock.patch("certbot_nginx.configurator.le_util." + with mock.patch("certbot_nginx.configurator.util." "exe_exists") as mock_exe_exists: mock_exe_exists.return_value = True config = configurator.NginxConfigurator( @@ -82,12 +83,15 @@ def filter_comments(tree): def traverse(tree): """Generator dropping comment nodes""" - for key, values in tree: + for entry in tree: + key, values = entry if isinstance(key, list): - yield [key, filter_comments(values)] + new = copy.deepcopy(entry) + new[1] = filter_comments(values) + yield new else: if key != '#': - yield [key, values] + yield entry return list(traverse(tree)) diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index e4c5d31a6..ebc92f5e3 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -46,8 +46,6 @@ class NginxTlsSni01(common.TLSSNI01): if not self.achalls: return [] - self.configurator.save() - addresses = [] default_addr = "{0} default_server ssl".format( self.configurator.config.tls_sni_01_port) @@ -93,10 +91,10 @@ class NginxTlsSni01(common.TLSSNI01): # Add the 'include' statement for the challenges if it doesn't exist # already in the main config included = False - include_directive = ['include', self.challenge_conf] + include_directive = ['include', ' ', self.challenge_conf] root = self.configurator.parser.loc["root"] - bucket_directive = ['server_names_hash_bucket_size', '128'] + bucket_directive = ['server_names_hash_bucket_size', ' ', '128'] main = self.configurator.parser.parsed[root] for key, body in main: @@ -118,6 +116,7 @@ class NginxTlsSni01(common.TLSSNI01): config = [self._make_server_block(pair[0], pair[1]) for pair in itertools.izip(self.achalls, ll_addrs)] + config = nginxparser.UnspacedList(config) self.configurator.reverter.register_file_creation( True, self.challenge_conf) @@ -142,19 +141,19 @@ class NginxTlsSni01(common.TLSSNI01): document_root = os.path.join( self.configurator.config.work_dir, "tls_sni_01_page") - block = [['listen', str(addr)] for addr in addrs] + block = [['listen', ' ', str(addr)] for addr in addrs] - block.extend([['server_name', + block.extend([['server_name', ' ', achall.response(achall.account_key).z_domain], - ['include', self.configurator.parser.loc["ssl_options"]], + ['include', ' ', self.configurator.parser.loc["ssl_options"]], # access and error logs necessary for # integration testing (non-root) - ['access_log', os.path.join( + ['access_log', ' ', os.path.join( self.configurator.config.work_dir, 'access.log')], - ['error_log', os.path.join( + ['error_log', ' ', os.path.join( self.configurator.config.work_dir, 'error.log')], - ['ssl_certificate', self.get_cert_path(achall)], - ['ssl_certificate_key', self.get_key_path(achall)], - [['location', '/'], [['root', document_root]]]]) + ['ssl_certificate', ' ', self.get_cert_path(achall)], + ['ssl_certificate_key', ' ', self.get_key_path(achall)], + [['location', ' ', '/'], [['root', ' ', document_root]]]]) return [['server'], block] diff --git a/certbot-nginx/docs/conf.py b/certbot-nginx/docs/conf.py index fa00e6503..167abb4fb 100644 --- a/certbot-nginx/docs/conf.py +++ b/certbot-nginx/docs/conf.py @@ -307,5 +307,5 @@ texinfo_documents = [ intersphinx_mapping = { 'python': ('https://docs.python.org/', None), 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), - 'certbot': ('https://letsencrypt.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), } diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 0e5c27a0a..62c705b4c 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.6.0.dev0' +version = '0.9.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index a48d62548..34358a5d9 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.6.0.dev0' +__version__ = '0.9.0.dev0' diff --git a/certbot/account.py b/certbot/account.py index cc50a6ea6..798f8664e 100644 --- a/certbot/account.py +++ b/certbot/account.py @@ -16,7 +16,7 @@ from acme import messages from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util logger = logging.getLogger(__name__) @@ -130,7 +130,7 @@ class AccountFileStorage(interfaces.AccountStorage): """ def __init__(self, config): self.config = config - le_util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(), + util.make_or_verify_dir(config.accounts_dir, 0o700, os.geteuid(), self.config.strict_permissions) def _account_dir_path(self, account_id): @@ -186,16 +186,29 @@ class AccountFileStorage(interfaces.AccountStorage): return acc def save(self, account): + self._save(account, regr_only=False) + + def save_regr(self, account): + """Save the registration resource. + + :param Account account: account whose regr should be saved + + """ + self._save(account, regr_only=True) + + def _save(self, account, regr_only): account_dir_path = self._account_dir_path(account.id) - le_util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), - self.config.strict_permissions) + util.make_or_verify_dir(account_dir_path, 0o700, os.geteuid(), + self.config.strict_permissions) try: with open(self._regr_path(account_dir_path), "w") as regr_file: regr_file.write(account.regr.json_dumps()) - with le_util.safe_open(self._key_path(account_dir_path), - "w", chmod=0o400) as key_file: - key_file.write(account.key.json_dumps()) - with open(self._metadata_path(account_dir_path), "w") as metadata_file: - metadata_file.write(account.meta.json_dumps()) + if not regr_only: + with util.safe_open(self._key_path(account_dir_path), + "w", chmod=0o400) as key_file: + key_file.write(account.key.json_dumps()) + with open(self._metadata_path( + account_dir_path), "w") as metadata_file: + metadata_file.write(account.meta.json_dumps()) except IOError as error: raise errors.AccountStorageError(error) diff --git a/certbot/cli.py b/certbot/cli.py index 97b1a5399..35b3b74ae 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1,15 +1,14 @@ """Certbot command line argument & config processing.""" from __future__ import print_function import argparse +import copy import glob import logging import logging.handlers import os import sys -import traceback import configargparse -import OpenSSL import six import certbot @@ -19,7 +18,7 @@ from certbot import crypto_util from certbot import errors from certbot import hooks from certbot import interfaces -from certbot import le_util +from certbot import util from certbot.plugins import disco as plugins_disco import certbot.plugins.selection as plugin_selection @@ -37,8 +36,14 @@ helpful_parser = None # should only be used for purposes where inability to detect letsencrypt-auto # fails safely +LEAUTO = "letsencrypt-auto" +if "CERTBOT_AUTO" in os.environ: + # if we're here, this is probably going to be certbot-auto, unless the + # user saved the script under a different name + LEAUTO = os.path.basename(os.environ["CERTBOT_AUTO"]) + fragment = os.path.join(".local", "share", "letsencrypt") -cli_command = "letsencrypt-auto" if fragment in sys.argv[0] else "certbot" +cli_command = LEAUTO if fragment in sys.argv[0] else "certbot" # Argparse's help formatting has a lot of unhelpful peculiarities, so we want # to replace as much of it as we can... @@ -57,6 +62,7 @@ cert. Major SUBCOMMANDS are: install Install a previously obtained cert in a server renew Renew previously obtained certs that are near expiry revoke Revoke a previously obtained certificate + register Perform tasks related to registering with the CA rollback Rollback server configuration changes made during install config_changes Show changes made to server config during installation plugins Display information about installed plugins @@ -82,7 +88,8 @@ More detailed help: the available topics are: all, automation, paths, security, testing, or any of the subcommands or - plugins (certonly, install, nginx, apache, standalone, webroot, etc) + plugins (certonly, install, register, nginx, apache, standalone, webroot, + etc.) """ @@ -141,6 +148,22 @@ def usage_strings(plugins): return USAGE % (apache_doc, nginx_doc), SHORT_USAGE +def possible_deprecation_warning(config): + "A deprecation warning for users with the old, not-self-upgrading letsencrypt-auto." + if cli_command != LEAUTO: + return + if config.no_self_upgrade: + # users setting --no-self-upgrade might be hanging on a clent version like 0.3.0 + # or 0.5.0 which is the new script, but doesn't set CERTBOT_AUTO; they don't + # need warnings + return + if "CERTBOT_AUTO" not in os.environ: + logger.warn("You are running with an old copy of letsencrypt-auto that does " + "not receive updates, and is less reliable than more recent versions. " + "We recommend upgrading to the latest certbot-auto script, or using native " + "OS packages.") + + class _Default(object): """A class to use as a default to detect if a value is set by a user""" @@ -189,6 +212,35 @@ def set_by_cli(var): set_by_cli.detector = None +def has_default_value(option, value): + """Does option have the default value? + + If the default value of option is not known, False is returned. + + :param str option: configuration variable being considered + :param value: value of the configuration variable named option + + :returns: True if option has the default value, otherwise, False + :rtype: bool + + """ + return (option in helpful_parser.defaults and + helpful_parser.defaults[option] == value) + + +def option_was_set(option, value): + """Was option set by the user or does it differ from the default? + + :param str option: configuration variable being considered + :param value: value of the configuration variable named option + + :returns: True if the option was set, otherwise, False + :rtype: bool + + """ + return set_by_cli(option) or not has_default_value(option, value) + + def argparse_type(variable): "Return our argparse type function for a config variable (default: str)" # pylint: disable=protected-access @@ -265,8 +317,9 @@ class HelpfulArgumentParser(object): self.VERBS = {"auth": main.obtain_cert, "certonly": main.obtain_cert, "config_changes": main.config_changes, "run": main.run, "install": main.install, "plugins": main.plugins_cmd, - "renew": main.renew, "revoke": main.revoke, - "rollback": main.rollback, "everything": main.run} + "register": main.register, "renew": main.renew, + "revoke": main.revoke, "rollback": main.rollback, + "everything": main.run} # List of topics for which additional help can be provided HELP_TOPICS = ["all", "security", "paths", "automation", "testing"] + list(self.VERBS) @@ -297,6 +350,7 @@ class HelpfulArgumentParser(object): sys.exit(0) self.visible_topics = self.determine_help_topics(self.help_arg) self.groups = {} # elements are added by .add_group() + self.defaults = {} # elements are added by .parse_args() def parse_args(self): """Parses command line arguments and returns the result. @@ -312,38 +366,60 @@ class HelpfulArgumentParser(object): if self.detect_defaults: return parsed_args + self.defaults = dict((key, copy.deepcopy(self.parser.get_default(key))) + for key in vars(parsed_args)) + # Do any post-parsing homework here + if self.verb == "renew" and not parsed_args.dialog_mode: + parsed_args.noninteractive_mode = True + if parsed_args.staging or parsed_args.dry_run: - if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): - conflicts = ["--staging"] if parsed_args.staging else [] - conflicts += ["--dry-run"] if parsed_args.dry_run else [] - raise errors.Error("--server value conflicts with {0}".format( - " and ".join(conflicts))) - - parsed_args.server = constants.STAGING_URI - - if parsed_args.dry_run: - if self.verb not in ["certonly", "renew"]: - raise errors.Error("--dry-run currently only works with the " - "'certonly' or 'renew' subcommands (%r)" % self.verb) - parsed_args.break_my_certs = parsed_args.staging = True - if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): - # The user has a prod account, but might not have a staging - # one; we don't want to start trying to perform interactive registration - parsed_args.tos = True - parsed_args.register_unsafely_without_email = True + self.set_test_server(parsed_args) if parsed_args.csr: - if parsed_args.allow_subset_of_names: - raise errors.Error("--allow-subset-of-names " - "cannot be used with --csr") self.handle_csr(parsed_args) - hooks.validate_hooks(parsed_args) + if parsed_args.must_staple: + parsed_args.staple = True + + # Avoid conflicting args + conficting_args = ["quiet", "noninteractive_mode", "text_mode"] + if parsed_args.dialog_mode: + for arg in conficting_args: + if getattr(parsed_args, arg): + raise errors.Error( + ("Conflicting values for displayer." + " {0} conflicts with dialog_mode").format(arg) + ) + + if parsed_args.validate_hooks: + hooks.validate_hooks(parsed_args) return parsed_args + def set_test_server(self, parsed_args): + """We have --staging/--dry-run; perform sanity check and set config.server""" + + if parsed_args.server not in (flag_default("server"), constants.STAGING_URI): + conflicts = ["--staging"] if parsed_args.staging else [] + conflicts += ["--dry-run"] if parsed_args.dry_run else [] + raise errors.Error("--server value conflicts with {0}".format( + " and ".join(conflicts))) + + parsed_args.server = constants.STAGING_URI + + if parsed_args.dry_run: + if self.verb not in ["certonly", "renew"]: + raise errors.Error("--dry-run currently only works with the " + "'certonly' or 'renew' subcommands (%r)" % self.verb) + parsed_args.break_my_certs = parsed_args.staging = True + if glob.glob(os.path.join(parsed_args.config_dir, constants.ACCOUNTS_DIR, "*")): + # The user has a prod account, but might not have a staging + # one; we don't want to start trying to perform interactive registration + parsed_args.tos = True + parsed_args.register_unsafely_without_email = True + def handle_csr(self, parsed_args): """Process a --csr flag.""" if parsed_args.verb != "certonly": @@ -351,21 +427,11 @@ class HelpfulArgumentParser(object): "when obtaining a new or replacement " "via the certonly command. Please try the " "certonly command instead.") + if parsed_args.allow_subset_of_names: + raise errors.Error("--allow-subset-of-names cannot be used with --csr") - try: - csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="der") - typ = OpenSSL.crypto.FILETYPE_ASN1 - domains = crypto_util.get_sans_from_csr(csr.data, OpenSSL.crypto.FILETYPE_ASN1) - except OpenSSL.crypto.Error: - try: - e1 = traceback.format_exc() - typ = OpenSSL.crypto.FILETYPE_PEM - csr = le_util.CSR(file=parsed_args.csr[0], data=parsed_args.csr[1], form="pem") - domains = crypto_util.get_sans_from_csr(csr.data, typ) - except OpenSSL.crypto.Error: - logger.debug("DER CSR parse error %s", e1) - logger.debug("PEM CSR parse error %s", traceback.format_exc()) - raise errors.Error("Failed to parse CSR file: {0}".format(parsed_args.csr[0])) + csrfile, contents = parsed_args.csr[0:2] + typ, csr, domains = crypto_util.import_csr_file(csrfile, contents) # This is not necessary for webroot to work, however, # obtain_certificate_from_csr requires parsed_args.domains to be set @@ -487,7 +553,7 @@ class HelpfulArgumentParser(object): :param int nargs: Number of arguments the option takes. """ - le_util.add_deprecated_argument( + util.add_deprecated_argument( self.parser.add_argument, argument_name, num_args) def add_group(self, topic, **kwargs): @@ -542,7 +608,7 @@ class HelpfulArgumentParser(object): return dict([(t, t == chosen_topic) for t in self.help_topics]) -def prepare_and_parse_args(plugins, args, detect_defaults=False): +def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: disable=too-many-statements """Returns parsed command line arguments. :param .PluginsRegistry plugins: available plugins @@ -552,6 +618,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): :rtype: argparse.Namespace """ + + # pylint: disable=too-many-statements + helpful = HelpfulArgumentParser(args, plugins, detect_defaults) # --help is automatically provided by argparse @@ -569,6 +638,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): help="Run without ever asking for user input. This may require " "additional command line flags; the client will try to explain " "which ones are required if it finds one missing") + helpful.add( + None, "--dialog", dest="dialog_mode", action="store_true", + help="Run using dialog") helpful.add( None, "--dry-run", action="store_true", dest="dry_run", help="Perform a test run of the client, obtaining test (invalid) certs" @@ -591,6 +663,11 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "certificates. Updates to the Subscriber Agreement will still " "affect you, and will be effective 14 days after posting an " "update to the web site.") + helpful.add( + "register", "--update-registration", action="store_true", + help="With the register verb, indicates that details associated " + "with an existing registration, such as the e-mail address, " + "should be updated, rather than registering a new account.") helpful.add(None, "-m", "--email", help=config_help("email")) # positional arg shadows --domains, instead of appending, and # --domains is useful, because it can be stored in config @@ -682,6 +759,9 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): helpful.add( "security", "--rsa-key-size", type=int, metavar="N", default=flag_default("rsa_key_size"), help=config_help("rsa_key_size")) + helpful.add( + "security", "--must-staple", action="store_true", + help=config_help("must_staple"), dest="must_staple", default=False) helpful.add( "security", "--redirect", action="store_true", help="Automatically redirect all HTTP traffic to HTTPS for the newly " @@ -693,7 +773,7 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): helpful.add( "security", "--hsts", action="store_true", help="Add the Strict-Transport-Security header to every HTTP response." - " Forcing browser to use always use SSL for the domain." + " Forcing browser to always use SSL for the domain." " Defends against SSL Stripping.", dest="hsts", default=False) helpful.add( "security", "--no-hsts", action="store_false", @@ -706,9 +786,20 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): " https:// for every http:// resource.", dest="uir", default=None) helpful.add( "security", "--no-uir", action="store_false", - help=" Do not automatically set the \"Content-Security-Policy:" + help="Do not automatically set the \"Content-Security-Policy:" " upgrade-insecure-requests\" header to every HTTP response.", dest="uir", default=None) + helpful.add( + "security", "--staple-ocsp", action="store_true", + help="Enables OCSP Stapling. A valid OCSP response is stapled to" + " the certificate that the server offers during TLS.", + dest="staple", default=None) + helpful.add( + "security", "--no-staple-ocsp", action="store_false", + help="Do not automatically enable OCSP Stapling.", + dest="staple", default=None) + + helpful.add( "security", "--strict-permissions", action="store_true", help="Require that all configuration files are owned by the current " @@ -723,7 +814,8 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): " certificate lineage. You can try it with `--dry-run` first. For" " more fine-grained control, you can renew individual lineages with" " the `certonly` subcommand. Hooks are available to run commands " - " before and after renewal; see XXX for more information on these.") + " before and after renewal; see" + " https://certbot.eff.org/docs/using.html#renewal for more information on these.") helpful.add( "renew", "--pre-hook", @@ -735,13 +827,22 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "renew", "--post-hook", help="Command to be run in a shell after attempting to obtain/renew " " certificates. Can be used to deploy renewed certificates, or to restart" - " any servers that were stopped by --pre-hook.") + " any servers that were stopped by --pre-hook. This is only run if" + " an attempt was made to obtain/renew a certificate.") helpful.add( "renew", "--renew-hook", help="Command to be run in a shell once for each successfully renewed certificate." "For this command, the shell variable $RENEWED_LINEAGE will point to the" "config live subdirectory containing the new certs and keys; the shell variable " "$RENEWED_DOMAINS will contain a space-delimited list of renewed cert domains") + helpful.add( + "renew", "--disable-hook-validation", + action='store_false', dest='validate_hooks', default=True, + help="Ordinarily the commands specified for --pre-hook/--post-hook/--renew-hook" + " will be checked for validity, to see if the programs being run are in the $PATH," + " so that mistakes can be caught early, even when the hooks aren't being run just yet." + " The validation is rather simplistic and fails if you use more advanced" + " shell constructs, so you can use this switch to disable it.") helpful.add_deprecated_argument("--agree-dev-preview", 0) @@ -904,7 +1005,7 @@ def add_domains(args_or_config, domains): """ validated_domains = [] for domain in domains.split(","): - domain = le_util.enforce_domain_sanity(domain.strip()) + domain = util.enforce_domain_sanity(domain.strip()) validated_domains.append(domain) if domain not in args_or_config.domains: args_or_config.domains.append(domain) diff --git a/certbot/client.py b/certbot/client.py index 6f41a3a0b..81b7ccdc6 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -21,9 +21,10 @@ from certbot import crypto_util from certbot import errors from certbot import error_handler from certbot import interfaces -from certbot import le_util +from certbot import util from certbot import reverter from certbot import storage +from certbot import cli from certbot.display import ops as display_ops from certbot.display import enhancements @@ -52,7 +53,7 @@ def _determine_user_agent(config): if config.user_agent is None: ua = "CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3}" - ua = ua.format(certbot.__version__, " ".join(le_util.get_os_info()), + ua = ua.format(certbot.__version__, util.get_os_info_ua(), config.authenticator, config.installer) else: ua = config.user_agent @@ -149,7 +150,7 @@ def perform_registration(acme, config): return acme.register(messages.NewRegistration.from_data(email=config.email)) except messages.Error as e: if e.typ == "urn:acme:error:invalidEmail": - config.namespace.email = display_ops.get_email(more=True, invalid=True) + config.namespace.email = display_ops.get_email(invalid=True) return perform_registration(acme, config) else: raise @@ -197,7 +198,7 @@ class Client(object): consistent with identifiers present in the `csr`. :param list domains: Domain names. - :param .le_util.CSR csr: DER-encoded Certificate Signing + :param .util.CSR csr: DER-encoded Certificate Signing Request. The key used to generate this CSR can be different than `authkey`. :param list authzr: List of @@ -236,8 +237,8 @@ class Client(object): :returns: `.CertificateResource`, certificate chain (as returned by `.fetch_chain`), and newly generated private key - (`.le_util.Key`) and DER-encoded Certificate Signing Request - (`.le_util.CSR`). + (`.util.Key`) and DER-encoded Certificate Signing Request + (`.util.CSR`). :rtype: tuple """ @@ -281,7 +282,7 @@ class Client(object): "by your operating system package manager") if self.config.dry_run: - logger.info("Dry run: Skipping creating new lineage for %s", + logger.debug("Dry run: Skipping creating new lineage for %s", domains[0]) return None else: @@ -311,29 +312,36 @@ class Client(object): """ for path in cert_path, chain_path, fullchain_path: - le_util.make_or_verify_dir( + util.make_or_verify_dir( os.path.dirname(path), 0o755, os.geteuid(), self.config.strict_permissions) cert_pem = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, certr.body.wrapped) - cert_file, act_cert_path = le_util.unique_file(cert_path, 0o644) + + cert_file, abs_cert_path = _open_pem_file('cert_path', cert_path) + try: cert_file.write(cert_pem) finally: cert_file.close() logger.info("Server issued certificate; certificate written to %s", - act_cert_path) + abs_cert_path) - cert_chain_abspath = None - fullchain_abspath = None - if chain_cert: + if not chain_cert: + return abs_cert_path, None, None + else: chain_pem = crypto_util.dump_pyopenssl_chain(chain_cert) - cert_chain_abspath = _save_chain(chain_pem, chain_path) - fullchain_abspath = _save_chain(cert_pem + chain_pem, - fullchain_path) - return os.path.abspath(act_cert_path), cert_chain_abspath, fullchain_abspath + chain_file, abs_chain_path =\ + _open_pem_file('chain_path', chain_path) + fullchain_file, abs_fullchain_path =\ + _open_pem_file('fullchain_path', fullchain_path) + + _save_chain(chain_pem, chain_file) + _save_chain(cert_pem + chain_pem, fullchain_file) + + return abs_cert_path, abs_chain_path, abs_fullchain_path def deploy_certificate(self, domains, privkey_path, cert_path, chain_path, fullchain_path): @@ -396,7 +404,8 @@ class Client(object): supported = self.installer.supported_enhancements() redirect = config.redirect if "redirect" in supported else False hsts = config.hsts if "ensure-http-header" in supported else False - uir = config.uir if "ensure-http-header" in supported else False + uir = config.uir if "ensure-http-header" in supported else False + staple = config.staple if "staple-ocsp" in supported else False if redirect is None: redirect = enhancements.ask("redirect") @@ -410,9 +419,11 @@ class Client(object): if uir: self.apply_enhancement(domains, "ensure-http-header", "Upgrade-Insecure-Requests") + if staple: + self.apply_enhancement(domains, "staple-ocsp") msg = ("We were unable to restart web server") - if redirect or hsts or uir: + if redirect or hsts or uir or staple: with error_handler.ErrorHandler(self._rollback_and_restart, msg): self.installer.restart() @@ -493,9 +504,9 @@ def validate_key_csr(privkey, csr=None): If csr is left as None, only the key will be validated. :param privkey: Key associated with CSR - :type privkey: :class:`certbot.le_util.Key` + :type privkey: :class:`certbot.util.Key` - :param .le_util.CSR csr: CSR + :param .util.CSR csr: CSR :raises .errors.Error: when validation fails @@ -512,7 +523,7 @@ def validate_key_csr(privkey, csr=None): if csr.form == "der": csr_obj = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_ASN1, csr.data) - csr = le_util.CSR(csr.file, OpenSSL.crypto.dump_certificate( + csr = util.CSR(csr.file, OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, csr_obj), "pem") # If CSR is provided, it must be readable and valid. @@ -562,24 +573,35 @@ def view_config_changes(config, num=None): rev.recovery_routine() rev.view_config_changes(num) +def _open_pem_file(cli_arg_path, pem_path): + """Open a pem file. -def _save_chain(chain_pem, chain_path): + If cli_arg_path was set by the client, open that. + Otherwise, uniquify the file path. + + :param str cli_arg_path: the cli arg name, e.g. cert_path + :param str pem_path: the pem file path to open + + :returns: a tuple of file object and its absolute file path + + """ + if cli.set_by_cli(cli_arg_path): + return util.safe_open(pem_path, chmod=0o644),\ + os.path.abspath(pem_path) + else: + uniq = util.unique_file(pem_path, 0o644) + return uniq[0], os.path.abspath(uniq[1]) + +def _save_chain(chain_pem, chain_file): """Saves chain_pem at a unique path based on chain_path. :param str chain_pem: certificate chain in PEM format - :param str chain_path: candidate path for the cert chain - - :returns: absolute path to saved cert chain - :rtype: str + :param str chain_file: chain file object """ - chain_file, act_chain_path = le_util.unique_file(chain_path, 0o644) try: chain_file.write(chain_pem) finally: chain_file.close() - logger.info("Cert chain written to %s", act_chain_path) - - # This expects a valid chain file - return os.path.abspath(act_chain_path) + logger.info("Cert chain written to %s", chain_file.name) diff --git a/certbot/colored_logging.py b/certbot/colored_logging.py index d42fb5966..93bf3a55a 100644 --- a/certbot/colored_logging.py +++ b/certbot/colored_logging.py @@ -2,7 +2,7 @@ import logging import sys -from certbot import le_util +from certbot import util class StreamHandler(logging.StreamHandler): @@ -40,6 +40,6 @@ class StreamHandler(logging.StreamHandler): if sys.version_info < (2, 7) else super(StreamHandler, self).format(record)) if self.colored and record.levelno >= self.red_level: - return ''.join((le_util.ANSI_SGR_RED, out, le_util.ANSI_SGR_RESET)) + return ''.join((util.ANSI_SGR_RED, out, util.ANSI_SGR_RESET)) else: return out diff --git a/certbot/configuration.py b/certbot/configuration.py index 172b35bfe..712135b8d 100644 --- a/certbot/configuration.py +++ b/certbot/configuration.py @@ -8,7 +8,7 @@ import zope.interface from certbot import constants from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util @zope.interface.implementer(interfaces.IConfig) @@ -132,4 +132,4 @@ def check_config_sanity(config): if config.namespace.domains is not None: for domain in config.namespace.domains: # This may be redundant, but let's be paranoid - le_util.enforce_domain_sanity(domain) + util.enforce_domain_sanity(domain) diff --git a/certbot/constants.py b/certbot/constants.py index 1d4efe80e..fb278161d 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -18,7 +18,7 @@ CLI_DEFAULTS = dict( os.path.join(os.environ.get("XDG_CONFIG_HOME", "~/.config"), "letsencrypt", "cli.ini"), ], - verbose_count=-(logging.WARNING / 10), + verbose_count=-(logging.INFO / 10), server="https://acme-v01.api.letsencrypt.org/directory", rsa_key_size=2048, rollback_checkpoints=1, diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index b699ce653..1e831dd8f 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -6,6 +6,7 @@ """ import logging import os +import traceback import OpenSSL import pyrfc3339 @@ -16,7 +17,7 @@ from acme import jose from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util logger = logging.getLogger(__name__) @@ -36,7 +37,7 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): :param str keyname: Filename of key :returns: Key - :rtype: :class:`certbot.le_util.Key` + :rtype: :class:`certbot.util.Key` :raises ValueError: If unable to generate the key given key_size. @@ -49,50 +50,51 @@ def init_save_key(key_size, key_dir, keyname="key-certbot.pem"): config = zope.component.getUtility(interfaces.IConfig) # Save file - le_util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), - config.strict_permissions) - key_f, key_path = le_util.unique_file( - os.path.join(key_dir, keyname), 0o600) + util.make_or_verify_dir(key_dir, 0o700, os.geteuid(), + config.strict_permissions) + key_f, key_path = util.unique_file(os.path.join(key_dir, keyname), 0o600) with key_f: key_f.write(key_pem) logger.info("Generating key (%d bits): %s", key_size, key_path) - return le_util.Key(key_path, key_pem) + return util.Key(key_path, key_pem) def init_save_csr(privkey, names, path, csrname="csr-certbot.pem"): """Initialize a CSR with the given private key. :param privkey: Key to include in the CSR - :type privkey: :class:`certbot.le_util.Key` + :type privkey: :class:`certbot.util.Key` :param set names: `str` names to include in the CSR :param str path: Certificate save directory. :returns: CSR - :rtype: :class:`certbot.le_util.CSR` + :rtype: :class:`certbot.util.CSR` """ - csr_pem, csr_der = make_csr(privkey.pem, names) - config = zope.component.getUtility(interfaces.IConfig) + + csr_pem, csr_der = make_csr(privkey.pem, names, + must_staple=config.must_staple) + # Save CSR - le_util.make_or_verify_dir(path, 0o755, os.geteuid(), + util.make_or_verify_dir(path, 0o755, os.geteuid(), config.strict_permissions) - csr_f, csr_filename = le_util.unique_file( + csr_f, csr_filename = util.unique_file( os.path.join(path, csrname), 0o644) csr_f.write(csr_pem) csr_f.close() logger.info("Creating CSR: %s", csr_filename) - return le_util.CSR(csr_filename, csr_der, "der") + return util.CSR(csr_filename, csr_der, "der") # Lower level functions -def make_csr(key_str, domains): +def make_csr(key_str, domains, must_staple=False): """Generate a CSR. :param str key_str: PEM-encoded RSA key. @@ -111,13 +113,19 @@ def make_csr(key_str, domains): req.get_subject().CN = domains[0] # TODO: what to put into req.get_subject()? # TODO: put SAN if len(domains) > 1 - req.add_extensions([ + extensions = [ OpenSSL.crypto.X509Extension( "subjectAltName", critical=False, value=", ".join("DNS:%s" % d for d in domains) - ), - ]) + ) + ] + if must_staple: + extensions.append(OpenSSL.crypto.X509Extension( + "1.3.6.1.5.5.7.1.24", + critical=False, + value="DER:30:03:02:01:05")) + req.add_extensions(extensions) req.set_version(2) req.set_pubkey(pkey) req.sign(pkey, "sha256") @@ -171,6 +179,30 @@ def csr_matches_pubkey(csr, privkey): return False +def import_csr_file(csrfile, data): + """Import a CSR file, which can be either PEM or DER. + + :param str csrfile: CSR filename + :param str data: contents of the CSR file + + :returns: (`OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1`, + util.CSR object representing the CSR, + list of domains requested in the CSR) + :rtype: tuple + + """ + for form, typ in (("der", OpenSSL.crypto.FILETYPE_ASN1,), + ("pem", OpenSSL.crypto.FILETYPE_PEM,),): + try: + domains = get_names_from_csr(data, typ) + except OpenSSL.crypto.Error: + logger.debug("CSR parse error (form=%s, typ=%s):", form, typ) + logger.debug(traceback.format_exc()) + continue + return typ, util.CSR(file=csrfile, data=data, form=form), domains + raise errors.Error("Failed to parse CSR file: {0}".format(csrfile)) + + def make_key(bits): """Generate PEM encoded RSA key. @@ -220,15 +252,20 @@ def pyopenssl_load_certificate(data): str(error) for error in openssl_errors))) -def _get_sans_from_cert_or_req(cert_or_req_str, load_func, - typ=OpenSSL.crypto.FILETYPE_PEM): +def _load_cert_or_req(cert_or_req_str, load_func, + typ=OpenSSL.crypto.FILETYPE_PEM): try: - cert_or_req = load_func(typ, cert_or_req_str) + return load_func(typ, cert_or_req_str) except OpenSSL.crypto.Error as error: logger.exception(error) raise + + +def _get_sans_from_cert_or_req(cert_or_req_str, load_func, + typ=OpenSSL.crypto.FILETYPE_PEM): # pylint: disable=protected-access - return acme_crypto_util._pyopenssl_cert_or_req_san(cert_or_req) + return acme_crypto_util._pyopenssl_cert_or_req_san(_load_cert_or_req( + cert_or_req_str, load_func, typ)) def get_sans_from_cert(cert, typ=OpenSSL.crypto.FILETYPE_PEM): @@ -259,6 +296,46 @@ def get_sans_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): csr, OpenSSL.crypto.load_certificate_request, typ) +def _get_names_from_cert_or_req(cert_or_req, load_func, typ): + loaded_cert_or_req = _load_cert_or_req(cert_or_req, load_func, typ) + common_name = loaded_cert_or_req.get_subject().CN + # pylint: disable=protected-access + sans = acme_crypto_util._pyopenssl_cert_or_req_san(loaded_cert_or_req) + + if common_name is None: + return sans + else: + return [common_name] + [d for d in sans if d != common_name] + + +def get_names_from_cert(csr, typ=OpenSSL.crypto.FILETYPE_PEM): + """Get a list of domains from a cert, including the CN if it is set. + + :param str cert: Certificate (encoded). + :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + + :returns: A list of domain names. + :rtype: list + + """ + return _get_names_from_cert_or_req( + csr, OpenSSL.crypto.load_certificate, typ) + + +def get_names_from_csr(csr, typ=OpenSSL.crypto.FILETYPE_PEM): + """Get a list of domains from a CSR, including the CN if it is set. + + :param str csr: CSR (encoded). + :param typ: `OpenSSL.crypto.FILETYPE_PEM` or `OpenSSL.crypto.FILETYPE_ASN1` + + :returns: A list of domain names. + :rtype: list + + """ + return _get_names_from_cert_or_req( + csr, OpenSSL.crypto.load_certificate_request, typ) + + def dump_pyopenssl_chain(chain, filetype=OpenSSL.crypto.FILETYPE_PEM): """Dump certificate chain into a bundle. diff --git a/certbot/display/ops.py b/certbot/display/ops.py index 6752bf0c1..c7e566256 100644 --- a/certbot/display/ops.py +++ b/certbot/display/ops.py @@ -6,7 +6,7 @@ import zope.component from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util from certbot.display import util as display_util logger = logging.getLogger(__name__) @@ -15,41 +15,56 @@ logger = logging.getLogger(__name__) z_util = zope.component.getUtility -def get_email(more=False, invalid=False): +def get_email(invalid=False, optional=True): """Prompt for valid email address. - :param bool more: explain why the email is strongly advisable, but how to - skip it - :param bool invalid: true if the user just typed something, but it wasn't - a valid-looking email + :param bool invalid: True if an invalid address was provided by the user + :param bool optional: True if the user can use + --register-unsafely-without-email to avoid providing an e-mail - :returns: Email or ``None`` if cancelled by user. + :returns: e-mail address :rtype: str - """ - msg = "Enter email address (used for urgent notices and lost key recovery)" - if invalid: - msg = "There seem to be problems with that address. " + msg - if more: - msg += ('\n\nIf you really want to skip this, you can run the client with ' - '--register-unsafely-without-email but make sure you backup your ' - 'account key from /etc/letsencrypt/accounts\n\n') - try: - code, email = zope.component.getUtility(interfaces.IDisplay).input(msg) - except errors.MissingCommandlineFlag: - msg = ("You should register before running non-interactively, or provide --agree-tos" - " and --email flags") - raise errors.MissingCommandlineFlag(msg) + :raises errors.Error: if the user cancels - if code == display_util.OK: - if le_util.safe_email(email): - return email + """ + invalid_prefix = "There seem to be problems with that address. " + msg = "Enter email address (used for urgent notices and lost key recovery)" + unsafe_suggestion = ("\n\nIf you really want to skip this, you can run " + "the client with --register-unsafely-without-email " + "but make sure you then backup your account key from " + "/etc/letsencrypt/accounts\n\n") + if optional: + if invalid: + msg += unsafe_suggestion else: - # TODO catch the server's ACME invalid email address error, and - # make a similar call when that happens - return get_email(more=True, invalid=(email != "")) + suggest_unsafe = True else: - return None + suggest_unsafe = False + + while True: + try: + code, email = z_util(interfaces.IDisplay).input( + invalid_prefix + msg if invalid else msg) + except errors.MissingCommandlineFlag: + msg = ("You should register before running non-interactively, " + "or provide --agree-tos and --email flags") + raise errors.MissingCommandlineFlag(msg) + + if code != display_util.OK: + if optional: + raise errors.Error( + "An e-mail address or " + "--register-unsafely-without-email must be provided.") + else: + raise errors.Error("An e-mail address must be provided.") + elif util.safe_email(email): + return email + elif suggest_unsafe: + msg += unsafe_suggestion + suggest_unsafe = False # add this message at most once + + invalid = bool(email) def choose_account(accounts): @@ -119,7 +134,7 @@ def get_valid_domains(domains): valid_domains = [] for domain in domains: try: - valid_domains.append(le_util.enforce_domain_sanity(domain)) + valid_domains.append(util.enforce_domain_sanity(domain)) except errors.ConfigurationError: continue return valid_domains @@ -163,7 +178,7 @@ def _choose_names_manually(): for i, domain in enumerate(domain_list): try: - domain_list[i] = le_util.enforce_domain_sanity(domain) + domain_list[i] = util.enforce_domain_sanity(domain) except errors.ConfigurationError as e: invalid_domains[domain] = e.message diff --git a/certbot/display/util.py b/certbot/display/util.py index 8de607534..b4004997f 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -41,10 +41,15 @@ def _wrap_lines(msg): """ lines = msg.splitlines() fixed_l = [] - for line in lines: - fixed_l.append(textwrap.fill(line, 80)) - return os.linesep.join(fixed_l) + for line in lines: + fixed_l.append(textwrap.fill( + line, + 80, + break_long_words=False, + break_on_hyphens=False)) + + return os.linesep.join(fixed_l) @zope.interface.implementer(interfaces.IDisplay) class NcursesDisplay(object): @@ -265,7 +270,11 @@ class FileDisplay(object): """ ans = raw_input( - textwrap.fill("%s (Enter 'c' to cancel): " % message, 80)) + textwrap.fill( + "%s (Enter 'c' to cancel): " % message, + 80, + break_long_words=False, + break_on_hyphens=False)) if ans == "c" or ans == "C": return CANCEL, "-1" @@ -402,7 +411,11 @@ class FileDisplay(object): # Write out the menu choices for i, desc in enumerate(choices, 1): self.outfile.write( - textwrap.fill("{num}: {desc}".format(num=i, desc=desc), 80)) + textwrap.fill( + "{num}: {desc}".format(num=i, desc=desc), + 80, + break_long_words=False, + break_on_hyphens=False)) # Keep this outside of the textwrap self.outfile.write(os.linesep) diff --git a/certbot/hooks.py b/certbot/hooks.py index 138e2addc..1a3e4a98e 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -27,7 +27,7 @@ def _validate_hook(shell_cmd, hook_name): :raises .errors.HookCommandNotFound: if the command is not found """ if shell_cmd: - cmd = shell_cmd.partition(" ")[0] + cmd = shell_cmd.split(None, 1)[0] if not _prog(cmd): path = os.environ["PATH"] msg = "Unable to find {2}-hook command {0} in the PATH.\n(PATH is {1})".format( @@ -39,7 +39,7 @@ def pre_hook(config): if config.pre_hook and not pre_hook.already: logger.info("Running pre-hook command: %s", config.pre_hook) _run_hook(config.pre_hook) - pre_hook.already = True + pre_hook.already = True pre_hook.already = False @@ -50,6 +50,11 @@ def post_hook(config, final=False): we're called with final=True before actually doing anything. """ if config.post_hook: + if not pre_hook.already: + logger.info("No renewals attempted, so not running post-hook") + if config.verb != "renew": + logger.warn("Sanity failure in renewal hooks") + return if final or config.verb != "renew": logger.info("Running post-hook command: %s", config.post_hook) _run_hook(config.post_hook) diff --git a/certbot/interfaces.py b/certbot/interfaces.py index d65f5cf01..e4e62e0a2 100644 --- a/certbot/interfaces.py +++ b/certbot/interfaces.py @@ -180,6 +180,9 @@ class IAuthenticator(IPlugin): def cleanup(achalls): """Revert changes and shutdown after challenges complete. + This method should be able to revert all changes made by + perform, even if perform exited abnormally. + :param list achalls: Non-empty (guaranteed) list of :class:`~certbot.achallenges.AnnotatedChallenge` instances, a subset of those previously passed to :func:`perform`. @@ -200,6 +203,10 @@ class IConfig(zope.interface.Interface): email = zope.interface.Attribute( "Email used for registration and recovery contact.") rsa_key_size = zope.interface.Attribute("Size of the RSA key.") + must_staple = zope.interface.Attribute( + "Adds the OCSP Must Staple extension to the certificate. " + "Autoconfigures OCSP Stapling for supported setups " + "(Apache version >= 2.3.3 ).") config_dir = zope.interface.Attribute("Configuration directory.") work_dir = zope.interface.Attribute("Working directory.") @@ -234,6 +241,14 @@ class IInstaller(IPlugin): Represents any server that an X509 certificate can be placed. + It is assumed that :func:`save` is the only method that finalizes a + checkpoint. This is important to ensure that checkpoints are + restored in a consistent manner if requested by the user or in case + of an error. + + Using :class:`certbot.reverter.Reverter` to implement checkpoints, + rollback, and recovery can dramatically simplify plugin development. + """ def get_all_names(): @@ -300,8 +315,11 @@ class IInstaller(IPlugin): Both title and temporary are needed because a save may be intended to be permanent, but the save is not ready to be a full - checkpoint. If an exception is raised, it is assumed a new - checkpoint was not created. + checkpoint. + + It is assumed that at most one checkpoint is finalized by this + method. Additionally, if an exception is raised, it is assumed a + new checkpoint was not finalized. :param str title: The title of the save. If a title is given, the configuration will be saved as a new checkpoint and put in a diff --git a/certbot/main.py b/certbot/main.py index 309889e8e..be68d694e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -1,6 +1,8 @@ """Certbot main entry point.""" from __future__ import print_function import atexit +import dialog +import errno import functools import logging.handlers import os @@ -24,7 +26,7 @@ from certbot import constants from certbot import errors from certbot import hooks from certbot import interfaces -from certbot import le_util +from certbot import util from certbot import log from certbot import reporter from certbot import renewal @@ -82,22 +84,25 @@ def _auth_from_domains(le_client, config, domains, lineage=None): if action == "reinstall": # The lineage already exists; allow the caller to try installing # it without getting a new certificate at all. + logger.info("Keeping the existing certificate") return lineage, "reinstall" hooks.pre_hook(config) try: if action == "renew": + logger.info("Renewing an existing certificate") renewal.renew_cert(config, domains, le_client, lineage) elif action == "newcert": # TREAT AS NEW REQUEST + logger.info("Obtaining a new certificate") lineage = le_client.obtain_and_enroll_certificate(domains) if lineage is False: raise errors.Error("Certificate could not be obtained") finally: - hooks.post_hook(config) + hooks.post_hook(config, final=False) if not config.dry_run and not config.verb == "renew": - _report_new_cert(lineage.cert, lineage.fullchain) + _report_new_cert(config, lineage.cert, lineage.fullchain) return lineage, action @@ -229,7 +234,7 @@ def _find_duplicative_certs(config, domains): cli_config = configuration.RenewerConfiguration(config) configs_dir = cli_config.renewal_configs_dir # Verify the directory is there - le_util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) + util.make_or_verify_dir(configs_dir, mode=0o755, uid=os.geteuid()) for renewal_file in renewal.renewal_conf_files(cli_config): try: @@ -267,7 +272,7 @@ def _find_domains(config, installer): return domains -def _report_new_cert(cert_path, fullchain_path): +def _report_new_cert(config, cert_path, fullchain_path): """Reports the creation of a new certificate to the user. :param str cert_path: path to cert @@ -285,12 +290,15 @@ def _report_new_cert(cert_path, fullchain_path): # Unless we're in .csr mode and there really isn't one and_chain = "has " path = cert_path + + verbswitch = ' with the "certonly" option' if config.verb == "run" else "" # XXX Perhaps one day we could detect the presence of known old webservers # and say something more informative here. - msg = ("Congratulations! Your certificate {0} been saved at {1}." - " Your cert will expire on {2}. To obtain a new version of the " - "certificate in the future, simply run Certbot again." - .format(and_chain, path, expiry)) + msg = ('Congratulations! Your certificate {0} been saved at {1}.' + ' Your cert will expire on {2}. To obtain a new or tweaked version of this ' + 'certificate in the future, simply run {3} again{4}. ' + 'To non-interactively renew *all* of your certificates, run "{3} renew"' + .format(and_chain, path, expiry, cli.cli_command, verbswitch)) reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY) @@ -363,6 +371,48 @@ def _init_le_client(config, authenticator, installer): return client.Client(config, acc, authenticator, installer, acme=acme) +def register(config, unused_plugins): + """Create or modify accounts on the server.""" + + # Portion of _determine_account logic to see whether accounts already + # exist or not. + account_storage = account.AccountFileStorage(config) + accounts = account_storage.find_all() + + # registering a new account + if not config.update_registration: + if len(accounts) > 0: + # TODO: add a flag to register a duplicate account (this will + # also require extending _determine_account's behavior + # or else extracting the registration code from there) + return ("There is an existing account; registration of a " + "duplicate account with this command is currently " + "unsupported.") + # _determine_account will register an account + _determine_account(config) + return + + # --update-registration + if len(accounts) == 0: + return "Could not find an existing account to update." + if config.email is None: + if config.register_unsafely_without_email: + return ("--register-unsafely-without-email provided, however, a " + "new e-mail address must\ncurrently be provided when " + "updating a registration.") + config.namespace.email = display_ops.get_email(optional=False) + + acc, acme = _determine_account(config) + acme_client = client.Client(config, acc, None, None, acme=acme) + # We rely on an exception to interrupt this process if it didn't work. + acc.regr = acme_client.acme.update_registration(acc.regr.update( + body=acc.regr.body.update(contact=('mailto:' + config.email,)))) + account_storage.save_regr(acc) + reporter_util = zope.component.getUtility(interfaces.IReporter) + msg = "Your e-mail address was updated to {0}.".format(config.email) + reporter_util.add_message(msg, reporter_util.MEDIUM_PRIORITY) + + def install(config, plugins): """Install a previously obtained cert in a server.""" # XXX: Update for renewer/RenewableCert @@ -480,12 +530,12 @@ def _csr_obtain_cert(config, le_client): csr, typ = config.actual_csr certr, chain = le_client.obtain_certificate_from_csr(config.domains, csr, typ) if config.dry_run: - logger.info( + logger.debug( "Dry run: skipping saving certificate to %s", config.cert_path) else: cert_path, _, cert_fullchain = le_client.save_certificate( certr, chain, config.cert_path, config.chain_path, config.fullchain_path) - _report_new_cert(cert_path, cert_fullchain) + _report_new_cert(config, cert_path, cert_fullchain) def obtain_cert(config, plugins, lineage=None): @@ -543,8 +593,16 @@ def renew(config, unused_plugins): def setup_log_file_handler(config, logfile, fmt): """Setup file debug logging.""" log_file_path = os.path.join(config.logs_dir, logfile) - handler = logging.handlers.RotatingFileHandler( - log_file_path, maxBytes=2 ** 20, backupCount=10) + try: + handler = logging.handlers.RotatingFileHandler( + log_file_path, maxBytes=2 ** 20, backupCount=10) + except IOError as e: + if e.errno == errno.EACCES: + msg = ("Access denied writing to {0}. To run as non-root, set " + + "--logs-dir, --config-dir, --work-dir to writable paths.") + raise errors.Error(msg.format(log_file_path)) + else: + raise # rotate on each invocation, rollover only possible when maxBytes # is nonzero and backupCount is nonzero, so we set maxBytes as big # as possible not to overrun in single CLI invocation (1MB). @@ -570,11 +628,12 @@ def _cli_log_handler(config, level, fmt): def setup_logging(config, cli_handler_factory, logfile): """Setup logging.""" - fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + file_fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + cli_fmt = "%(message)s" level = -config.verbose_count * 10 file_handler, log_file_path = setup_log_file_handler( - config, logfile=logfile, fmt=fmt) - cli_handler = cli_handler_factory(config, level, fmt) + config, logfile=logfile, fmt=file_fmt) + cli_handler = cli_handler_factory(config, level, cli_fmt) # TODO: use fileConfig? @@ -620,7 +679,10 @@ def _handle_exception(exc_type, exc_value, trace, config): # Here we're passing a client or ACME error out to the client at the shell # Tell the user a bit about what happened, without overwhelming # them with a full traceback - err = traceback.format_exception_only(exc_type, exc_value)[0] + if issubclass(exc_type, dialog.error): + err = exc_value.complete_message() + else: + err = traceback.format_exception_only(exc_type, exc_value)[0] # Typical error from the ACME module: # acme.messages.Error: urn:acme:error:malformed :: The request message was # malformed :: Error creating new registration :: Validation of contact @@ -653,14 +715,15 @@ def main(cli_args=sys.argv[1:]): # Setup logging ASAP, otherwise "No handlers could be found for # logger ..." TODO: this should be done before plugins discovery for directory in config.config_dir, config.work_dir: - le_util.make_or_verify_dir( + util.make_or_verify_dir( directory, constants.CONFIG_DIRS_MODE, os.geteuid(), "--strict-permissions" in cli_args) # TODO: logs might contain sensitive data such as contents of the # private key! #525 - le_util.make_or_verify_dir( + util.make_or_verify_dir( config.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) setup_logging(config, _cli_log_handler, logfile='letsencrypt.log') + cli.possible_deprecation_warning(config) logger.debug("certbot version: %s", certbot.__version__) # do not log `config`, as it contains sensitive data (e.g. revoke --key)! @@ -677,9 +740,6 @@ def main(cli_args=sys.argv[1:]): displayer = display_util.NoninteractiveDisplay(sys.stdout) elif config.text_mode: displayer = display_util.FileDisplay(sys.stdout) - elif config.verb == "renew": - config.noninteractive_mode = True - displayer = display_util.NoninteractiveDisplay(sys.stdout) else: displayer = display_util.NcursesDisplay() zope.component.provideUtility(displayer) diff --git a/certbot/plugins/common.py b/certbot/plugins/common.py index 757bf19d8..007105c7b 100644 --- a/certbot/plugins/common.py +++ b/certbot/plugins/common.py @@ -12,7 +12,7 @@ from acme.jose import util as jose_util from certbot import constants from certbot import interfaces -from certbot import le_util +from certbot import util def option_namespace(name): @@ -255,7 +255,7 @@ class TLSSNI01(object): # Write out challenge cert and key with open(cert_path, "wb") as cert_chall_fd: cert_chall_fd.write(cert_pem) - with le_util.safe_open(key_path, 'wb', chmod=0o400) as key_file: + with util.safe_open(key_path, 'wb', chmod=0o400) as key_file: key_file.write(key_pem) return response diff --git a/certbot/plugins/common_test.py b/certbot/plugins/common_test.py index 0dd1cd522..f3ea714c4 100644 --- a/certbot/plugins/common_test.py +++ b/certbot/plugins/common_test.py @@ -193,7 +193,7 @@ class TLSSNI01Test(unittest.TestCase): with mock.patch("certbot.plugins.common.open", mock_open, create=True): - with mock.patch("certbot.plugins.common.le_util.safe_open", + with mock.patch("certbot.plugins.common.util.safe_open", mock_safe_open): # pylint: disable=protected-access self.assertEqual(response, self.sni._setup_challenge_cert( diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index a3bb1d8f0..8e1cb72a4 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -120,6 +120,14 @@ def supported_challenges_validator(data): """ challs = data.split(",") + + # tls-sni-01 was dvsni during private beta + if "dvsni" in challs: + logger.info("Updating legacy standalone_supported_challenges value") + challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall + for chall in challs] + data = ",".join(challs) + unrecognized = [name for name in challs if name not in challenges.Challenge.TYPES] if unrecognized: diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 9f5b14591..eb6631732 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -85,6 +85,11 @@ class SupportedChallengesValidatorTest(unittest.TestCase): def test_not_subset(self): self.assertRaises(argparse.ArgumentTypeError, self._call, "dns") + def test_dvsni(self): + self.assertEqual("tls-sni-01", self._call("dvsni")) + self.assertEqual("http-01,tls-sni-01", self._call("http-01,dvsni")) + self.assertEqual("tls-sni-01,http-01", self._call("dvsni,http-01")) + class AuthenticatorTest(unittest.TestCase): """Tests for certbot.plugins.standalone.Authenticator.""" diff --git a/certbot/plugins/webroot.py b/certbot/plugins/webroot.py index fbe703f40..624ee2ff4 100644 --- a/certbot/plugins/webroot.py +++ b/certbot/plugins/webroot.py @@ -181,12 +181,8 @@ to serve all files under specified web root ({0}).""" os.chown(self.full_roots[name], stat_path.st_uid, stat_path.st_gid) except OSError as exception: - if exception.errno == errno.EACCES: - logger.debug("Insufficient permissions to change owner and uid - ignoring") - else: - raise errors.PluginError( - "Couldn't create root for {0} http-01 " - "challenge responses: {1}", name, exception) + logger.info("Unable to change owner and uid of webroot directory") + logger.debug("Error was: %s", exception) except OSError as exception: if exception.errno != errno.EEXIST: @@ -235,14 +231,9 @@ to serve all files under specified web root ({0}).""" logger.debug("All challenges cleaned up, removing %s", root_path) except OSError as exc: - if exc.errno == errno.ENOTEMPTY: - logger.debug("Challenges cleaned up but %s not empty", - root_path) - elif exc.errno == errno.EACCES: - logger.debug("Challenges cleaned up but no permissions for %s", - root_path) - else: - raise + logger.info( + "Unable to clean up challenge directory %s", root_path) + logger.debug("Error was: %s", exc) class _WebrootMapAction(argparse.Action): diff --git a/certbot/plugins/webroot_test.py b/certbot/plugins/webroot_test.py index 3f429ec34..5d784a75c 100644 --- a/certbot/plugins/webroot_test.py +++ b/certbot/plugins/webroot_test.py @@ -138,15 +138,10 @@ class AuthenticatorTest(unittest.TestCase): os.chmod(self.path, 0o700) @mock.patch("certbot.plugins.webroot.os.chown") - def test_failed_chown_eacces(self, mock_chown): + def test_failed_chown(self, mock_chown): mock_chown.side_effect = OSError(errno.EACCES, "msg") self.auth.perform([self.achall]) # exception caught and logged - @mock.patch("certbot.plugins.webroot.os.chown") - def test_failed_chown_not_eacces(self, mock_chown): - mock_chown.side_effect = OSError() - self.assertRaises(errors.PluginError, self.auth.perform, []) - def test_perform_permissions(self): self.auth.prepare() @@ -200,7 +195,7 @@ class AuthenticatorTest(unittest.TestCase): os.rmdir(leftover_path) @mock.patch('os.rmdir') - def test_cleanup_permission_denied(self, mock_rmdir): + def test_cleanup_failure(self, mock_rmdir): self.auth.prepare() self.auth.perform([self.achall]) @@ -212,19 +207,6 @@ class AuthenticatorTest(unittest.TestCase): self.assertFalse(os.path.exists(self.validation_path)) self.assertTrue(os.path.exists(self.root_challenge_path)) - @mock.patch('os.rmdir') - def test_cleanup_oserror(self, mock_rmdir): - self.auth.prepare() - self.auth.perform([self.achall]) - - os_error = OSError() - os_error.errno = errno.ENOENT - mock_rmdir.side_effect = os_error - - self.assertRaises(OSError, self.auth.cleanup, [self.achall]) - self.assertFalse(os.path.exists(self.validation_path)) - self.assertTrue(os.path.exists(self.root_challenge_path)) - class WebrootActionTest(unittest.TestCase): """Tests for webroot argparse actions.""" diff --git a/certbot/renewal.py b/certbot/renewal.py index 3682c50d5..d0c797a08 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -18,7 +18,7 @@ from certbot import constants from certbot import crypto_util from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util from certbot import hooks from certbot import storage from certbot.plugins import disco as plugins_disco @@ -60,7 +60,8 @@ def _reconstitute(config, full_path): try: renewal_candidate = storage.RenewableCert( full_path, configuration.RenewerConfiguration(config)) - except (errors.CertStorageError, IOError): + except (errors.CertStorageError, IOError) as exc: + logger.warning(exc) logger.warning("Renewal configuration file %s is broken. Skipping.", full_path) logger.debug("Traceback was:\n%s", traceback.format_exc()) return None @@ -86,7 +87,7 @@ def _reconstitute(config, full_path): return None try: - config.domains = [le_util.enforce_domain_sanity(d) + config.domains = [util.enforce_domain_sanity(d) for d in renewal_candidate.names()] except errors.ConfigurationError as error: logger.warning("Renewal configuration file %s references a cert " @@ -107,7 +108,7 @@ def _restore_webroot_config(config, renewalparams): if not cli.set_by_cli("webroot_map"): config.namespace.webroot_map = renewalparams["webroot_map"] elif "webroot_path" in renewalparams: - logger.info("Ancient renewal conf file without webroot-map, restoring webroot-path") + logger.debug("Ancient renewal conf file without webroot-map, restoring webroot-path") wp = renewalparams["webroot_path"] if isinstance(wp, str): # prior to 0.1.0, webroot_path was a string wp = [wp] @@ -193,7 +194,7 @@ def _restore_required_config_elements(config, renewalparams): def should_renew(config, lineage): "Return true if any of the circumstances for automatic renewal apply." if config.renew_by_default: - logger.info("Auto-renewal forced with --force-renewal...") + logger.debug("Auto-renewal forced with --force-renewal...") return True if lineage.should_autorenew(interactive=True): logger.info("Cert is due for renewal, auto-renewing...") @@ -235,7 +236,7 @@ def renew_cert(config, domains, le_client, lineage): _avoid_invalidating_lineage(config, lineage, original_server) new_certr, new_chain, new_key, _ = le_client.obtain_certificate(domains) if config.dry_run: - logger.info("Dry run: skipping updating lineage at %s", + logger.debug("Dry run: skipping updating lineage at %s", os.path.dirname(lineage.cert)) else: prior_version = lineage.latest_common_version() @@ -301,7 +302,10 @@ def _renew_describe_results(config, renew_successes, renew_failures, def renew_all_lineages(config): """Examine each lineage; renew if due and report results""" - if config.domains != []: + # This is trivially False if config.domains is empty + if any(domain not in config.webroot_map for domain in config.domains): + # If more plugins start using cli.add_domains, + # we may want to only log a warning here raise errors.Error("Currently, the renew verb is only capable of " "renewing all installed certificates that are due " "to be renewed; individual domains cannot be " diff --git a/certbot/reporter.py b/certbot/reporter.py index d509cb0b8..118b13166 100644 --- a/certbot/reporter.py +++ b/certbot/reporter.py @@ -11,11 +11,16 @@ from six.moves import queue # pylint: disable=import-error import zope.interface from certbot import interfaces -from certbot import le_util +from certbot import util logger = logging.getLogger(__name__) +# Store the pid of the process that first imported this module so that +# atexit_print_messages side-effects such as error reporting can be limited to +# this process and not any fork()'d children. +INITIAL_PID = os.getpid() + @zope.interface.implementer(interfaces.IReporter) class Reporter(object): @@ -53,14 +58,16 @@ class Reporter(object): """ assert self.HIGH_PRIORITY <= priority <= self.LOW_PRIORITY self.messages.put(self._msg_type(priority, msg, on_crash)) - logger.info("Reporting to user: %s", msg) + logger.debug("Reporting to user: %s", msg) - def atexit_print_messages(self, pid=os.getpid()): + def atexit_print_messages(self, pid=None): """Function to be registered with atexit to print messages. :param int pid: Process ID """ + if pid is None: + pid = INITIAL_PID # This ensures that messages are only printed from the process that # created the Reporter. if pid == os.getpid(): @@ -79,13 +86,18 @@ class Reporter(object): bold_on = sys.stdout.isatty() if not self.config.quiet: if bold_on: - print(le_util.ANSI_SGR_BOLD) + print(util.ANSI_SGR_BOLD) print('IMPORTANT NOTES:') first_wrapper = textwrap.TextWrapper( - initial_indent=' - ', subsequent_indent=(' ' * 3)) + initial_indent=' - ', + subsequent_indent=(' ' * 3), + break_long_words=False, + break_on_hyphens=False) next_wrapper = textwrap.TextWrapper( initial_indent=first_wrapper.subsequent_indent, - subsequent_indent=first_wrapper.subsequent_indent) + subsequent_indent=first_wrapper.subsequent_indent, + break_long_words=False, + break_on_hyphens=False) while not self.messages.empty(): msg = self.messages.get() if self.config.quiet: @@ -96,7 +108,7 @@ class Reporter(object): if no_exception or msg.on_crash: if bold_on and msg.priority > self.HIGH_PRIORITY: if not self.config.quiet: - sys.stdout.write(le_util.ANSI_SGR_RESET) + sys.stdout.write(util.ANSI_SGR_RESET) bold_on = False lines = msg.text.splitlines() print(first_wrapper.fill(lines[0])) @@ -104,4 +116,4 @@ class Reporter(object): print("\n".join( next_wrapper.fill(line) for line in lines[1:])) if bold_on and not self.config.quiet: - sys.stdout.write(le_util.ANSI_SGR_RESET) + sys.stdout.write(util.ANSI_SGR_RESET) diff --git a/certbot/reverter.py b/certbot/reverter.py index fe6d9f24f..6dde05077 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -7,12 +7,13 @@ import shutil import time import traceback + import zope.component from certbot import constants from certbot import errors from certbot import interfaces -from certbot import le_util +from certbot import util from certbot.display import util as display_util @@ -23,6 +24,39 @@ logger = logging.getLogger(__name__) class Reverter(object): """Reverter Class - save and revert configuration checkpoints. + This class can be used by the plugins, especially Installers, to + undo changes made to the user's system. Modifications to files and + commands to do undo actions taken by the plugin should be registered + with this class before the action is taken. + + Once a change has been registered with this class, there are three + states the change can be in. First, the change can be a temporary + change. This should be used for changes that will soon be reverted, + such as config changes for the purpose of solving a challenge. + Changes are added to this state through calls to + :func:`~add_to_temp_checkpoint` and reverted when + :func:`~revert_temporary_config` or :func:`~recovery_routine` is + called. + + The second state a change can be in is in progress. These changes + are not temporary, however, they also have not been finalized in a + checkpoint. A change must become in progress before it can be + finalized. Changes are added to this state through calls to + :func:`~add_to_checkpoint` and reverted when + :func:`~recovery_routine` is called. + + The last state a change can be in is finalized in a checkpoint. A + change is put into this state by first becoming an in progress + change and then calling :func:`~finalize_checkpoint`. Changes + in this state can be reverted through calls to + :func:`~rollback_checkpoints`. + + As a final note, creating new files and registering undo commands + are handled specially and use the methods + :func:`~register_file_creation` and :func:`~register_undo_command` + respectively. Both of these methods can be used to create either + temporary or in progress changes. + .. note:: Consider moving everything over to CSV format. :param config: Configuration. @@ -32,7 +66,7 @@ class Reverter(object): def __init__(self, config): self.config = config - le_util.make_or_verify_dir( + util.make_or_verify_dir( config.backup_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), self.config.strict_permissions) @@ -184,7 +218,7 @@ class Reverter(object): :raises .ReverterError: if unable to add checkpoint """ - le_util.make_or_verify_dir( + util.make_or_verify_dir( cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), self.config.strict_permissions) @@ -280,7 +314,7 @@ class Reverter(object): csvreader = csv.reader(csvfile) for command in reversed(list(csvreader)): try: - le_util.run_script(command) + util.run_script(command) except errors.SubprocessError: logger.error( "Unable to run undo command: %s", " ".join(command)) @@ -396,7 +430,7 @@ class Reverter(object): else: cp_dir = self.config.in_progress_dir - le_util.make_or_verify_dir( + util.make_or_verify_dir( cp_dir, constants.CONFIG_DIRS_MODE, os.geteuid(), self.config.strict_permissions) @@ -489,7 +523,7 @@ class Reverter(object): if not os.path.exists(changes_since_path): logger.info("Rollback checkpoint is empty (no changes made?)") - with open(self.config.changes_since_path) as f: + with open(changes_since_path, 'w') as f: f.write("No changes\n") # Add title to self.config.in_progress_dir CHANGES_SINCE diff --git a/certbot/storage.py b/certbot/storage.py index c4bfb3e28..82fdbfd54 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -7,16 +7,20 @@ import re import configobj import parsedatetime import pytz +import six +import certbot +from certbot import cli from certbot import constants from certbot import crypto_util from certbot import errors from certbot import error_handler -from certbot import le_util +from certbot import util logger = logging.getLogger(__name__) ALL_FOUR = ("cert", "privkey", "chain", "fullchain") +CURRENT_VERSION = util.get_strict_version(certbot.__version__) def config_with_defaults(config=None): @@ -63,6 +67,7 @@ def write_renewal_config(o_filename, n_filename, target, relevant_data): """ config = configobj.ConfigObj(o_filename) + config["version"] = certbot.__version__ for kind in ALL_FOUR: config[kind] = target[kind] @@ -155,36 +160,13 @@ def relevant_values(all_values): :param dict all_values: The original values. :returns: A new dictionary containing items that can be used in renewal. - :rtype dict:""" + :rtype dict: - from certbot import cli - - def _is_cli_default(option, value): - # Look through the CLI parser defaults and see if this option is - # both present and equal to the specified value. If not, return - # False. - # pylint: disable=protected-access - for x in cli.helpful_parser.parser._actions: - if x.dest == option: - if x.default == value: - return True - else: - break - return False - - values = dict() - for option, value in all_values.iteritems(): - # Try to find reasons to store this item in the - # renewal config. It can be stored if it is relevant and - # (it is set_by_cli() or flag_default() is different - # from the value or flag_default() doesn't exist). - if _relevant(option): - if (cli.set_by_cli(option) - or not _is_cli_default(option, value)): -# or option not in constants.CLI_DEFAULTS -# or constants.CLI_DEFAULTS[option] != value): - values[option] = value - return values + """ + return dict( + (option, value) + for option, value in six.iteritems(all_values) + if _relevant(option) and cli.option_was_set(option, value)) class RenewableCert(object): # pylint: disable=too-many-instance-attributes @@ -259,6 +241,14 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes "renewal config file {0} is missing a required " "file reference".format(self.configfile)) + conf_version = self.configuration.get("version") + if (conf_version is not None and + util.get_strict_version(conf_version) > CURRENT_VERSION): + logger.warning( + "Attempting to parse the version %s renewal configuration " + "file found at %s with version %s of Certbot. This might not " + "work.", conf_version, config_filename, certbot.__version__) + self.cert = self.configuration["cert"] self.privkey = self.configuration["privkey"] self.chain = self.configuration["chain"] @@ -605,7 +595,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes if target is None: raise errors.CertStorageError("could not find cert file") with open(target) as f: - return crypto_util.get_sans_from_cert(f.read()) + return crypto_util.get_names_from_cert(f.read()) def autodeployment_is_enabled(self): """Is automatic deployment enabled for this cert? @@ -758,7 +748,7 @@ class RenewableCert(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(i): os.makedirs(i, 0o700) logger.debug("Creating directory %s.", i) - config_file, config_filename = le_util.unique_lineage_name( + config_file, config_filename = util.unique_lineage_name( cli_config.renewal_configs_dir, lineagename) if not config_filename.endswith(".conf"): raise errors.CertStorageError( diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index a96e57507..4cd2bfebf 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -137,6 +137,16 @@ class AccountFileStorageTest(unittest.TestCase): # restore self.assertEqual(self.acc, self.storage.load(self.acc.id)) + def test_save_regr(self): + self.storage.save_regr(self.acc) + account_path = os.path.join(self.config.accounts_dir, self.acc.id) + self.assertTrue(os.path.exists(account_path)) + self.assertTrue(os.path.exists(os.path.join( + account_path, "regr.json"))) + for file_name in "meta.json", "private_key.json": + self.assertFalse(os.path.exists( + os.path.join(account_path, file_name))) + def test_find_all(self): self.storage.save(self.acc) self.assertEqual([self.acc], self.storage.find_all()) diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 3facd4f7c..eccc36418 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -11,7 +11,7 @@ from acme import messages from certbot import achallenges from certbot import errors -from certbot import le_util +from certbot import util from certbot.tests import acme_util @@ -69,7 +69,7 @@ class GetAuthorizationsTest(unittest.TestCase): self.mock_auth.perform.side_effect = gen_auth_resp - self.mock_account = mock.Mock(key=le_util.Key("file_path", "PEM")) + self.mock_account = mock.Mock(key=util.Key("file_path", "PEM")) self.mock_net = mock.MagicMock(spec=acme_client.Client) self.handler = AuthHandler( diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 31056cafe..896550837 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -2,6 +2,7 @@ from __future__ import print_function import argparse +import dialog import functools import itertools import os @@ -22,7 +23,7 @@ from certbot import configuration from certbot import constants from certbot import crypto_util from certbot import errors -from certbot import le_util +from certbot import util from certbot import main from certbot import renewal from certbot import storage @@ -149,6 +150,14 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods args.extend(['--email', 'io@io.is']) self._cli_missing_flag(args, "--agree-tos") + @mock.patch('certbot.main.renew') + def test_gui(self, renew): + args = ['renew', '--dialog'] + # --text conflicts with --dialog + self.standard_args.remove('--text') + self._call(args) + self.assertFalse(renew.call_args[0][0].noninteractive_mode) + @mock.patch('certbot.main.client.acme_client.Client') @mock.patch('certbot.main._determine_account') @mock.patch('certbot.main.client.Client.obtain_and_enroll_certificate') @@ -163,13 +172,13 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods with mock.patch('certbot.main.client.acme_client.ClientNetwork') as acme_net: self._call_no_clientmock(args) - os_ver = " ".join(le_util.get_os_info()) + os_ver = util.get_os_info_ua() ua = acme_net.call_args[1]["user_agent"] self.assertTrue(os_ver in ua) import platform plat = platform.platform() if "linux" in plat.lower(): - self.assertTrue(platform.linux_distribution()[0] in ua) + self.assertTrue(util.get_os_info_ua() in ua) with mock.patch('certbot.main.client.acme_client.ClientNetwork') as acme_net: ua = "bandersnatch" @@ -201,7 +210,7 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods '--key-path', 'key', '--chain-path', 'chain']) self.assertEqual(mock_pick_installer.call_count, 1) - @mock.patch('certbot.le_util.exe_exists') + @mock.patch('certbot.util.exe_exists') def test_configurator_selection(self, mock_exe_exists): mock_exe_exists.return_value = True real_plugins = disco.PluginsRegistry.find_all() @@ -333,11 +342,11 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods # FQDN self.assertRaises(errors.ConfigurationError, self._call, - ['-d', 'comma,gotwrong.tld']) + ['-d', 'a' * 64]) # FQDN 2 self.assertRaises(errors.ConfigurationError, self._call, - ['-d', 'illegal.character=.tld']) + ['-d', (('a' * 50) + '.') * 10]) # Wildcard self.assertRaises(errors.ConfigurationError, self._call, @@ -349,8 +358,9 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods ['-d', '204.11.231.35']) def test_csr_with_besteffort(self): - args = ["--csr", CSR, "--allow-subset-of-names"] - self.assertRaises(errors.Error, self._call, args) + self.assertRaises( + errors.Error, self._call, + 'certonly --csr {0} --allow-subset-of-names'.format(CSR).split()) def test_run_with_csr(self): # This is an error because you can only use --csr with certonly @@ -361,6 +371,17 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods return assert False, "Expected supplying --csr to fail with default verb" + def test_csr_with_no_domains(self): + self.assertRaises( + errors.Error, self._call, + 'certonly --csr {0}'.format( + test_util.vector_path('csr-nonames.pem')).split()) + + def test_csr_with_inconsistent_domains(self): + self.assertRaises( + errors.Error, self._call, + 'certonly -d example.org --csr {0}'.format(CSR).split()) + def _get_argument_parser(self): plugins = disco.PluginsRegistry.find_all() return functools.partial(cli.prepare_and_parse_args, plugins) @@ -410,6 +431,13 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods for arg in conflicting_args: self.assertTrue(arg in error.message) + def test_must_staple_flag(self): + parse = self._get_argument_parser() + short_args = ['--must-staple'] + namespace = parse(short_args) + self.assertTrue(namespace.must_staple) + self.assertTrue(namespace.staple) + def test_staging_flag(self): parse = self._get_argument_parser() short_args = ['--staging'] @@ -420,6 +448,19 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods short_args += '--server example.com'.split() self._check_server_conflict_message(short_args, '--staging') + def test_option_was_set(self): + key_size_option = 'rsa_key_size' + key_size_value = cli.flag_default(key_size_option) + self._get_argument_parser()( + '--rsa-key-size {0}'.format(key_size_value).split()) + + self.assertTrue(cli.option_was_set(key_size_option, key_size_value)) + self.assertTrue(cli.option_was_set('no_verify_ssl', True)) + + config_dir_option = 'config_dir' + self.assertFalse(cli.option_was_set( + config_dir_option, cli.flag_default(config_dir_option))) + def _assert_dry_run_flag_worked(self, namespace, existing_account): self.assertTrue(namespace.dry_run) self.assertTrue(namespace.break_my_certs) @@ -624,6 +665,18 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods out = stdout.getvalue() self.assertEqual("", out) + def test_renew_hook_validation(self): + self._make_test_renewal_conf('sample-renewal.conf') + args = ["renew", "--dry-run", "--post-hook=no-such-command"] + self._test_renewal_common(True, [], args=args, should_renew=False, + error_expected=True) + + def test_renew_no_hook_validation(self): + self._make_test_renewal_conf('sample-renewal.conf') + args = ["renew", "--dry-run", "--post-hook=no-such-command", + "--disable-hook-validation"] + self._test_renewal_common(True, [], args=args, should_renew=True, + error_expected=False) @mock.patch("certbot.cli.set_by_cli") def test_ancient_webroot_renewal_conf(self, mock_set_by_cli): @@ -712,6 +765,12 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._test_renew_common(renewalparams=renewalparams, assert_oc_called=True) + def test_renew_with_webroot_map(self): + renewalparams = {'authenticator': 'webroot'} + self._test_renew_common( + renewalparams=renewalparams, assert_oc_called=True, + args=['renew', '--webroot-map', '{"example.com": "/tmp"}']) + def test_renew_reconstitute_error(self): # pylint: disable=protected-access with mock.patch('certbot.main.renewal._reconstitute') as mock_reconstitute: @@ -864,6 +923,13 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods mock_sys.exit.assert_called_with(''.join( traceback.format_exception_only(KeyboardInterrupt, interrupt))) + # Test dialog errors + exception = dialog.error(message="test message") + main._handle_exception( + dialog.DialogError, exc_value=exception, trace=None, config=None) + error_msg = mock_sys.exit.call_args_list[-1][0][0] + self.assertTrue("test message" in error_msg) + def test_read_file(self): rel_test_path = os.path.relpath(os.path.join(self.tmp_dir, 'foo')) self.assertRaises( @@ -882,6 +948,78 @@ class CLITest(unittest.TestCase): # pylint: disable=too-many-public-methods self._call(['-c', test_util.vector_path('cli.ini')]) self.assertTrue(mocked_run.called) + def test_register(self): + with mock.patch('certbot.main.client') as mocked_client: + acc = mock.MagicMock() + acc.id = "imaginary_account" + mocked_client.register.return_value = (acc, "worked") + self._call_no_clientmock(["register", "--email", "user@example.org"]) + # TODO: It would be more correct to explicitly check that + # _determine_account() gets called in the above case, + # but coverage statistics should also show that it did. + with mock.patch('certbot.main.account') as mocked_account: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = ["an account"] + x = self._call_no_clientmock(["register", "--email", "user@example.org"]) + self.assertTrue("There is an existing account" in x[0]) + + def test_update_registration_no_existing_accounts(self): + # with mock.patch('certbot.main.client') as mocked_client: + with mock.patch('certbot.main.account') as mocked_account: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = [] + x = self._call_no_clientmock( + ["register", "--update-registration", "--email", + "user@example.org"]) + self.assertTrue("Could not find an existing account" in x[0]) + + def test_update_registration_unsafely(self): + # This test will become obsolete when register --update-registration + # supports removing an e-mail address from the account + with mock.patch('certbot.main.account') as mocked_account: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = ["an account"] + x = self._call_no_clientmock( + "register --update-registration " + "--register-unsafely-without-email".split()) + self.assertTrue("--register-unsafely-without-email" in x[0]) + + @mock.patch('certbot.main.display_ops.get_email') + @mock.patch('certbot.main.zope.component.getUtility') + def test_update_registration_with_email(self, mock_utility, mock_email): + email = "user@example.com" + mock_email.return_value = email + with mock.patch('certbot.main.client') as mocked_client: + with mock.patch('certbot.main.account') as mocked_account: + with mock.patch('certbot.main._determine_account') as mocked_det: + with mock.patch('certbot.main.client') as mocked_client: + mocked_storage = mock.MagicMock() + mocked_account.AccountFileStorage.return_value = mocked_storage + mocked_storage.find_all.return_value = ["an account"] + mocked_det.return_value = (mock.MagicMock(), "foo") + acme_client = mock.MagicMock() + mocked_client.Client.return_value = acme_client + x = self._call_no_clientmock( + ["register", "--update-registration"]) + # When registration change succeeds, the return value + # of register() is None + self.assertTrue(x[0] is None) + # and we got supposedly did update the registration from + # the server + self.assertTrue( + acme_client.acme.update_registration.called) + # and we saved the updated registration on disk + self.assertTrue(mocked_storage.save_regr.called) + self.assertTrue( + email in mock_utility().add_message.call_args[0][0]) + + def test_conflicting_args(self): + args = ['renew', '--dialog', '--text'] + self.assertRaises(errors.Error, self._call, args) + class DetermineAccountTest(unittest.TestCase): """Tests for certbot.cli._determine_account.""" @@ -958,7 +1096,7 @@ class DuplicativeCertsTest(storage_test.BaseRenewableCertTest): def tearDown(self): shutil.rmtree(self.tempdir) - @mock.patch('certbot.le_util.make_or_verify_dir') + @mock.patch('certbot.util.make_or_verify_dir') def test_find_duplicative_names(self, unused_makedir): from certbot.main import _find_duplicative_certs test_cert = test_util.load_vector('cert-san.pem') diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 8ceefe8ae..9156277a9 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -11,7 +11,7 @@ from acme import jose from certbot import account from certbot import errors -from certbot import le_util +from certbot import util from certbot.tests import test_util @@ -134,63 +134,45 @@ class ClientTest(unittest.TestCase): self.acme.fetch_chain.assert_called_once_with(mock.sentinel.certr) - # FIXME move parts of this to test_cli.py... @mock.patch("certbot.client.logger") def test_obtain_certificate_from_csr(self, mock_logger): self._mock_obtain_certificate() - from certbot import cli - test_csr = le_util.CSR(form="der", file=None, data=CSR_SAN) - mock_parsed_args = mock.MagicMock() - # The CLI should believe that this is a certonly request, because - # a CSR would not be allowed with other kinds of requests! - mock_parsed_args.verb = "certonly" - with mock.patch("certbot.client.le_util.CSR") as mock_CSR: - mock_CSR.return_value = test_csr - mock_parsed_args.domains = self.eg_domains[:] - mock_parser = mock.MagicMock(cli.HelpfulArgumentParser) - cli.HelpfulArgumentParser.handle_csr(mock_parser, mock_parsed_args) + test_csr = util.CSR(form="der", file=None, data=CSR_SAN) + auth_handler = self.client.auth_handler - # Now provoke an inconsistent domains error... - mock_parsed_args.domains.append("hippopotamus.io") - self.assertRaises(errors.ConfigurationError, - cli.HelpfulArgumentParser.handle_csr, mock_parser, mock_parsed_args) - - authzr = self.client.auth_handler.get_authorizations(self.eg_domains, False) - - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=authzr)) - # and that the cert was obtained correctly - self._check_obtain_certificate() - - # Test for authzr=None - self.assertEqual( - (mock.sentinel.certr, mock.sentinel.chain), - self.client.obtain_certificate_from_csr( - self.eg_domains, - test_csr, - authzr=None)) - - self.client.auth_handler.get_authorizations.assert_called_with( - self.eg_domains) - - # Test for no auth_handler - self.client.auth_handler = None - self.assertRaises( - errors.Error, - self.client.obtain_certificate_from_csr, + authzr = auth_handler.get_authorizations(self.eg_domains, False) + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( self.eg_domains, - test_csr) - mock_logger.warning.assert_called_once_with(mock.ANY) + test_csr, + authzr=authzr)) + # and that the cert was obtained correctly + self._check_obtain_certificate() + + # Test for authzr=None + self.assertEqual( + (mock.sentinel.certr, mock.sentinel.chain), + self.client.obtain_certificate_from_csr( + self.eg_domains, + test_csr, + authzr=None)) + auth_handler.get_authorizations.assert_called_with(self.eg_domains) + + # Test for no auth_handler + self.client.auth_handler = None + self.assertRaises( + errors.Error, + self.client.obtain_certificate_from_csr, + self.eg_domains, + test_csr) + mock_logger.warning.assert_called_once_with(mock.ANY) @mock.patch("certbot.client.crypto_util") def test_obtain_certificate(self, mock_crypto_util): self._mock_obtain_certificate() - csr = le_util.CSR(form="der", file=None, data=CSR_SAN) + csr = util.CSR(form="der", file=None, data=CSR_SAN) mock_crypto_util.init_save_csr.return_value = csr mock_crypto_util.init_save_key.return_value = mock.sentinel.key domains = ["example.com", "www.example.com"] @@ -221,7 +203,9 @@ class ClientTest(unittest.TestCase): mock.sentinel.key, domains, self.config.csr_dir) self._check_obtain_certificate() - def test_save_certificate(self): + @mock.patch("certbot.cli.helpful_parser") + def test_save_certificate(self, mock_parser): + # pylint: disable=too-many-locals certs = ["matching_cert.pem", "cert.pem", "cert-san.pem"] tmp_path = tempfile.mkdtemp() os.chmod(tmp_path, 0o755) # TODO: really?? @@ -232,6 +216,10 @@ class ClientTest(unittest.TestCase): candidate_cert_path = os.path.join(tmp_path, "certs", "cert.pem") candidate_chain_path = os.path.join(tmp_path, "chains", "chain.pem") candidate_fullchain_path = os.path.join(tmp_path, "chains", "fullchain.pem") + mock_parser.verb = "certonly" + mock_parser.args = ["--cert-path", candidate_cert_path, + "--chain-path", candidate_chain_path, + "--fullchain-path", candidate_fullchain_path] cert_path, chain_path, fullchain_path = self.client.save_certificate( certr, chain_cert, candidate_cert_path, candidate_chain_path, diff --git a/certbot/tests/colored_logging_test.py b/certbot/tests/colored_logging_test.py index 91c6b8c08..0a7929561 100644 --- a/certbot/tests/colored_logging_test.py +++ b/certbot/tests/colored_logging_test.py @@ -4,7 +4,7 @@ import unittest import six -from certbot import le_util +from certbot import util class StreamHandlerTest(unittest.TestCase): @@ -32,9 +32,9 @@ class StreamHandlerTest(unittest.TestCase): self.logger.debug(msg) self.assertEqual(self.stream.getvalue(), - '{0}{1}{2}\n'.format(le_util.ANSI_SGR_RED, + '{0}{1}{2}\n'.format(util.ANSI_SGR_RED, msg, - le_util.ANSI_SGR_RESET)) + util.ANSI_SGR_RESET)) if __name__ == "__main__": diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 52e595577..5a592bbb1 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -10,6 +10,7 @@ import zope.component from certbot import errors from certbot import interfaces +from certbot import util from certbot.tests import test_util @@ -62,7 +63,7 @@ class InitSaveCSRTest(unittest.TestCase): shutil.rmtree(self.csr_dir) @mock.patch('certbot.crypto_util.make_csr') - @mock.patch('certbot.crypto_util.le_util.make_or_verify_dir') + @mock.patch('certbot.crypto_util.util.make_or_verify_dir') def test_it(self, unused_mock_verify, mock_csr): from certbot.crypto_util import init_save_csr @@ -95,6 +96,25 @@ class MakeCSRTest(unittest.TestCase): ['example.com', 'www.example.com'], get_sans_from_csr( csr_der, OpenSSL.crypto.FILETYPE_ASN1)) + def test_must_staple(self): + # TODO: Fails for RSA256_KEY + csr_pem, _ = self._call( + RSA512_KEY, ['example.com', 'www.example.com'], must_staple=True) + csr = OpenSSL.crypto.load_certificate_request( + OpenSSL.crypto.FILETYPE_PEM, csr_pem) + + # In pyopenssl 0.13 (used with TOXENV=py26-oldest and py27-oldest), csr + # objects don't have a get_extensions() method, so we skip this test if + # the method isn't available. + if hasattr(csr, 'get_extensions'): + # NOTE: Ideally we would filter by the TLS Feature OID, but + # OpenSSL.crypto.X509Extension doesn't give us the extension's raw OID, + # and the shortname field is just "UNDEF" + must_staple_exts = [e for e in csr.get_extensions() + if e.get_data() == "0\x03\x02\x01\x05"] + self.assertEqual(len(must_staple_exts), 1, + "Expected exactly one Must Staple extension") + class ValidCSRTest(unittest.TestCase): """Tests for certbot.crypto_util.valid_csr.""" @@ -140,6 +160,44 @@ class CSRMatchesPubkeyTest(unittest.TestCase): test_util.load_vector('csr.pem'), RSA256_KEY)) +class ImportCSRFileTest(unittest.TestCase): + """Tests for certbot.certbot_util.import_csr_file.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.crypto_util import import_csr_file + return import_csr_file(*args, **kwargs) + + def test_der_csr(self): + csrfile = test_util.vector_path('csr.der') + data = test_util.load_vector('csr.der') + + self.assertEqual( + (OpenSSL.crypto.FILETYPE_ASN1, + util.CSR(file=csrfile, + data=data, + form="der"), + ["example.com"],), + self._call(csrfile, data)) + + def test_pem_csr(self): + csrfile = test_util.vector_path('csr.pem') + data = test_util.load_vector('csr.pem') + + self.assertEqual( + (OpenSSL.crypto.FILETYPE_PEM, + util.CSR(file=csrfile, + data=data, + form="pem"), + ["example.com"],), + self._call(csrfile, data)) + + def test_bad_csr(self): + self.assertRaises(errors.Error, self._call, + test_util.vector_path('cert.pem'), + test_util.load_vector('cert.pem')) + + class MakeKeyTest(unittest.TestCase): # pylint: disable=too-few-public-methods """Tests for certbot.crypto_util.make_key.""" @@ -215,6 +273,62 @@ class GetSANsFromCSRTest(unittest.TestCase): [], self._call(test_util.load_vector('csr-nosans.pem'))) +class GetNamesFromCertTest(unittest.TestCase): + """Tests for certbot.crypto_util.get_names_from_cert.""" + + @classmethod + def _call(cls, *args, **kwargs): + from certbot.crypto_util import get_names_from_cert + return get_names_from_cert(*args, **kwargs) + + def test_single(self): + self.assertEqual( + ['example.com'], + self._call(test_util.load_vector('cert.pem'))) + + def test_san(self): + self.assertEqual( + ['example.com', 'www.example.com'], + self._call(test_util.load_vector('cert-san.pem'))) + + def test_common_name_sans_order(self): + # Tests that the common name comes first + # followed by the SANS in alphabetical order + self.assertEqual( + ['example.com'] + ['{0}.example.com'.format(c) for c in 'abcd'], + self._call(test_util.load_vector('cert-5sans.pem'))) + + +class GetNamesFromCSRTest(unittest.TestCase): + """Tests for certbot.crypto_util.get_names_from_csr.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.crypto_util import get_names_from_csr + return get_names_from_csr(*args, **kwargs) + + def test_extract_one_san(self): + self.assertEqual(['example.com'], self._call( + test_util.load_vector('csr.pem'))) + + def test_extract_two_sans(self): + self.assertEqual(set(('example.com', 'www.example.com',)), set( + self._call(test_util.load_vector('csr-san.pem')))) + + def test_extract_six_sans(self): + self.assertEqual( + set(self._call(test_util.load_vector('csr-6sans.pem'))), + set(("example.com", "example.org", "example.net", + "example.info", "subdomain.example.com", + "other.subdomain.example.com",))) + + def test_parse_non_csr(self): + self.assertRaises(OpenSSL.crypto.Error, self._call, "hello there") + + def test_parse_no_sans(self): + self.assertEqual(["example.org"], + self._call(test_util.load_vector('csr-nosans.pem'))) + + class CertLoaderTest(unittest.TestCase): """Tests for certbot.crypto_util.pyopenssl_load_certificate""" diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 05cb6b12d..26f67b69f 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -12,6 +12,7 @@ from acme import jose from acme import messages from certbot import account +from certbot import errors from certbot import interfaces from certbot.display import util as display_util @@ -37,41 +38,39 @@ class GetEmailTest(unittest.TestCase): def test_cancel_none(self): self.input.return_value = (display_util.CANCEL, "foo@bar.baz") - self.assertTrue(self._call() is None) + self.assertRaises(errors.Error, self._call) + self.assertRaises(errors.Error, self._call, optional=False) def test_ok_safe(self): self.input.return_value = (display_util.OK, "foo@bar.baz") - with mock.patch("certbot.display.ops.le_util.safe_email") as mock_safe_email: + with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True self.assertTrue(self._call() is "foo@bar.baz") def test_ok_not_safe(self): self.input.return_value = (display_util.OK, "foo@bar.baz") - with mock.patch("certbot.display.ops.le_util.safe_email") as mock_safe_email: + with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.side_effect = [False, True] self.assertTrue(self._call() is "foo@bar.baz") - def test_more_and_invalid_flags(self): - more_txt = "--register-unsafely-without-email" + def test_invalid_flag(self): invalid_txt = "There seem to be problems" - base_txt = "Enter email" self.input.return_value = (display_util.OK, "foo@bar.baz") - with mock.patch("certbot.display.ops.le_util.safe_email") as mock_safe_email: + with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: mock_safe_email.return_value = True self._call() - msg = self.input.call_args[0][0] - self.assertTrue(more_txt not in msg) - self.assertTrue(invalid_txt not in msg) - self.assertTrue(base_txt in msg) - self._call(more=True) - msg = self.input.call_args[0][0] - self.assertTrue(more_txt in msg) - self.assertTrue(invalid_txt not in msg) - self._call(more=True, invalid=True) - msg = self.input.call_args[0][0] - self.assertTrue(more_txt in msg) - self.assertTrue(invalid_txt in msg) - self.assertTrue(base_txt in msg) + self.assertTrue(invalid_txt not in self.input.call_args[0][0]) + self._call(invalid=True) + self.assertTrue(invalid_txt in self.input.call_args[0][0]) + + def test_optional_flag(self): + self.input.return_value = (display_util.OK, "foo@bar.baz") + with mock.patch("certbot.display.ops.util.safe_email") as mock_safe_email: + mock_safe_email.side_effect = [False, True] + self._call(optional=False) + for call in self.input.call_args_list: + self.assertTrue( + "--register-unsafely-without-email" not in call[0][0]) class ChooseAccountTest(unittest.TestCase): @@ -249,9 +248,9 @@ class ChooseNamesTest(unittest.TestCase): def test_get_valid_domains(self): from certbot.display.ops import get_valid_domains all_valid = ["example.com", "second.example.com", - "also.example.com"] - all_invalid = ["xn--ls8h.tld", "*.wildcard.com", "notFQDN", - "uniçodé.com"] + "also.example.com", "under_score.example.com", + "justtld"] + all_invalid = ["xn--ls8h.tld", "*.wildcard.com", "uniçodé.com"] two_valid = ["example.com", "xn--ls8h.tld", "also.example.com"] self.assertEqual(get_valid_domains(all_valid), all_valid) self.assertEqual(get_valid_domains(all_invalid), []) @@ -277,19 +276,18 @@ class ChooseNamesTest(unittest.TestCase): mock_util().input.return_value = (display_util.OK, "xn--ls8h.tld") self.assertEqual(_choose_names_manually(), []) - # non-FQDN and no retry - mock_util().input.return_value = (display_util.OK, - "notFQDN") - self.assertEqual(_choose_names_manually(), []) - # Two valid domains + # Valid domains mock_util().input.return_value = (display_util.OK, ("example.com," + "under_score.example.com," + "justtld," "valid.example.com")) self.assertEqual(_choose_names_manually(), - ["example.com", "valid.example.com"]) + ["example.com", "under_score.example.com", + "justtld", "valid.example.com"]) # Three iterations mock_util().input.return_value = (display_util.OK, - "notFQDN") + "uniçodé.com") yn = mock.MagicMock() yn.side_effect = [True, True, False] mock_util().yesno = yn diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index ce78b5dc9..be7fb852d 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -56,14 +56,22 @@ class HookTest(unittest.TestCase): return mock_logger.warning def test_pre_hook(self): + hooks.pre_hook.already = False config = mock.MagicMock(pre_hook="true") self._test_a_hook(config, hooks.pre_hook, 1) config = mock.MagicMock(pre_hook="") self._test_a_hook(config, hooks.pre_hook, 0) def test_post_hook(self): + hooks.pre_hook.already = False + # if pre-hook isn't called, post-hook shouldn't be config = mock.MagicMock(post_hook="true", verb="splonk") + self._test_a_hook(config, hooks.post_hook, 0) + + config = mock.MagicMock(post_hook="true", verb="splonk") + self._test_a_hook(config, hooks.pre_hook, 1) self._test_a_hook(config, hooks.post_hook, 2) + config = mock.MagicMock(post_hook="true", verb="renew") self._test_a_hook(config, hooks.post_hook, 0) diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index eda5ffb36..58cc68dce 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -34,6 +34,20 @@ class ReverterCheckpointLocalTest(unittest.TestCase): logging.disable(logging.NOTSET) + @mock.patch("certbot.reverter.Reverter._read_and_append") + def test_no_change(self, mock_read): + mock_read.side_effect = OSError("cannot even") + try: + self.reverter.add_to_checkpoint(self.sets[0], "save1") + except OSError: + pass + self.reverter.finalize_checkpoint("blah") + path = os.listdir(self.reverter.config.backup_dir)[0] + no_change = os.path.join(self.reverter.config.backup_dir, path, "CHANGES_SINCE") + with open(no_change, "r") as f: + x = f.read() + self.assertTrue("No changes" in x) + def test_basic_add_to_temp_checkpoint(self): # These shouldn't conflict even though they are both named config.txt self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -150,7 +164,7 @@ class ReverterCheckpointLocalTest(unittest.TestCase): errors.ReverterError, self.reverter.register_undo_command, True, ["command"]) - @mock.patch("certbot.le_util.run_script") + @mock.patch("certbot.util.run_script") def test_run_undo_commands(self, mock_run): mock_run.side_effect = ["", errors.SubprocessError] coms = [ diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index be626edc5..261500b98 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -10,6 +10,8 @@ import configobj import mock import pytz +import certbot +from certbot import cli from certbot import configuration from certbot import errors from certbot.storage import ALL_FOUR @@ -83,18 +85,20 @@ class BaseRenewableCertTest(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) + def _write_out_kind(self, kind, ver, value=None): + link = getattr(self.test_rc, kind) + if os.path.lexists(link): + os.unlink(link) + os.symlink(os.path.join(os.path.pardir, os.path.pardir, "archive", + "example.org", "{0}{1}.pem".format(kind, ver)), + link) + with open(link, "w") as f: + f.write(kind if value is None else value) + def _write_out_ex_kinds(self): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}12.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}11.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 12) + self._write_out_kind(kind, 11) class RenewableCertTests(BaseRenewableCertTest): @@ -137,6 +141,28 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertRaises(errors.CertStorageError, storage.RenewableCert, config.filename, self.cli_config) + def test_no_renewal_version(self): + from certbot import storage + + self._write_out_ex_kinds() + self.assertTrue("version" not in self.config) + + with mock.patch("certbot.storage.logger") as mock_logger: + storage.RenewableCert(self.config.filename, self.cli_config) + self.assertFalse(mock_logger.warning.called) + + def test_renewal_newer_version(self): + from certbot import storage + + self._write_out_ex_kinds() + self.config["version"] = "99.99.99" + self.config.write() + + with mock.patch("certbot.storage.logger") as mock_logger: + storage.RenewableCert(self.config.filename, self.cli_config) + self.assertTrue(mock_logger.warning.called) + self.assertTrue("version" in mock_logger.warning.call_args[0][0]) + def test_consistent(self): # pylint: disable=too-many-statements,protected-access oldcert = self.test_rc.cert @@ -181,10 +207,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_current_target(self): # Relative path logic - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert17.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") + self._write_out_kind("cert", 17) self.assertTrue(os.path.samefile(self.test_rc.current_target("cert"), os.path.join(self.tempdir, "archive", "example.org", @@ -202,12 +225,8 @@ class RenewableCertTests(BaseRenewableCertTest): def test_current_version(self): for ver in (1, 5, 10, 20): - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert{0}.pem".format(ver)), - self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") - os.unlink(self.test_rc.cert) + self._write_out_kind("cert", ver) + os.unlink(self.test_rc.cert) os.symlink(os.path.join("..", "..", "archive", "example.org", "cert10.pem"), self.test_rc.cert) self.assertEqual(self.test_rc.current_version("cert"), 10) @@ -218,61 +237,30 @@ class RenewableCertTests(BaseRenewableCertTest): def test_latest_and_next_versions(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(self.test_rc.latest_common_version(), 5) self.assertEqual(self.test_rc.next_free_version(), 6) # Having one kind of file of a later version doesn't change the # result - os.unlink(self.test_rc.privkey) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "privkey7.pem"), self.test_rc.privkey) - with open(self.test_rc.privkey, "w") as f: - f.write("privkey") + self._write_out_kind("privkey", 7) self.assertEqual(self.test_rc.latest_common_version(), 5) # ... although it does change the next free version self.assertEqual(self.test_rc.next_free_version(), 8) # Nor does having three out of four change the result - os.unlink(self.test_rc.cert) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert7.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") - os.unlink(self.test_rc.fullchain) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "fullchain7.pem"), self.test_rc.fullchain) - with open(self.test_rc.fullchain, "w") as f: - f.write("fullchain") + self._write_out_kind("cert", 7) + self._write_out_kind("fullchain", 7) self.assertEqual(self.test_rc.latest_common_version(), 5) # If we have everything from a much later version, it does change # the result - ver = 17 for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 17) self.assertEqual(self.test_rc.latest_common_version(), 17) self.assertEqual(self.test_rc.next_free_version(), 18) def test_update_link_to(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) # pylint: disable=protected-access self.test_rc._update_link_to("cert", 3) @@ -289,10 +277,7 @@ class RenewableCertTests(BaseRenewableCertTest): "chain3000.pem") def test_version(self): - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert12.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write("cert") + self._write_out_kind("cert", 12) # TODO: We should probably test that the directory is still the # same, but it's tricky because we can get an absolute # path out when we put a relative path in. @@ -302,13 +287,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_update_all_links_to_success(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) self.assertEqual(self.test_rc.latest_common_version(), 5) for ver in xrange(1, 6): @@ -353,13 +332,7 @@ class RenewableCertTests(BaseRenewableCertTest): def test_has_pending_deployment(self): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertEqual(ver, self.test_rc.current_version(kind)) for ver in xrange(1, 6): self.test_rc.update_all_links_to(ver) @@ -372,24 +345,22 @@ class RenewableCertTests(BaseRenewableCertTest): def test_names(self): # Trying the current version - test_cert = test_util.load_vector("cert-san.pem") - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert12.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write(test_cert) + self._write_out_kind("cert", 12, test_util.load_vector("cert-san.pem")) self.assertEqual(self.test_rc.names(), ["example.com", "www.example.com"]) # Trying a non-current version - test_cert = test_util.load_vector("cert.pem") - os.unlink(self.test_rc.cert) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "cert15.pem"), self.test_rc.cert) - with open(self.test_rc.cert, "w") as f: - f.write(test_cert) + self._write_out_kind("cert", 15, test_util.load_vector("cert.pem")) self.assertEqual(self.test_rc.names(12), ["example.com", "www.example.com"]) + # Testing common name is listed first + self._write_out_kind( + "cert", 12, test_util.load_vector("cert-5sans.pem")) + self.assertEqual( + self.test_rc.names(12), + ["example.com"] + ["{0}.example.com".format(c) for c in "abcd"]) + # Trying missing cert os.unlink(self.test_rc.cert) self.assertRaises(errors.CertStorageError, self.test_rc.names) @@ -457,13 +428,7 @@ class RenewableCertTests(BaseRenewableCertTest): # No pending deployment for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.assertFalse(self.test_rc.should_autodeploy()) def test_autorenewal_is_enabled(self): @@ -484,11 +449,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(self.test_rc.should_autorenew()) self.test_rc.configuration["autorenew"] = "1" for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}12.pem".format(kind)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, 12) # Mandatory renewal on the basis of OCSP revocation mock_ocsp.return_value = True self.assertTrue(self.test_rc.should_autorenew()) @@ -502,13 +463,7 @@ class RenewableCertTests(BaseRenewableCertTest): for ver in xrange(1, 6): for kind in ALL_FOUR: - where = getattr(self.test_rc, kind) - if os.path.islink(where): - os.unlink(where) - os.symlink(os.path.join("..", "..", "archive", "example.org", - "{0}{1}.pem".format(kind, ver)), where) - with open(where, "w") as f: - f.write(kind) + self._write_out_kind(kind, ver) self.test_rc.update_all_links_to(3) self.assertEqual( 6, self.test_rc.save_successor(3, "new cert", None, @@ -563,39 +518,33 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertFalse(os.path.islink(self.test_rc.version("privkey", 10))) self.assertFalse(os.path.exists(temp_config_file)) - @mock.patch("certbot.cli.helpful_parser") - def test_relevant_values(self, mock_parser): + def _test_relevant_values_common(self, values): + option = "rsa_key_size" + mock_parser = mock.Mock(args=["--standalone"], verb="certonly", + defaults={option: cli.flag_default(option)}) + + from certbot.storage import relevant_values + with mock.patch("certbot.cli.helpful_parser", mock_parser): + return relevant_values(values) + + def test_relevant_values(self): """Test that relevant_values() can reject an irrelevant value.""" - # pylint: disable=protected-access - from certbot import storage - mock_parser.verb = "certonly" - mock_parser.args = ["--standalone"] - mock_action = mock.Mock(dest="rsa_key_size", default=2048) - mock_parser.parser._actions = [mock_action] - self.assertEqual(storage.relevant_values({"hello": "there"}), {}) + self.assertEqual( + self._test_relevant_values_common({"hello": "there"}), {}) - @mock.patch("certbot.cli.helpful_parser") - def test_relevant_values_default(self, mock_parser): + def test_relevant_values_default(self): """Test that relevant_values() can reject a default value.""" - # pylint: disable=protected-access - from certbot import storage - mock_parser.verb = "certonly" - mock_parser.args = ["--standalone"] - mock_action = mock.Mock(dest="rsa_key_size", default=2048) - mock_parser.parser._actions = [mock_action] - self.assertEqual(storage.relevant_values({"rsa_key_size": 2048}), {}) + option = "rsa_key_size" + values = {option: cli.flag_default(option)} + self.assertEqual(self._test_relevant_values_common(values), {}) - @mock.patch("certbot.cli.helpful_parser") - def test_relevant_values_nondefault(self, mock_parser): + def test_relevant_values_nondefault(self): """Test that relevant_values() can retain a non-default value.""" - # pylint: disable=protected-access - from certbot import storage - mock_parser.verb = "certonly" - mock_parser.args = ["--standalone"] - mock_action = mock.Mock(dest="rsa_key_size", default=2048) - mock_parser.parser._actions = [mock_action] - self.assertEqual(storage.relevant_values({"rsa_key_size": 12}), - {"rsa_key_size": 12}) + values = {"rsa_key_size": 12} + # A copy is given to _test_relevant_values_common + # to make sure values isn't modified by the method + self.assertEqual( + self._test_relevant_values_common(values.copy()), values) @mock.patch("certbot.storage.relevant_values") def test_new_lineage(self, mock_rv): @@ -659,7 +608,7 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue(os.path.exists(os.path.join( self.cli_config.archive_dir, "the-lineage.com", "privkey1.pem"))) - @mock.patch("certbot.storage.le_util.unique_lineage_name") + @mock.patch("certbot.storage.util.unique_lineage_name") def test_invalid_config_filename(self, mock_uln): from certbot import storage mock_uln.return_value = "this_does_not_end_with_dot_conf", "yikes" @@ -760,11 +709,14 @@ class RenewableCertTests(BaseRenewableCertTest): with open(temp2, "r") as f: content = f.read() # useful value was updated - assert "useful = new_value" in content + self.assertTrue("useful = new_value" in content) # associated comment was preserved - assert "A useful value" in content + self.assertTrue("A useful value" in content) # useless value was deleted - assert "useless" not in content + self.assertTrue("useless" not in content) + # check version was stored + self.assertTrue("version = {0}".format(certbot.__version__) in content) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/tests/testdata/cert-5sans.pem b/certbot/tests/testdata/cert-5sans.pem new file mode 100644 index 000000000..5de7cc6cb --- /dev/null +++ b/certbot/tests/testdata/cert-5sans.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICkTCCAjugAwIBAgIJAJNbfABWQ8bbMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp +c2NvMScwJQYDVQQKDB5FbGVjdHJvbmljIEZyb250aWVyIEZvdW5kYXRpb24xFDAS +BgNVBAMMC2V4YW1wbGUuY29tMB4XDTE2MDYwOTIzMDEzNloXDTE2MDcwOTIzMDEz +NloweTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xJzAlBgNVBAoMHkVsZWN0cm9uaWMgRnJvbnRpZXIgRm91 +bmRhdGlvbjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wXDANBgkqhkiG9w0BAQEFAANL +ADBIAkEArHVztFHtH92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580rv2+6QWE +30cWgdmJS86ObRz6lUTor4R0T+3C5QIDAQABo4GlMIGiMB0GA1UdDgQWBBQmz8jt +S9eUsuQlA1gkjwTAdNWXijAfBgNVHSMEGDAWgBQmz8jtS9eUsuQlA1gkjwTAdNWX +ijAMBgNVHRMEBTADAQH/MFIGA1UdEQRLMEmCDWEuZXhhbXBsZS5jb22CDWIuZXhh +bXBsZS5jb22CDWMuZXhhbXBsZS5jb22CDWQuZXhhbXBsZS5jb22CC2V4YW1wbGUu +Y29tMA0GCSqGSIb3DQEBCwUAA0EAVXmZxB+IJdgFvY2InOYeytTD1QmouDZRtj/T +H/HIpSdsfO7qr4d/ZprI2IhLRxp2S4BiU5Qc5HUkeADcpNd06A== +-----END CERTIFICATE----- diff --git a/certbot/tests/testdata/csr-nonames.pem b/certbot/tests/testdata/csr-nonames.pem new file mode 100644 index 000000000..abe1029ca --- /dev/null +++ b/certbot/tests/testdata/csr-nonames.pem @@ -0,0 +1,8 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIH/MIGqAgEAMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwXDANBgkqhkiG9w0BAQEF +AANLADBIAkEArHVztFHtH92ucFJD/N/HW9AsdRsUuHUBBBDlHwNlRd3fp580rv2+ +6QWE30cWgdmJS86ObRz6lUTor4R0T+3C5QIDAQABoAAwDQYJKoZIhvcNAQELBQAD +QQBt9XLSZ9DGfWcGGaBUTCiSY7lWBegpNlCeo8pK3ydWmKpjcza+j7lF5paph2LH +lKWVQ8+xwYMscGWK0NApHGco +-----END CERTIFICATE REQUEST----- diff --git a/certbot/tests/testdata/os-release b/certbot/tests/testdata/os-release new file mode 100644 index 000000000..cd5297acf --- /dev/null +++ b/certbot/tests/testdata/os-release @@ -0,0 +1,7 @@ +NAME="SystemdOS" +VERSION="42.42.42 LTS, Unreal" +ID=systemdos +ID_LIKE=debian +VERSION_ID="42" +HOME_URL="http://www.example.com/" +SUPPORT_URL="http://help.example.com/" diff --git a/certbot/tests/le_util_test.py b/certbot/tests/util_test.py similarity index 66% rename from certbot/tests/le_util_test.py rename to certbot/tests/util_test.py index b6da4525f..8e1b330ed 100644 --- a/certbot/tests/le_util_test.py +++ b/certbot/tests/util_test.py @@ -1,4 +1,4 @@ -"""Tests for certbot.le_util.""" +"""Tests for certbot.util.""" import argparse import errno import os @@ -11,16 +11,17 @@ import mock import six from certbot import errors +from certbot.tests import test_util class RunScriptTest(unittest.TestCase): - """Tests for certbot.le_util.run_script.""" + """Tests for certbot.util.run_script.""" @classmethod def _call(cls, params): - from certbot.le_util import run_script + from certbot.util import run_script return run_script(params) - @mock.patch("certbot.le_util.subprocess.Popen") + @mock.patch("certbot.util.subprocess.Popen") def test_default(self, mock_popen): """These will be changed soon enough with reload.""" mock_popen().returncode = 0 @@ -30,13 +31,13 @@ class RunScriptTest(unittest.TestCase): self.assertEqual(out, "stdout") self.assertEqual(err, "stderr") - @mock.patch("certbot.le_util.subprocess.Popen") + @mock.patch("certbot.util.subprocess.Popen") def test_bad_process(self, mock_popen): mock_popen.side_effect = OSError self.assertRaises(errors.SubprocessError, self._call, ["test"]) - @mock.patch("certbot.le_util.subprocess.Popen") + @mock.patch("certbot.util.subprocess.Popen") def test_failure(self, mock_popen): mock_popen().communicate.return_value = ("", "") mock_popen().returncode = 1 @@ -45,29 +46,29 @@ class RunScriptTest(unittest.TestCase): class ExeExistsTest(unittest.TestCase): - """Tests for certbot.le_util.exe_exists.""" + """Tests for certbot.util.exe_exists.""" @classmethod def _call(cls, exe): - from certbot.le_util import exe_exists + from certbot.util import exe_exists return exe_exists(exe) - @mock.patch("certbot.le_util.os.path.isfile") - @mock.patch("certbot.le_util.os.access") + @mock.patch("certbot.util.os.path.isfile") + @mock.patch("certbot.util.os.access") def test_full_path(self, mock_access, mock_isfile): mock_access.return_value = True mock_isfile.return_value = True self.assertTrue(self._call("/path/to/exe")) - @mock.patch("certbot.le_util.os.path.isfile") - @mock.patch("certbot.le_util.os.access") + @mock.patch("certbot.util.os.path.isfile") + @mock.patch("certbot.util.os.access") def test_on_path(self, mock_access, mock_isfile): mock_access.return_value = True mock_isfile.return_value = True self.assertTrue(self._call("exe")) - @mock.patch("certbot.le_util.os.path.isfile") - @mock.patch("certbot.le_util.os.access") + @mock.patch("certbot.util.os.path.isfile") + @mock.patch("certbot.util.os.access") def test_not_found(self, mock_access, mock_isfile): mock_access.return_value = False mock_isfile.return_value = True @@ -75,7 +76,7 @@ class ExeExistsTest(unittest.TestCase): class MakeOrVerifyDirTest(unittest.TestCase): - """Tests for certbot.le_util.make_or_verify_dir. + """Tests for certbot.util.make_or_verify_dir. Note that it is not possible to test for a wrong directory owner, as this testing script would have to be run as root. @@ -93,7 +94,7 @@ class MakeOrVerifyDirTest(unittest.TestCase): shutil.rmtree(self.root_path, ignore_errors=True) def _call(self, directory, mode): - from certbot.le_util import make_or_verify_dir + from certbot.util import make_or_verify_dir return make_or_verify_dir(directory, mode, self.uid, strict=True) def test_creates_dir_when_missing(self): @@ -116,7 +117,7 @@ class MakeOrVerifyDirTest(unittest.TestCase): class CheckPermissionsTest(unittest.TestCase): - """Tests for certbot.le_util.check_permissions. + """Tests for certbot.util.check_permissions. Note that it is not possible to test for a wrong file owner, as this testing script would have to be run as root. @@ -131,7 +132,7 @@ class CheckPermissionsTest(unittest.TestCase): os.remove(self.path) def _call(self, mode): - from certbot.le_util import check_permissions + from certbot.util import check_permissions return check_permissions(self.path, mode, self.uid) def test_ok_mode(self): @@ -144,7 +145,7 @@ class CheckPermissionsTest(unittest.TestCase): class UniqueFileTest(unittest.TestCase): - """Tests for certbot.le_util.unique_file.""" + """Tests for certbot.util.unique_file.""" def setUp(self): self.root_path = tempfile.mkdtemp() @@ -154,7 +155,7 @@ class UniqueFileTest(unittest.TestCase): shutil.rmtree(self.root_path, ignore_errors=True) def _call(self, mode=0o600): - from certbot.le_util import unique_file + from certbot.util import unique_file return unique_file(self.default_name, mode) def test_returns_fd_for_writing(self): @@ -189,7 +190,7 @@ class UniqueFileTest(unittest.TestCase): class UniqueLineageNameTest(unittest.TestCase): - """Tests for certbot.le_util.unique_lineage_name.""" + """Tests for certbot.util.unique_lineage_name.""" def setUp(self): self.root_path = tempfile.mkdtemp() @@ -198,7 +199,7 @@ class UniqueLineageNameTest(unittest.TestCase): shutil.rmtree(self.root_path, ignore_errors=True) def _call(self, filename, mode=0o777): - from certbot.le_util import unique_lineage_name + from certbot.util import unique_lineage_name return unique_lineage_name(self.root_path, filename, mode) def test_basic(self): @@ -213,14 +214,14 @@ class UniqueLineageNameTest(unittest.TestCase): self.assertTrue(isinstance(name, str)) self.assertTrue("wow-0009.conf" in name) - @mock.patch("certbot.le_util.os.fdopen") + @mock.patch("certbot.util.os.fdopen") def test_failure(self, mock_fdopen): err = OSError("whoops") err.errno = errno.EIO mock_fdopen.side_effect = err self.assertRaises(OSError, self._call, "wow") - @mock.patch("certbot.le_util.os.fdopen") + @mock.patch("certbot.util.os.fdopen") def test_subsequent_failure(self, mock_fdopen): self._call("wow") err = OSError("whoops") @@ -230,7 +231,7 @@ class UniqueLineageNameTest(unittest.TestCase): class SafelyRemoveTest(unittest.TestCase): - """Tests for certbot.le_util.safely_remove.""" + """Tests for certbot.util.safely_remove.""" def setUp(self): self.tmp = tempfile.mkdtemp() @@ -240,7 +241,7 @@ class SafelyRemoveTest(unittest.TestCase): shutil.rmtree(self.tmp) def _call(self): - from certbot.le_util import safely_remove + from certbot.util import safely_remove return safely_remove(self.path) def test_exists(self): @@ -254,7 +255,7 @@ class SafelyRemoveTest(unittest.TestCase): # no error, yay! self.assertFalse(os.path.exists(self.path)) - @mock.patch("certbot.le_util.os.remove") + @mock.patch("certbot.util.os.remove") def test_other_error_passthrough(self, mock_remove): mock_remove.side_effect = OSError self.assertRaises(OSError, self._call) @@ -264,7 +265,7 @@ class SafeEmailTest(unittest.TestCase): """Test safe_email.""" @classmethod def _call(cls, addr): - from certbot.le_util import safe_email + from certbot.util import safe_email return safe_email(addr) def test_valid_emails(self): @@ -292,7 +293,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): self.parser = argparse.ArgumentParser() def _call(self, argument_name, nargs): - from certbot.le_util import add_deprecated_argument + from certbot.util import add_deprecated_argument add_deprecated_argument(self.parser.add_argument, argument_name, nargs) @@ -308,14 +309,14 @@ class AddDeprecatedArgumentTest(unittest.TestCase): def _get_argparse_warnings(self, args): stderr = six.StringIO() - with mock.patch("certbot.le_util.sys.stderr", new=stderr): + with mock.patch("certbot.util.sys.stderr", new=stderr): self.parser.parse_args(args) return stderr.getvalue() def test_help(self): self._call("--old-option", 2) stdout = six.StringIO() - with mock.patch("certbot.le_util.sys.stdout", new=stdout): + with mock.patch("certbot.util.sys.stdout", new=stdout): try: self.parser.parse_args(["-h"]) except SystemExit: @@ -327,7 +328,7 @@ class EnforceDomainSanityTest(unittest.TestCase): """Test enforce_domain_sanity.""" def _call(self, domain): - from certbot.le_util import enforce_domain_sanity + from certbot.util import enforce_domain_sanity return enforce_domain_sanity(domain) def test_nonascii_str(self): @@ -339,5 +340,67 @@ class EnforceDomainSanityTest(unittest.TestCase): u"eichh\u00f6rnchen.example.com") +class OsInfoTest(unittest.TestCase): + """Test OS / distribution detection""" + + def test_systemd_os_release(self): + from certbot.util import (get_os_info, get_systemd_os_info, + get_os_info_ua) + + with mock.patch('os.path.isfile', return_value=True): + self.assertEqual(get_os_info( + test_util.vector_path("os-release"))[0], 'systemdos') + self.assertEqual(get_os_info( + test_util.vector_path("os-release"))[1], '42') + self.assertEqual(get_systemd_os_info("/dev/null"), ("", "")) + self.assertEqual(get_os_info_ua( + test_util.vector_path("os-release")), + "SystemdOS") + with mock.patch('os.path.isfile', return_value=False): + self.assertEqual(get_systemd_os_info(), ("", "")) + + @mock.patch("certbot.util.subprocess.Popen") + def test_non_systemd_os_info(self, popen_mock): + from certbot.util import (get_os_info, get_python_os_info, + get_os_info_ua) + with mock.patch('os.path.isfile', return_value=False): + with mock.patch('platform.system_alias', + return_value=('NonSystemD', '42', '42')): + self.assertEqual(get_os_info()[0], 'nonsystemd') + self.assertEqual(get_os_info_ua(), + " ".join(get_python_os_info())) + + with mock.patch('platform.system_alias', + return_value=('darwin', '', '')): + comm_mock = mock.Mock() + comm_attrs = {'communicate.return_value': + ('42.42.42', 'error')} + comm_mock.configure_mock(**comm_attrs) # pylint: disable=star-args + popen_mock.return_value = comm_mock + self.assertEqual(get_os_info()[0], 'darwin') + self.assertEqual(get_os_info()[1], '42.42.42') + + with mock.patch('platform.system_alias', + return_value=('linux', '', '')): + with mock.patch('platform.linux_distribution', + return_value=('', '', '')): + self.assertEqual(get_python_os_info(), ("linux", "")) + + with mock.patch('platform.linux_distribution', + return_value=('testdist', '42', '')): + self.assertEqual(get_python_os_info(), ("testdist", "42")) + + with mock.patch('platform.system_alias', + return_value=('freebsd', '9.3-RC3-p1', '')): + self.assertEqual(get_python_os_info(), ("freebsd", "9")) + + with mock.patch('platform.system_alias', + return_value=('windows', '', '')): + with mock.patch('platform.win32_ver', + return_value=('4242', '95', '2', '')): + self.assertEqual(get_python_os_info(), + ("windows", "95")) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot/le_util.py b/certbot/util.py similarity index 73% rename from certbot/le_util.py rename to certbot/util.py index f5148b949..301fc669b 100644 --- a/certbot/le_util.py +++ b/certbot/util.py @@ -1,6 +1,9 @@ """Utilities for all Certbot.""" import argparse import collections +# distutils.version under virtualenv confuses pylint +# For more info, see: https://github.com/PyCQA/pylint/issues/73 +import distutils.version # pylint: disable=import-error,no-name-in-module import errno import logging import os @@ -151,7 +154,8 @@ def _unique_file(path, filename_pat, count, mode): while True: current_path = os.path.join(path, filename_pat(count)) try: - return safe_open(current_path, chmod=mode), current_path + return safe_open(current_path, chmod=mode),\ + os.path.abspath(current_path) except OSError as err: # "File exists," is okay, try a different name. if err.errno != errno.EEXIST: @@ -209,9 +213,95 @@ def safely_remove(path): raise -def get_os_info(): +def get_os_info(filepath="/etc/os-release"): + """ + Get OS name and version + + :param str filepath: File path of os-release file + :returns: (os_name, os_version) + :rtype: `tuple` of `str` + """ + + if os.path.isfile(filepath): + # Systemd os-release parsing might be viable + os_name, os_version = get_systemd_os_info(filepath=filepath) + if os_name: + return (os_name, os_version) + + # Fallback to platform module + return get_python_os_info() + + +def get_os_info_ua(filepath="/etc/os-release"): + """ + Get OS name and version string for User Agent + + :param str filepath: File path of os-release file + :returns: os_ua + :rtype: `str` + """ + + if os.path.isfile(filepath): + os_ua = _get_systemd_os_release_var("PRETTY_NAME", filepath=filepath) + if not os_ua: + os_ua = _get_systemd_os_release_var("NAME", filepath=filepath) + if os_ua: + return os_ua + + # Fallback + return " ".join(get_python_os_info()) + + +def get_systemd_os_info(filepath="/etc/os-release"): + """ + Parse systemd /etc/os-release for distribution information + + :param str filepath: File path of os-release file + :returns: (os_name, os_version) + :rtype: `tuple` of `str` + """ + + os_name = _get_systemd_os_release_var("ID", filepath=filepath) + os_version = _get_systemd_os_release_var("VERSION_ID", filepath=filepath) + + return (os_name, os_version) + + +def _get_systemd_os_release_var(varname, filepath="/etc/os-release"): + """ + Get single value from systemd /etc/os-release + + :param str varname: Name of variable to fetch + :param str filepath: File path of os-release file + :returns: requested value + :rtype: `str` + """ + + var_string = varname+"=" + if not os.path.isfile(filepath): + return "" + with open(filepath, 'r') as fh: + contents = fh.readlines() + + for line in contents: + if line.strip().startswith(var_string): + # Return the value of var, normalized + return _normalize_string(line.strip()[len(var_string):]) + return "" + + +def _normalize_string(orig): + """ + Helper function for _get_systemd_os_release_var() to remove quotes + and whitespaces + """ + return orig.replace('"', '').replace("'", "").strip() + + +def get_python_os_info(): """ Get Operating System type/distribution and major version + using python platform module :returns: (os_name, os_version) :rtype: `tuple` of `str` @@ -235,8 +325,7 @@ def get_os_info(): os_ver = subprocess.Popen( ["sw_vers", "-productVersion"], stdout=subprocess.PIPE - ).communicate()[0] - os_ver = os_ver.partition(".")[0] + ).communicate()[0].rstrip('\n') elif os_type.startswith('freebsd'): # eg "9.3-RC3-p1" os_ver = os_ver.partition("-")[0] @@ -313,7 +402,7 @@ def enforce_domain_sanity(domain): domain = domain.encode('ascii').lower() except UnicodeError: error_fmt = (u"Internationalized domain names " - "are not presently supported: {0}") + "are not presently supported: {0}") if isinstance(domain, six.text_type): raise errors.ConfigurationError(error_fmt.format(domain)) else: @@ -334,11 +423,29 @@ def enforce_domain_sanity(domain): # It wasn't an IP address, so that's good pass - # FQDN checks from - # http://www.mkyong.com/regular-expressions/domain-name-regular-expression-example/ - # Characters used, domain parts < 63 chars, tld > 1 < 64 chars - # first and last char is not "-" - fqdn = re.compile("^((?!-)[A-Za-z0-9-]{1,63}(? 255: + raise errors.ConfigurationError(msg + "it is too long.") + return domain + + +def get_strict_version(normalized): + """Converts a normalized version to a strict version. + + :param str normalized: normalized version string + + :returns: An equivalent strict version + :rtype: distutils.version.StrictVersion + + """ + # strict version ending with "a" and a number designates a pre-release + # pylint: disable=no-member + return distutils.version.StrictVersion(normalized.replace(".dev", "a")) diff --git a/docs/api/le_util.rst b/docs/api/le_util.rst deleted file mode 100644 index c9e332745..000000000 --- a/docs/api/le_util.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`certbot.le_util` --------------------------- - -.. automodule:: certbot.le_util - :members: diff --git a/docs/api/util.rst b/docs/api/util.rst new file mode 100644 index 000000000..7d0e33501 --- /dev/null +++ b/docs/api/util.rst @@ -0,0 +1,5 @@ +:mod:`certbot.util` +-------------------------- + +.. automodule:: certbot.util + :members: diff --git a/docs/ciphers.rst b/docs/ciphers.rst index ef644b7a0..8996dd9ef 100644 --- a/docs/ciphers.rst +++ b/docs/ciphers.rst @@ -17,15 +17,13 @@ Autoupdates Within certain limits, TLS server software can choose what kind of cryptography to use when a client connects. These choices can affect security, compatibility, and performance in complex ways. Most of -these options are independent of a particular certificate. The Let's -Encrypt client tries to provide defaults that we think are most useful -to our users. +these options are independent of a particular certificate. Certbot +tries to provide defaults that we think are most useful to our users. -As described below, the Let's Encrypt client will default to modifying +As described below, Certbot will default to modifying server software's cryptographic settings to keep these up-to-date with -what we think are appropriate defaults when new versions of the Let's -Encrypt client are installed (for example, by an operating system package -manager). +what we think are appropriate defaults when new versions of the Certbot +are installed (for example, by an operating system package manager). When this feature is implemented, this document will be updated to describe how to disable these automatic changes. @@ -54,7 +52,7 @@ improve, others' security. But important information that improves our understanding of the state of the art is published regularly. When enabling TLS support in a compatible web server (which is a separate -step from obtaining a certificate), Let's Encrypt has the ability to +step from obtaining a certificate), Certbot has the ability to update that web server's TLS configuration. Again, this is *different from the cryptographic particulars of the certificate itself*; the certificate as of the initial release will be RSA-signed using one of @@ -80,30 +78,29 @@ art. However, the Let's Encrypt certificate authority does *not* dictate end-users' security policy, and any site is welcome to change its preferences in accordance with its own policy or its administrators' preferences, and use different cryptographic mechanisms or parameters, -or a different priority order, than the defaults provided by the Let's -Encrypt client. +or a different priority order, than the defaults provided by Certbot. -If you don't use the Let's Encrypt client to configure your server -directly, because the client doesn't integrate with your server software -or because you chose not to use this integration, then the cryptographic -defaults haven't been modified, and the cryptography chosen by the server -will still be whatever the default for your software was. For example, -if you obtain a certificate using *standalone* mode and then manually -install it in an IMAP or LDAP server, your cryptographic settings will -not be modified by the client in any way. +If you don't use Certbot to configure your server directly, because the +client doesn't integrate with your server software or because you chose +not to use this integration, then the cryptographic defaults haven't been +modified, and the cryptography chosen by the server will still be whatever +the default for your software was. For example, if you obtain a +certificate using *standalone* mode and then manually install it in an IMAP +or LDAP server, your cryptographic settings will not be modified by the +client in any way. Sources of defaults ------------------- -Initially, the Let's Encrypt client will configure users' servers to -use the cryptographic defaults recommended by the Mozilla project. -These settings are well-reasoned recommendations that carefully -consider client software compatibility. They are described at +Initially, Certbot will configure users' servers to use the cryptographic +defaults recommended by the Mozilla project. These settings are well-reasoned +recommendations that carefully consider client software compatibility. They +are described at https://wiki.mozilla.org/Security/Server_Side_TLS -and the version implemented by the Let's Encrypt client will be the +and the version implemented by Certbot will be the version that was most current as of the release date of each client version. Mozilla offers three separate sets of cryptographic options, which trade off security and compatibility differently. These are @@ -113,12 +110,12 @@ to most-backwards compatible). The client will follow the Mozilla defaults for the *Intermediate* configuration by default, at least with regards to ciphersuites and TLS versions. Mozilla's web site describes which client software will be compatible with each configuration. You can also use -the Qualys SSL Labs site, which the Let's Encrypt software will suggest +the Qualys SSL Labs site, which Certbot will suggest when installing a certificate, to test your server and see whether it will be compatible with particular software versions. -It will be possible to ask the Let's Encrypt client to instead apply -(and track) Modern or Old configurations. +It will be possible to ask Certbot to instead apply (and track) Modern +or Old configurations. The Let's Encrypt project expects to follow the Mozilla recommendations in the future as those recommendations are updated. (For example, some @@ -127,15 +124,15 @@ which uses the ChaCha and Poly1305 algorithms, and which is already implemented by the Chrome browser. Mozilla has delayed recommending ``0xcc13`` over compatibility and standardization concerns, but is likely to recommend it in the future once these concerns have been addressed. At -that point, the Let's Encrypt client would likely follow the Mozilla -recommendations and favor the use of this ciphersuite as well.) +that point, Certbot would likely follow the Mozilla recommendations and favor +the use of this ciphersuite as well.) The Let's Encrypt project may deviate from the Mozilla recommendations in the future if good cause is shown and we believe our users' priorities would be well-served by doing so. In general, please address relevant proposals for changing priorities to the Mozilla security -team first, before asking the Let's Encrypt project to change the -client's priorities. The Mozilla security team is likely to have more +team first, before asking the Certbot developers to change +Certbot's priorities. The Mozilla security team is likely to have more resources and expertise to bring to bear on evaluating reasons why its recommendations should be updated. @@ -144,8 +141,8 @@ small number of alternative configurations (apart from Modern, Intermediate, and Old) that there's reason to believe would be widely used by sysadmins; this would usually be a preferable course to modifying an existing configuration. For example, if many sysadmins want their -servers configured to track a different expert recommendation, Let's -Encrypt could add an option to do so. +servers configured to track a different expert recommendation, Certbot +could add an option to do so. Resources for recommendations @@ -156,9 +153,9 @@ recommendations with sources of expert guidance on ciphersuites and other cryptographic parameters. We're grateful to everyone who contributed suggestions. The recommendations we received are available at -https://github.com/letsencrypt/letsencrypt/wiki/Ciphersuite-guidance +https://github.com/certbot/certbot/wiki/Ciphersuite-guidance -Let's Encrypt client users are welcome to review these authorities to +Certbot users are welcome to review these authorities to better inform their own cryptographic parameter choices. We also welcome suggestions of other resources to add to this list. Please keep in mind that different recommendations may reflect different priorities @@ -172,26 +169,25 @@ This will probably look something like .. code-block:: shell - letsencrypt --cipher-recommendations mozilla-secure - letsencrypt --cipher-recommendations mozilla-intermediate - letsencrypt --cipher-recommendations mozilla-old + certbot --cipher-recommendations mozilla-secure + certbot --cipher-recommendations mozilla-intermediate + certbot --cipher-recommendations mozilla-old to track Mozilla's *Secure*, *Intermediate*, or *Old* recommendations, and .. code-block:: shell - letsencrypt --update-ciphers on + certbot --update-ciphers on -to enable updating ciphers with each new Let's Encrypt client release, -or +to enable updating ciphers with each new Certbot release, or .. code-block:: shell - letsencrypt --update-ciphers off + certbot --update-ciphers off to disable automatic configuration updates. These features have not yet -been implemented and this syntax may change then they are implemented. +been implemented and this syntax may change when they are implemented. TODO @@ -200,7 +196,7 @@ TODO The status of this feature is tracked as part of issue #1123 in our bug tracker. -https://github.com/letsencrypt/letsencrypt/issues/1123 +https://github.com/certbot/certbot/issues/1123 Prior to implementation of #1123, the client does not actually modify ciphersuites (this is intended to be implemented as a "configuration diff --git a/docs/cli-help.txt b/docs/cli-help.txt new file mode 100644 index 000000000..749983d0e --- /dev/null +++ b/docs/cli-help.txt @@ -0,0 +1,367 @@ +usage: + certbot [SUBCOMMAND] [options] [-d domain] [-d domain] ... + +Certbot can obtain and install HTTPS/TLS/SSL certificates. By default, +it will attempt to use a webserver both for obtaining and installing the +cert. Major SUBCOMMANDS are: + + (default) run Obtain & install a cert in your current webserver + certonly Obtain cert, but do not install it (aka "auth") + install Install a previously obtained cert in a server + renew Renew previously obtained certs that are near expiry + revoke Revoke a previously obtained certificate + register Perform tasks related to registering with the CA + rollback Rollback server configuration changes made during install + config_changes Show changes made to server config during installation + plugins Display information about installed plugins + +optional arguments: + -h, --help show this help message and exit + -c CONFIG_FILE, --config CONFIG_FILE + config file path (default: None) + -v, --verbose This flag can be used multiple times to incrementally + increase the verbosity of output, e.g. -vvv. (default: + -3) + -t, --text Use the text output instead of the curses UI. + (default: False) + -n, --non-interactive, --noninteractive + Run without ever asking for user input. This may + require additional command line flags; the client will + try to explain which ones are required if it finds one + missing (default: False) + --dialog Run using dialog (default: False) + --dry-run Perform a test run of the client, obtaining test + (invalid) certs but not saving them to disk. This can + currently only be used with the 'certonly' and 'renew' + subcommands. Note: Although --dry-run tries to avoid + making any persistent changes on a system, it is not + completely side-effect free: if used with webserver + authenticator plugins like apache and nginx, it makes + and then reverts temporary config changes in order to + obtain test certs, and reloads webservers to deploy + and then roll back those changes. It also calls --pre- + hook and --post-hook commands if they are defined + because they may be necessary to accurately simulate + renewal. --renew-hook commands are not called. + (default: False) + --register-unsafely-without-email + Specifying this flag enables registering an account + with no email address. This is strongly discouraged, + because in the event of key loss or account compromise + you will irrevocably lose access to your account. You + will also be unable to receive notice about impending + expiration or revocation of your certificates. Updates + to the Subscriber Agreement will still affect you, and + will be effective 14 days after posting an update to + the web site. (default: False) + --update-registration + With the register verb, indicates that details + associated with an existing registration, such as the + e-mail address, should be updated, rather than + registering a new account. (default: False) + -m EMAIL, --email EMAIL + Email used for registration and recovery contact. + (default: None) + -d DOMAIN, --domains DOMAIN, --domain DOMAIN + Domain names to apply. For multiple domains you can + use multiple -d flags or enter a comma separated list + of domains as a parameter. (default: []) + --user-agent USER_AGENT + Set a custom user agent string for the client. User + agent strings allow the CA to collect high level + statistics about success rates by OS and plugin. If + you wish to hide your server OS version from the Let's + Encrypt server, set this to "". (default: None) + +automation: + Arguments for automating execution & other tweaks + + --keep-until-expiring, --keep, --reinstall + If the requested cert matches an existing cert, always + keep the existing one until it is due for renewal (for + the 'run' subcommand this means reinstall the existing + cert) (default: False) + --expand If an existing cert covers some subset of the + requested names, always expand and replace it with the + additional names. (default: False) + --version show program's version number and exit + --force-renewal, --renew-by-default + If a certificate already exists for the requested + domains, renew it now, regardless of whether it is + near expiry. (Often --keep-until-expiring is more + appropriate). Also implies --expand. (default: False) + --allow-subset-of-names + When performing domain validation, do not consider it + a failure if authorizations can not be obtained for a + strict subset of the requested domains. This may be + useful for allowing renewals for multiple domains to + succeed even if some domains no longer point at this + system. This option cannot be used with --csr. + (default: False) + --agree-tos Agree to the ACME Subscriber Agreement (default: + False) + --account ACCOUNT_ID Account ID to use (default: None) + --duplicate Allow making a certificate lineage that duplicates an + existing one (both can be renewed in parallel) + (default: False) + --os-packages-only (letsencrypt-auto only) install OS package + dependencies and then stop (default: False) + --no-self-upgrade (letsencrypt-auto only) prevent the letsencrypt-auto + script from upgrading itself to newer released + versions (default: False) + -q, --quiet Silence all output except errors. Useful for + automation via cron. Implies --non-interactive. + (default: False) + +testing: + The following flags are meant for testing purposes only! Do NOT change + them, unless you really know what you're doing! + + --debug Show tracebacks in case of errors, and allow + letsencrypt-auto execution on experimental platforms + (default: False) + --no-verify-ssl Disable SSL certificate verification. (default: False) + --tls-sni-01-port TLS_SNI_01_PORT + Port number to perform tls-sni-01 challenge. Boulder + in testing mode defaults to 5001. (default: 443) + --http-01-port HTTP01_PORT + Port used in the SimpleHttp challenge. (default: 80) + --break-my-certs Be willing to replace or renew valid certs with + invalid (testing/staging) certs (default: False) + --test-cert, --staging + Use the staging server to obtain test (invalid) certs; + equivalent to --server https://acme- + staging.api.letsencrypt.org/directory (default: False) + +security: + Security parameters & server settings + + --rsa-key-size N Size of the RSA key. (default: 2048) + --must-staple Adds the OCSP Must Staple extension to the + certificate. Autoconfigures OCSP Stapling for + supported setups (Apache version >= 2.3.3 ). (default: + False) + --redirect Automatically redirect all HTTP traffic to HTTPS for + the newly authenticated vhost. (default: None) + --no-redirect Do not automatically redirect all HTTP traffic to + HTTPS for the newly authenticated vhost. (default: + None) + --hsts Add the Strict-Transport-Security header to every HTTP + response. Forcing browser to always use SSL for + the domain. Defends against SSL Stripping. (default: + False) + --no-hsts Do not automatically add the Strict-Transport-Security + header to every HTTP response. (default: False) + --uir Add the "Content-Security-Policy: upgrade-insecure- + requests" header to every HTTP response. Forcing the + browser to use https:// for every http:// resource. + (default: None) + --no-uir Do not automatically set the "Content-Security-Policy: + upgrade-insecure-requests" header to every HTTP + response. (default: None) + --staple-ocsp Enables OCSP Stapling. A valid OCSP response is + stapled to the certificate that the server offers + during TLS. (default: None) + --no-staple-ocsp Do not automatically enable OCSP Stapling. (default: + None) + --strict-permissions Require that all configuration files are owned by the + current user; only needed if your config is somewhere + unsafe like /tmp/ (default: False) + +renew: + The 'renew' subcommand will attempt to renew all certificates (or more + precisely, certificate lineages) you have previously obtained if they are + close to expiry, and print a summary of the results. By default, 'renew' + will reuse the options used to create obtain or most recently successfully + renew each certificate lineage. You can try it with `--dry-run` first. For + more fine-grained control, you can renew individual lineages with the + `certonly` subcommand. Hooks are available to run commands before and + after renewal; see https://certbot.eff.org/docs/using.html#renewal for + more information on these. + + --pre-hook PRE_HOOK Command to be run in a shell before obtaining any + certificates. Intended primarily for renewal, where it + can be used to temporarily shut down a webserver that + might conflict with the standalone plugin. This will + only be called if a certificate is actually to be + obtained/renewed. (default: None) + --post-hook POST_HOOK + Command to be run in a shell after attempting to + obtain/renew certificates. Can be used to deploy + renewed certificates, or to restart any servers that + were stopped by --pre-hook. This is only run if an + attempt was made to obtain/renew a certificate. + (default: None) + --renew-hook RENEW_HOOK + Command to be run in a shell once for each + successfully renewed certificate.For this command, the + shell variable $RENEWED_LINEAGE will point to + theconfig live subdirectory containing the new certs + and keys; the shell variable $RENEWED_DOMAINS will + contain a space-delimited list of renewed cert domains + (default: None) + --disable-hook-validation + Ordinarily the commands specified for --pre-hook + /--post-hook/--renew-hook will be checked for + validity, to see if the programs being run are in the + $PATH, so that mistakes can be caught early, even when + the hooks aren't being run just yet. The validation is + rather simplistic and fails if you use more advanced + shell constructs, so you can use this switch to + disable it. (default: True) + +certonly: + Options for modifying how a cert is obtained + + --csr CSR Path to a Certificate Signing Request (CSR) in DER + format; note that the .csr file *must* contain a + Subject Alternative Name field for each domain you + want certified. Currently --csr only works with the + 'certonly' subcommand' (default: None) + +install: + Options for modifying how a cert is deployed + +revoke: + Options for revocation of certs + +rollback: + Options for reverting config changes + + --checkpoints N Revert configuration N number of checkpoints. + (default: 1) + +plugins: + Plugin options + + --init Initialize plugins. (default: False) + --prepare Initialize and prepare plugins. (default: False) + --authenticators Limit to authenticator plugins only. (default: None) + --installers Limit to installer plugins only. (default: None) + +config_changes: + Options for showing a history of config changes + + --num NUM How many past revisions you want to be displayed + (default: None) + +paths: + Arguments changing execution paths & servers + + --cert-path CERT_PATH + Path to where cert is saved (with auth --csr), + installed from or revoked. (default: None) + --key-path KEY_PATH Path to private key for cert installation or + revocation (if account key is missing) (default: None) + --fullchain-path FULLCHAIN_PATH + Accompanying path to a full certificate chain (cert + plus chain). (default: None) + --chain-path CHAIN_PATH + Accompanying path to a certificate chain. (default: + None) + --config-dir CONFIG_DIR + Configuration directory. (default: /etc/letsencrypt) + --work-dir WORK_DIR Working directory. (default: /var/lib/letsencrypt) + --logs-dir LOGS_DIR Logs directory. (default: /var/log/letsencrypt) + --server SERVER ACME Directory Resource URI. (default: + https://acme-v01.api.letsencrypt.org/directory) + +plugins: + Certbot client supports an extensible plugins architecture. See 'certbot + plugins' for a list of all installed plugins and their names. You can + force a particular plugin by setting options provided below. Running + --help will list flags specific to that plugin. + + -a AUTHENTICATOR, --authenticator AUTHENTICATOR + Authenticator plugin name. (default: None) + -i INSTALLER, --installer INSTALLER + Installer plugin name (also used to find domains). + (default: None) + --configurator CONFIGURATOR + Name of the plugin that is both an authenticator and + an installer. Should not be used together with + --authenticator or --installer. (default: None) + --apache Obtain and install certs using Apache (default: False) + --nginx Obtain and install certs using Nginx (default: False) + --standalone Obtain certs using a "standalone" webserver. (default: + False) + --manual Provide laborious manual instructions for obtaining a + cert (default: False) + --webroot Obtain certs by placing files in a webroot directory. + (default: False) + +standalone: + Automatically use a temporary webserver + + --standalone-supported-challenges STANDALONE_SUPPORTED_CHALLENGES + Supported challenges. Preferred in the order they are + listed. (default: tls-sni-01,http-01) + +manual: + Manually configure an HTTP server + + --manual-test-mode Test mode. Executes the manual command in subprocess. + (default: False) + --manual-public-ip-logging-ok + Automatically allows public IP logging. (default: + False) + +nginx: + Nginx Web Server - currently doesn't work + + --nginx-server-root NGINX_SERVER_ROOT + Nginx server root directory. (default: /etc/nginx) + --nginx-ctl NGINX_CTL + Path to the 'nginx' binary, used for 'configtest' and + retrieving nginx version number. (default: nginx) + +webroot: + Place files in webroot directory + + --webroot-path WEBROOT_PATH, -w WEBROOT_PATH + public_html / webroot path. This can be specified + multiple times to handle different domains; each + domain will have the webroot path that preceded it. + For instance: `-w /var/www/example -d example.com -d + www.example.com -w /var/www/thing -d thing.net -d + m.thing.net` (default: []) + --webroot-map WEBROOT_MAP + JSON dictionary mapping domains to webroot paths; this + implies -d for each entry. You may need to escape this + from your shell. E.g.: --webroot-map + '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' + This option is merged with, but takes precedence over, + -w / -d entries. At present, if you put webroot-map in + a config file, it needs to be on a single line, like: + webroot-map = {"example.com":"/var/www"}. (default: + {}) + +apache: + Apache Web Server - Alpha + + --apache-enmod APACHE_ENMOD + Path to the Apache 'a2enmod' binary. (default: + a2enmod) + --apache-dismod APACHE_DISMOD + Path to the Apache 'a2dismod' binary. (default: + a2dismod) + --apache-le-vhost-ext APACHE_LE_VHOST_EXT + SSL vhost configuration extension. (default: -le- + ssl.conf) + --apache-server-root APACHE_SERVER_ROOT + Apache server root directory. (default: /etc/apache2) + --apache-vhost-root APACHE_VHOST_ROOT + Apache server VirtualHost configuration root (default: + /etc/apache2/sites-available) + --apache-challenge-location APACHE_CHALLENGE_LOCATION + Directory path for challenge configuration. (default: + /etc/apache2) + --apache-handle-modules APACHE_HANDLE_MODULES + Let installer handle enabling required modules for + you.(Only Ubuntu/Debian currently) (default: True) + --apache-handle-sites APACHE_HANDLE_SITES + Let installer handle enabling sites for you.(Only + Ubuntu/Debian currently) (default: True) + +null: + Null Installer diff --git a/docs/conf.py b/docs/conf.py index fb2bdea73..e387e1eae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,8 +64,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Let\'s Encrypt' -copyright = u'2014-2015, Let\'s Encrypt Project' +project = u'Certbot' +copyright = u'2014-2016 - The Certbot software and documentation are licensed under the Apache 2.0 license as described at https://eff.org/cb-license ' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -225,7 +225,7 @@ html_static_path = ['_static'] #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'LetsEncryptdoc' +htmlhelp_basename = 'Certbotdoc' # -- Options for LaTeX output --------------------------------------------- @@ -247,8 +247,8 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'LetsEncrypt.tex', u'Let\'s Encrypt Documentation', - u'Let\'s Encrypt Project', 'manual'), + ('index', 'Certbot.tex', u'Certbot Documentation', + u'Certbot Project', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -277,7 +277,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'certbot', u'Let\'s Encrypt Documentation', + ('index', 'certbot', u'Certbot Documentation', [project], 7), ('man/certbot', 'certbot', u'certbot script documentation', [project], 1), @@ -293,8 +293,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'LetsEncrypt', u'Let\'s Encrypt Documentation', - u'Let\'s Encrypt Project', 'LetsEncrypt', 'One line description of project.', + ('index', 'Certbot', u'Certbot Documentation', + u'Certbot Project', 'Certbot', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/contributing.rst b/docs/contributing.rst index 216cd4888..bdda0e2e6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -20,8 +20,8 @@ once: .. code-block:: shell - git clone https://github.com/letsencrypt/letsencrypt - cd letsencrypt + git clone https://github.com/certbot/certbot + cd certbot ./letsencrypt-auto-source/letsencrypt-auto --os-packages-only ./tools/venv.sh @@ -57,8 +57,8 @@ your pull request must have thorough unit test coverage, pass our `integration`_ tests, and be compliant with the :ref:`coding style `. -.. _github issue tracker: https://github.com/letsencrypt/letsencrypt/issues -.. _Good Volunteer Task: https://github.com/letsencrypt/letsencrypt/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+Volunteer+Task%22 +.. _github issue tracker: https://github.com/certbot/certbot/issues +.. _Good Volunteer Task: https://github.com/certbot/certbot/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+Volunteer+Task%22 Testing ------- @@ -71,6 +71,9 @@ The following tools are there to help you: experimental, non-production Apache2 install on them. ``tox -e apacheconftest`` can be used to run those specific Apache conf tests. +- ``tox --skip-missing-interpreters`` runs tox while ignoring missing versions + of Python needed for running the tests. + - ``tox -e py27``, ``tox -e py26`` etc, run unit tests for specific Python versions. @@ -175,8 +178,8 @@ Configurators may implement just one of those). There are also `~certbot.interfaces.IDisplay` plugins, which implement bindings to alternative UI libraries. -.. _interfaces.py: https://github.com/letsencrypt/letsencrypt/blob/master/certbot/interfaces.py -.. _plugins/common.py: https://github.com/letsencrypt/letsencrypt/blob/master/certbot/plugins/common.py#L34 +.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py +.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 Authenticators @@ -266,8 +269,7 @@ with the core upstream source code. An example is provided in it with any necessary API changes. .. _`setuptools entry points`: - https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins - + http://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _coding-style: @@ -297,7 +299,7 @@ Please: 4. Remember to use ``pylint``. .. _Google Python Style Guide: - https://google-styleguide.googlecode.com/svn/trunk/pyguide.html + https://google.github.io/styleguide/pyguide.html .. _Sphinx-style: http://sphinx-doc.org/ .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 @@ -314,7 +316,9 @@ Steps: 3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. Fix any errors. 4. Run ``tox -e lint`` to check for pylint errors. Fix any errors. -5. Run ``tox`` to run the entire test suite including coverage. Fix any errors. +5. Run ``tox --skip-missing-interpreters`` to run the entire test suite + including coverage. The ``--skip-missing-interpreters`` argument ignores + missing versions of Python needed for running the tests. Fix any errors. 6. If your code touches communication with an ACME server/Boulder, you should run the integration tests, see `integration`_. See `Known Issues`_ for some common failures that have nothing to do with your code. @@ -323,7 +327,7 @@ Steps: See `Known Issues`_. If it's not a known issue, fix any errors. .. _Known Issues: - https://github.com/letsencrypt/letsencrypt/wiki/Known-issues + https://github.com/certbot/certbot/wiki/Known-issues Updating the documentation ========================== @@ -333,7 +337,7 @@ commands: .. code-block:: shell - make -C docs clean html + make -C docs clean html man This should generate documentation in the ``docs/_build/html`` directory. diff --git a/docs/index.rst b/docs/index.rst index 68289d760..b541e376e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -Welcome to the Let's Encrypt client documentation! +Welcome to the Certbot documentation! ================================================== .. toctree:: diff --git a/docs/intro.rst b/docs/intro.rst index 188ff4302..2fffbec68 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -1,6 +1,6 @@ -============ -Introduction -============ +===================== +README / Introduction +===================== .. include:: ../README.rst .. include:: ../CHANGES.rst diff --git a/docs/man/certbot.rst b/docs/man/certbot.rst index 7382d7811..d03f3eed4 100644 --- a/docs/man/certbot.rst +++ b/docs/man/certbot.rst @@ -1 +1 @@ -.. program-output:: certbot --help all +.. literalinclude:: ../cli-help.txt diff --git a/docs/using.rst b/docs/using.rst index 60c074d75..fb96bb853 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -5,66 +5,38 @@ User Guide .. contents:: Table of Contents :local: -.. _installation: +Getting Certbot +=============== -Installation -============ +To get specific instructions for installing Certbot on your OS, we recommend +visiting certbot.eff.org_. If you're offline, you can find some general +instructions `in the README / Introduction `__ -.. _letsencrypt-auto: +__ installation_ +.. _certbot.eff.org: https://certbot.eff.org -letsencrypt-auto ----------------- +.. _certbot-auto: -``letsencrypt-auto`` is a wrapper which installs some dependencies -from your OS standard package repositories (e.g. using `apt-get` or -`yum`), and for other dependencies it sets up a virtualized Python -environment with packages downloaded from PyPI [#venv]_. It also -provides automated updates. +The name of the certbot command +------------------------------- -To install and run the client, just type... - -.. code-block:: shell - - ./letsencrypt-auto - -.. hint:: During the beta phase, Let's Encrypt enforces strict rate limits on - the number of certificates issued for one domain. It is recommended to - initially use the test server via `--test-cert` until you get the desired - certificates. - -Throughout the documentation, whenever you see references to -``letsencrypt`` script/binary, you can substitute in -``letsencrypt-auto``. For example, to get basic help you would type: - -.. code-block:: shell - - ./letsencrypt-auto --help - -or for full help, type: - -.. code-block:: shell - - ./letsencrypt-auto --help all - - -``letsencrypt-auto`` is the recommended method of running the Let's Encrypt -client beta releases on systems that don't have a packaged version. Debian, -Arch Linux, Gentoo, FreeBSD, and OpenBSD now have native packages, so on those -systems you can just install ``letsencrypt`` (and perhaps -``letsencrypt-apache``). If you'd like to run the latest copy from Git, or -run your own locally modified copy of the client, follow the instructions in -the :doc:`contributing`. Some `other methods of installation`_ are discussed -below. +Many platforms now have native packages that give you a ``certbot`` or (for +older packages) ``letsencrypt`` command you can run. On others, the +``certbot-auto`` / ``letsencrypt-auto`` installer and wrapper script is a +stand-in. Throughout the documentation, whenever you see references to +``certbot`` script/binary, you should substitute in the name of the command +that certbot.eff.org_ told you to use on your system (``certbot``, +``letsencrypt``, or ``certbot-auto``). Plugins ======= -The Let's Encrypt client supports a number of different "plugins" that can be +The Certbot client supports a number of different "plugins" that can be used to obtain and/or install certificates. Plugins that can obtain a cert are called "authenticators" and can be used with the "certonly" command. Plugins that can install a cert are called "installers". Plugins that do both -can be used with the "letsencrypt run" command, which is the default. +can be used with the "certbot run" command, which is the default. =========== ==== ==== =============================================================== Plugin Auth Inst Notes @@ -79,27 +51,10 @@ standalone_ Y N Uses a "standalone" webserver to obtain a cert. Requires webserver is not supported or not desired. manual_ Y N Helps you obtain a cert by giving you instructions to perform domain validation yourself. -nginx_ Y Y Very experimental and not included in letsencrypt-auto_. +nginx_ Y Y Very experimental and not included in certbot-auto_. =========== ==== ==== =============================================================== -There are also a number of third-party plugins for the client, provided by other developers: - -=========== ==== ==== =============================================================== -Plugin Auth Inst Notes -=========== ==== ==== =============================================================== -plesk_ Y Y Integration with the Plesk web hosting tool -haproxy_ Y Y Integration with the HAProxy load balancer -s3front_ Y Y Integration with Amazon CloudFront distribution of S3 buckets -gandi_ Y Y Integration with Gandi's hosting products and API -=========== ==== ==== =============================================================== - -.. _plesk: https://github.com/plesk/letsencrypt-plesk -.. _haproxy: https://code.greenhost.net/open/letsencrypt-haproxy -.. _s3front: https://github.com/dlapiduz/letsencrypt-s3front -.. _gandi: https://github.com/Gandi/letsencrypt-gandi - -Future plugins for IMAP servers, SMTP servers, IRC servers, etc, are likely to -be installers but not authenticators. +There are many third-party-plugins_ available. Apache ------ @@ -130,16 +85,16 @@ specified ``--webroot-path``. So, for instance, :: - letsencrypt certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net + certbot certonly --webroot -w /var/www/example/ -d www.example.com -d example.com -w /var/www/other -d other.example.net -d another.other.example.net would obtain a single certificate for all of those names, using the ``/var/www/example`` webroot directory for the first two, and ``/var/www/other`` for the second two. The webroot plugin works by creating a temporary file for each of your requested -domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's -Encrypt validation server makes HTTP requests to validate that the DNS for each -requested domain resolves to the server running letsencrypt. An example request +domains in ``${webroot-path}/.well-known/acme-challenge``. Then the Let's Encrypt +validation server makes HTTP requests to validate that the DNS for each +requested domain resolves to the server running certbot. An example request made to your web server would look like: :: @@ -173,7 +128,7 @@ specified port using each requested domain name. Manual ------ -If you'd like to obtain a cert running ``letsencrypt`` on a machine +If you'd like to obtain a cert running ``certbot`` on a machine other than your target webserver or perform the steps for domain validation yourself, you can use the manual plugin. While hidden from the UI, you can use the plugin to obtain a cert by specifying @@ -186,16 +141,49 @@ Nginx In the future, if you're running Nginx you will hopefully be able to use this plugin to automatically obtain and install your certificate. The Nginx plugin is -still experimental, however, and is not installed with letsencrypt-auto_. If +still experimental, however, and is not installed with certbot-auto_. If installed, you can select this plugin on the command line by including ``--nginx``. +.. _third-party-plugins: + Third-party plugins ------------------- -These plugins are listed at -https://github.com/letsencrypt/letsencrypt/wiki/Plugins. If you're -interested, you can also :ref:`write your own plugin `. +There are also a number of third-party plugins for the client, provided by +other developers. Many are beta/experimental, but some are already in +widespread use: + +=========== ==== ==== =============================================================== +Plugin Auth Inst Notes +=========== ==== ==== =============================================================== +plesk_ Y Y Integration with the Plesk web hosting tool +haproxy_ Y Y Integration with the HAProxy load balancer +s3front_ Y Y Integration with Amazon CloudFront distribution of S3 buckets +gandi_ Y Y Integration with Gandi's hosting products and API +varnish_ Y N Obtain certs via a Varnish server +external_ Y N A plugin for convenient scripting (See also ticket 2782_) +icecast_ N Y Deploy certs to Icecast 2 streaming media servers +pritunl_ N Y Install certs in pritunl distributed OpenVPN servers +proxmox_ N Y Install certs in Proxmox Virtualization servers +postfix_ N Y STARTTLS Everywhere is becoming a Certbot Postfix/Exim plugin +=========== ==== ==== =============================================================== + +.. _plesk: https://github.com/plesk/letsencrypt-plesk +.. _haproxy: https://code.greenhost.net/open/letsencrypt-haproxy +.. _s3front: https://github.com/dlapiduz/letsencrypt-s3front +.. _gandi: https://github.com/Gandi/letsencrypt-gandi +.. _icecast: https://github.com/e00E/lets-encrypt-icecast +.. _varnish: http://git.sesse.net/?p=letsencrypt-varnish-plugin +.. _2782: https://github.com/certbot/certbot/issues/2782 +.. _pritunl: https://github.com/kharkevich/letsencrypt-pritunl +.. _proxmox: https://github.com/kharkevich/letsencrypt-proxmox +.. _external: https://github.com/marcan/letsencrypt-external +.. _postfix: https://github.com/EFForg/starttls-everywhere + +If you're interested, you can also :ref:`write your own plugin `. + + Renewal ======= @@ -204,23 +192,36 @@ Renewal days). Make sure you renew the certificates at least once in 3 months. -The ``letsencrypt`` client now supports a ``renew`` action to check +The ``certbot`` client now supports a ``renew`` action to check all installed certificates for impending expiry and attempt to renew them. The simplest form is simply -``letsencrypt renew`` +``certbot renew`` This will attempt to renew any previously-obtained certificates that expire in less than 30 days. The same plugin and options that were used at the time the certificate was originally issued will be used for the renewal attempt, unless you specify other plugins or options. +You can also specify hooks to be run before or after a certificate is +renewed. For example, if you want to use the standalone_ plugin to renew +your certificates, you may want to use a command like + +``certbot renew --standalone --pre-hook "service nginx stop" --post-hook "service nginx start"`` + +This will stop Nginx so standalone can bind to the necessary ports and +then restart Nginx after the plugin is finished. The hooks will only be +run if a certificate is due for renewal, so you can run this command +frequently without unnecessarily stopping your webserver. More +information about renewal hooks can be found by running +``certbot --help renew``. + If you're sure that this command executes successfully without human intervention, you can add the command to ``crontab`` (since certificates are only renewed when they're determined to be near expiry, the command -can run on a regular basis, like every week or every day); note that -the current version provides detailed output describing either renewal -success or failure. +can run on a regular basis, like every week or every day). In that case, +you are likely to want to use the ``-q`` or ``--quiet`` quiet flag to +silence all output except errors. The ``--force-renew`` flag may be helpful for automating renewal; it causes the expiration time of the certificate(s) to be ignored when @@ -229,9 +230,9 @@ certificate regardless of its age. (This form is not appropriate to run daily because each certificate will be renewed every day, which will quickly run into the certificate authority rate limit.) -Note that options provided to ``letsencrypt renew`` will apply to +Note that options provided to ``certbot renew`` will apply to *every* certificate for which renewal is attempted; for example, -``letsencrypt renew --rsa-key-size 4096`` would try to replace every +``certbot renew --rsa-key-size 4096`` would try to replace every near-expiry certificate with an equivalent certificate using a 4096-bit RSA public key. If a certificate is successfully renewed using specified options, those options will be saved and used for future @@ -240,10 +241,12 @@ renewals of that certificate. An alternative form that provides for more fine-grained control over the renewal process (while renewing specified certificates one at a time), -is ``letsencrypt certonly`` with the complete set of subject domains of -a specific certificate specified via `-d` flags, like +is ``certbot certonly`` with the complete set of subject domains of +a specific certificate specified via `-d` flags. You may also want to +include the ``-n`` or ``--noninteractive`` flag to prevent blocking on +user input (which is useful when running the command from cron). -``letsencrypt certonly -d example.com -d www.example.com`` +``certbot certonly -n -d example.com -d www.example.com`` (All of the domains covered by the certificate must be specified in this case in order to renew and replace the old certificate rather @@ -256,21 +259,25 @@ The ``certonly`` form attempts to renew one individual certificate. Please note that the CA will send notification emails to the address you provide if you do not renew certificates that are about to expire. -Let's Encrypt is working hard on improving the renewal process, and we +Certbot is working hard on improving the renewal process, and we apologize for any inconveniences you encounter in integrating these commands into your individual environment. +.. _command-line: + +Command line options +==================== + +Certbot supports a lot of command line options. Here's the full list, from +``certbot --help all``: + +.. literalinclude:: cli-help.txt .. _where-certs: Where are my certificates? ========================== -First of all, we encourage you to use Apache or nginx installers, both -which perform the certificate management automatically. If, however, -you prefer to manage everything by hand, this section provides -information on where to find necessary files. - All generated keys and issued certificates can be found in ``/etc/letsencrypt/live/$domain``. Rather than copying, please point your (web) server configuration directly to those files (or create @@ -287,7 +294,7 @@ The following files are available: Private key for the certificate. .. warning:: This **must be kept secret at all times**! Never share - it with anyone, including Let's Encrypt developers. You cannot + it with anyone, including Certbot developers. You cannot put it into a safe, however - your server still needs to access this file in order for SSL/TLS to work. @@ -330,8 +337,8 @@ will cause nasty errors served through the browsers! .. note:: All files are PEM-encoded (as the filename suffix suggests). If you need other format, such as DER or PFX, then you - could convert using ``openssl``, but this means you will not - benefit from automatic renewal_! + could convert using ``openssl``. You can automate that with + ``--renew-hook`` if you're using automatic renewal_. .. _config-file: @@ -340,7 +347,7 @@ Configuration file ================== It is possible to specify configuration file with -``letsencrypt-auto --config cli.ini`` (or shorter ``-c cli.ini``). An +``certbot-auto --config cli.ini`` (or shorter ``-c cli.ini``). An example configuration file is shown below: .. include:: ../examples/cli.ini @@ -359,13 +366,14 @@ By default, the following locations are searched: Getting help ============ -If you're having problems you can chat with us on `IRC (#letsencrypt @ -Freenode) `_ or -get support on our `forums `_. +If you're having problems you can chat with us on `IRC (#certbot @ +OFTC) `_ or at +`IRC (#letsencrypt @ freenode) `_ +or get support on the Let's Encrypt `forums `_. If you find a bug in the software, please do report it in our `issue tracker -`_. Remember to +`_. Remember to give us as much information as possible: - copy and paste exact command line used and the output (though mind @@ -373,9 +381,9 @@ give us as much information as possible: information, including your email and domains) - copy and paste logs from ``/var/log/letsencrypt`` (though mind they also might contain personally identifiable information) -- copy and paste ``letsencrypt --version`` output +- copy and paste ``certbot --version`` output - your operating system, including specific version -- specify which installation_ method you've chosen +- specify which installation method you've chosen Other methods of installation ============================= @@ -390,10 +398,10 @@ plugins cannot reach it from inside the Docker container. You should definitely read the :ref:`where-certs` section, in order to know how to manage the certs -manually. https://github.com/letsencrypt/letsencrypt/wiki/Ciphersuite-guidance +manually. https://github.com/certbot/certbot/wiki/Ciphersuite-guidance provides some information about recommended ciphersuites. If none of these make much sense to you, you should definitely use the -letsencrypt-auto_ method, which enables you to use installer plugins +certbot-auto_ method, which enables you to use installer plugins that cover both of those hard topics. If you're still not convinced and have decided to use this method, @@ -402,7 +410,7 @@ to, `install Docker`_, then issue the following command: .. code-block:: shell - sudo docker run -it --rm -p 443:443 -p 80:80 --name letsencrypt \ + sudo docker run -it --rm -p 443:443 -p 80:80 --name certbot \ -v "/etc/letsencrypt:/etc/letsencrypt" \ -v "/var/lib/letsencrypt:/var/lib/letsencrypt" \ quay.io/letsencrypt/letsencrypt:latest auth @@ -420,7 +428,7 @@ Operating System Packages **FreeBSD** - * Port: ``cd /usr/ports/security/py-letsencrypt && make install clean`` + * Port: ``cd /usr/ports/security/py-certbot && make install clean`` * Package: ``pkg install py27-letsencrypt`` **OpenBSD** @@ -432,19 +440,19 @@ Operating System Packages .. code-block:: shell - sudo pacman -S letsencrypt letsencrypt-apache + sudo pacman -S certbot **Debian** -If you run Debian Stretch or Debian Sid, you can install letsencrypt packages. +If you run Debian Stretch or Debian Sid, you can install certbot packages. .. code-block:: shell sudo apt-get update - sudo apt-get install letsencrypt python-letsencrypt-apache + sudo apt-get install certbot python-certbot-apache If you don't want to use the Apache plugin, you can omit the -``python-letsencrypt-apache`` package. +``python-certbot-apache`` package. Packages exist for Debian Jessie via backports. First you'll have to follow the instructions at http://backports.debian.org/Instructions/ to enable the Jessie backports @@ -452,7 +460,7 @@ repo, if you have not already done so. Then run: .. code-block:: shell - sudo apt-get install certbot python-certbot-apache -t jessie-backports + sudo apt-get install letsencrypt python-letsencrypt-apache -t jessie-backports **Fedora** @@ -462,7 +470,7 @@ repo, if you have not already done so. Then run: **Gentoo** -The official Let's Encrypt client is available in Gentoo Portage. If you +The official Certbot client is available in Gentoo Portage. If you want to use the Apache plugin, it has to be installed separately: .. code-block:: shell @@ -471,8 +479,12 @@ want to use the Apache plugin, it has to be installed separately: emerge -av app-crypt/letsencrypt-apache Currently, only the Apache plugin is included in Portage. However, if you -want the nginx plugin, you can use Layman to add the mrueg overlay which -does include the nginx plugin package: +Warning! +You can use Layman to add the mrueg overlay which does include a package for the +Certbot Nginx plugin, however, this plugin is known to be buggy and should only +be used with caution after creating a backup up your Nginx configuration. +We strongly recommend you use the app-crypt/letsencrypt package instead until +the Nginx plugin is ready. .. code-block:: shell @@ -509,7 +521,7 @@ Note: this change is not required for the other plugins. **Other Operating Systems** OS packaging is an ongoing effort. If you'd like to package -Let's Encrypt client for your distribution of choice please have a +Certbot for your distribution of choice please have a look at the :doc:`packaging`. @@ -525,25 +537,18 @@ whole process is described in the :doc:`contributing`. environment, e.g. ``sudo python setup.py install``, ``sudo pip install``, ``sudo ./venv/bin/...``. These modes of operation might corrupt your operating system and are **not supported** by the - Let's Encrypt team! + Certbot team! Comparison of different methods ------------------------------- -Unless you have a very specific requirements, we kindly ask you to use -the letsencrypt-auto_ method. It's the fastest, the most thoroughly +Unless you have a very specific requirements, we kindly suggest that you use +the certbot-auto_ method. It's the fastest, the most thoroughly tested and the most reliable way of getting our software and the free -SSL certificates! +TLS/SSL certificates! Beyond the methods discussed here, other methods may be possible, such as -installing Let's Encrypt directly with pip from PyPI or downloading a ZIP +installing Certbot directly with pip from PyPI or downloading a ZIP archive from GitHub may be technically possible but are not presently recommended or supported. - - -.. rubric:: Footnotes - -.. [#venv] By using this virtualized Python environment (`virtualenv - `_) we don't pollute the main - OS space with packages from PyPI! diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index a52044f87..09703841c 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -16,7 +16,7 @@ here = os.path.abspath(os.path.dirname(__file__)) readme = read_file(os.path.join(here, 'README.rst')) -version = '0.6.0.dev0' +version = '0.8.0.dev0' # This package is a simple shim around certbot-apache diff --git a/letsencrypt-auto b/letsencrypt-auto index 942fd8ea2..80f39cf59 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -1,6 +1,6 @@ #!/bin/sh # -# Download and run the latest release version of the Let's Encrypt client. +# Download and run the latest release version of the Certbot client. # # NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING # @@ -19,11 +19,25 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.5.0" +LE_AUTO_VERSION="0.8.1" +BASENAME=$(basename $0) +USAGE="Usage: $BASENAME [OPTIONS] +A self-updating wrapper script for the Certbot ACME client. When run, updates +to both this script and certbot will be downloaded and installed. After +ensuring you have the latest versions installed, certbot will be invoked with +all arguments you have provided. + +Help for certbot itself cannot be provided until it is installed. + + --debug attempt experimental installation + -h, --help print this help + -n, --non-interactive, --noninteractive run without asking for user input + --no-self-upgrade do not download updates + --os-packages-only install OS dependencies and exit + -v, --verbose provide more output + +All arguments are accepted and forwarded to the Certbot client when run." -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) for arg in "$@" ; do case "$arg" in --debug) @@ -34,25 +48,43 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --help) + HELP=1;; + --noninteractive|--non-interactive) + ASSUME_YES=1;; --verbose) VERBOSE=1;; - [!-]*|-*[!v]*|-) - # Anything that isn't -v, -vv, etc.: that is, anything that does not - # start with a -, contains anything that's not a v, or is just "-" - ;; - *) # -v+ remains. - VERBOSE=1;; + -[!-]*) + while getopts ":hnv" short_arg $arg; do + case "$short_arg" in + h) + HELP=1;; + n) + ASSUME_YES=1;; + v) + VERBOSE=1;; + esac + done;; esac done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation +if [ $BASENAME = "letsencrypt-auto" ]; then + # letsencrypt-auto does not respect --help or --yes for backwards compatibility + ASSUME_YES=1 + HELP=0 +fi + +# certbot-auto needs root access to bootstrap OS dependencies, and +# certbot itself needs root access for almost all modes of operation # The "normal" case is that sudo is used for the steps that need root, but # this script *can* be run as root (not recommended), or fall back to using # `su` +SUDO_ENV="" +export CERTBOT_AUTO="$0" if test "`id -u`" -ne "0" ; then if command -v sudo 1>/dev/null 2>&1; then SUDO=sudo + SUDO_ENV="CERTBOT_AUTO=$0" else echo \"sudo\" is not available, will use \"su\" for installation steps... # Because the parameters in `su -c` has to be a string, @@ -140,7 +172,7 @@ BootstrapDebCommon() { # distro version (#346) virtualenv= - if apt-cache show virtualenv > /dev/null 2>&1; then + if apt-cache show virtualenv > /dev/null 2>&1 && ! apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then virtualenv="virtualenv" fi @@ -151,30 +183,45 @@ BootstrapDebCommon() { augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + AddBackportRepo() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" + echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + add_backports=1 + else + read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response + case $response in + [yY][eE][sS]|[yY]|"") + add_backports=1;; + *) + add_backports=0;; + esac + fi + if [ "$add_backports" = 1 ]; then + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update fi - - $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - $SUDO apt-get update fi fi - $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - + if [ "$add_backports" != 0 ]; then + $SUDO apt-get install $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + fi } @@ -186,12 +233,12 @@ BootstrapDebCommon() { AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" - echo "Let's Encrypt apache plugin..." + echo "Certbot apache plugin..." fi # XXX add a case for ubuntu PPAs fi - $SUDO apt-get install -y --no-install-recommends \ + $SUDO apt-get install $YES_FLAG --no-install-recommends \ python \ python-dev \ $virtualenv \ @@ -212,9 +259,10 @@ BootstrapDebCommon() { BootstrapRpmCommon() { # Tested with: - # - Fedora 22, 23 (x64) + # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) if type dnf 2>/dev/null then @@ -228,54 +276,62 @@ BootstrapRpmCommon() { exit 1 fi + pkgs=" + gcc + dialog + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + # Some distros and older versions of current distros use a "python27" # instead of "python" naming convention. Try both conventions. - if ! $SUDO $tool install -y \ - python \ - python-devel \ - python-virtualenv \ - python-tools \ - python-pip - then - if ! $SUDO $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv \ - python27-tools \ - python27-pip - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi + if $SUDO $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python + python-devel + python-virtualenv + python-tools + python-pip + " + else + pkgs="$pkgs + python27 + python27-devel + python27-virtualenv + python27-tools + python27-pip + " fi - if ! $SUDO $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates - then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi - - if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then - if ! $SUDO $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi + pkgs="$pkgs + mod_ssl + " + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + + if ! $SUDO $tool install $yes_flag $pkgs; then + echo "Could not install OS dependencies. Aborting bootstrap!" + exit 1 fi } BootstrapSuseCommon() { # SLE12 don't have python-virtualenv - $SUDO zypper -nq in -l \ + if [ "$ASSUME_YES" = 1 ]; then + zypper_flags="-nq" + install_flags="-l" + fi + + $SUDO zypper $zypper_flags in $install_flags \ python \ python-devel \ python-virtualenv \ @@ -310,8 +366,12 @@ BootstrapArchCommon() { # pacman -T exits with 127 if there are missing dependencies missing=$($SUDO pacman -T $deps) || true + if [ "$ASSUME_YES" = 1 ]; then + noconfirm="--noconfirm" + fi + if [ "$missing" ]; then - $SUDO pacman -S --needed $missing + $SUDO pacman -S --needed $missing $noconfirm fi } @@ -365,7 +425,8 @@ BootstrapMac() { $pkgcmd augeas $pkgcmd dialog - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" ]; then + if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ + -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. echo "Installing python..." @@ -375,7 +436,8 @@ BootstrapMac() { # Workaround for _dlopen not finding augeas on OS X if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then echo "Applying augeas workaround" - $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib + $SUDO mkdir -p /usr/local/lib/ + $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then @@ -391,12 +453,44 @@ BootstrapMac() { fi } +BootstrapSmartOS() { + pkgin update + pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' +} + +BootstrapMageiaCommon() { + if ! $SUDO urpmi --force \ + python \ + libpython-devel \ + python-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + + if ! $SUDO urpmi --force \ + git \ + gcc \ + cdialog \ + python-augeas \ + libopenssl-devel \ + libffi-devel \ + rootcerts + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} + # Install required OS packages: Bootstrap() { if [ -f /etc/debian_version ]; then echo "Bootstrapping dependencies for Debian-based OSes..." BootstrapDebCommon + elif [ -f /etc/mageia-release ] ; then + # Mageia has both /etc/mageia-release and /etc/redhat-release + ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon @@ -409,7 +503,7 @@ Bootstrap() { BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" + echo "# pacman -S certbot certbot-apache" echo echo "If you would like to use the virtualenv way, please run the script again with the" echo "--debug flag." @@ -423,14 +517,17 @@ Bootstrap() { ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd elif uname | grep -iq Darwin ; then ExperimentalBootstrap "Mac OS X" BootstrapMac - elif grep -iq "Amazon Linux" /etc/issue ; then + elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then + ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo echo "You will need to bootstrap, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." + exit 1 fi } @@ -446,7 +543,8 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg if [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep ^letsencrypt | cut -d " " -f 2) + # grep for both certbot and letsencrypt until certbot and shim packages have been released + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) else INSTALLED_VERSION="none" fi @@ -462,11 +560,12 @@ if [ "$1" = "--le-auto-phase2" ]; then echo "Installing Python packages..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -# This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# This is the flattened list of packages certbot-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e certbot-apache`, and # then use `hashin` or a more secure method to gather the hashes. argparse==1.4.0 \ @@ -642,18 +741,21 @@ zope.interface==4.1.3 \ mock==1.0.1 \ --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.5.0 \ - --hash=sha256:ceb4127c13213f0006a564be82176b968c6b374d20d9fc78555d0658a252b275 \ - --hash=sha256:0605c63c656d33c883a05675f5db9cfb85d503f2771c885031800e0da7631abd -letsencrypt==0.5.0 \ - --hash=sha256:f90f883e99cdbdf8142335bdbf4f74a8af143ee4b4ec60fb49c6e47418c1114c \ - --hash=sha256:e38a2b70b82be79bc195307652244a3e012ec73d897d4dbd3f80cf698496d15a -letsencrypt-apache==0.5.0 \ - --hash=sha256:a767882164a7b09d9c12c80684a28a782135fdaf35654ef5a02c0b7b1d27ab8d \ - --hash=sha256:c20e7b9c517aa4a7d70e6bd9382da7259f00bc191b9e60d8e312e48837a00c41 +acme==0.8.1 \ + --hash=sha256:ccd7883772efbf933f91713b8241455993834f3620c8fbd459d9ed5e50bbaaca \ + --hash=sha256:d3ea4acf280bf6253ad7d641cb0970f230a19805acfed809e7a8ddcf62157d9f +certbot==0.8.1 \ + --hash=sha256:89805d9f70249ae859ec4d7a99c00b4bb7083ca90cd12d4d202b76dfc284f7c5 \ + --hash=sha256:6ca8df3d310ced6687d38aac17c0fb8c1b2ec7a3bea156a254e4cc2a1c132771 +certbot-apache==0.8.1 \ + --hash=sha256:c9e3fdc15e65589c2e39eb0e6b1f61f0c0a1db3c17b00bb337f0ff636cc61cb3 \ + --hash=sha256:0faf2879884d3b7a58b071902fba37d4b8b58a50e2c3b8ac262c0a74134045ed UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -813,7 +915,6 @@ UNLIKELY_EOF PIP_OUT=`"$VENV_BIN/pip" install --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` PIP_STATUS=$? set -e - rm -rf "$TEMP_DIR" if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while installing Python packages:" @@ -823,18 +924,32 @@ UNLIKELY_EOF fi echo "Installation succeeded." fi - echo "Requesting root privileges to run letsencrypt..." - echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" - $SUDO "$VENV_BIN/letsencrypt" "$@" + if [ -n "$SUDO" ]; then + # SUDO is su wrapper or sudo + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" + fi + if [ -z "$SUDO_ENV" ] ; then + # SUDO is su wrapper / noop + $SUDO "$VENV_BIN/letsencrypt" "$@" + else + # sudo + $SUDO "$SUDO_ENV" "$VENV_BIN/letsencrypt" "$@" + fi + else - # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # Phase 1: Upgrade certbot-auto if neceesary, then self-invoke. # # Each phase checks the version of only the thing it is responsible for # upgrading. Phase 1 checks the version of the latest release of - # letsencrypt-auto (which is always the same as that of the letsencrypt - # package). Phase 2 checks the version of the locally installed letsencrypt. + # certbot-auto (which is always the same as that of the certbot + # package). Phase 2 checks the version of the locally installed certbot. if [ ! -f "$VENV_BIN/letsencrypt" ]; then + if [ "$HELP" = 1 ]; then + echo "$USAGE" + exit 0 + fi # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi @@ -844,8 +959,8 @@ else fi if [ "$NO_SELF_UPGRADE" != 1 ]; then - echo "Checking for new version..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" """Do downloading and JSON parsing without additional dependencies. :: @@ -918,7 +1033,7 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/letsencrypt/json'))) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most @@ -937,7 +1052,7 @@ def verified_new_le_auto(get, tag, temp_dir): """ le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'https://raw.githubusercontent.com/certbot/certbot/%s/' 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') @@ -953,7 +1068,7 @@ def verified_new_le_auto(get, tag, temp_dir): stderr=dev_null) except CalledProcessError as exc: raise ExpectedError("Couldn't verify signature of downloaded " - "letsencrypt-auto.", exc) + "certbot-auto.", exc) def main(): @@ -978,32 +1093,28 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion - REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` - if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + echo "WARNING: unable to check for updates." + elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - # Install new copy of letsencrypt-auto. + # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing letsencrypt-auto..." + echo "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: - echo " " $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" # Using mv rather than cp leaves the old file descriptor pointing to the # original copy so the shell can continue to read it unmolested. mv across # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the # cp is unlikely to fail (esp. under sudo) if the rm doesn't. - echo " " $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - # TODO: Clean up temp dir safely, even if it has quotes in its path. - rm -rf "$TEMP_DIR" fi # A newer version is available. fi # Self-upgrading is allowed. diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc new file mode 100644 index 000000000..5bb725c96 --- /dev/null +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1 + +iQEcBAABAgAGBQJXYJmBAAoJEE0XyZXNl3XyyIMH/jtYFb7rl5XXN8hjlKuK5frq +z7/jdK7fvI+mtYJ4i2Cy3yMz8T4wscXGkhxNtipbATWlpevPfjYzm4ZGC25coFZx +fDX44w0hBBgel7EISXGR1ABXb2rj24TZxIYXwaeClylsK9n5CxcWBocn8tDlfr8t +7VQUJEL3l1IlrnKnvpoL4Eq11sxlIPtitDPJ5c98ZM1293ZbWzIqyZKoXLIUkKHg +pkaa80j/QMmFumxzXFenU91JusLdeoblvjjg+kzjGonjslAYIuH4wEEjz2VJuUYe +P2+2ZyW4eLA6rRZhZ3CMtV79HzTPTWiELCYbXezb+yXJJEqzCYtIXkmbNQ3jUEY= +=86lB +-----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 111f2b272..22c914ed8 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -1,6 +1,6 @@ #!/bin/sh # -# Download and run the latest release version of the Let's Encrypt client. +# Download and run the latest release version of the Certbot client. # # NOTE: THIS SCRIPT IS AUTO-GENERATED AND SELF-UPDATING # @@ -19,11 +19,26 @@ XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share} VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.6.0.dev0" +LE_AUTO_VERSION="0.9.0.dev0" +BASENAME=$(basename $0) +USAGE="Usage: $BASENAME [OPTIONS] +A self-updating wrapper script for the Certbot ACME client. When run, updates +to both this script and certbot will be downloaded and installed. After +ensuring you have the latest versions installed, certbot will be invoked with +all arguments you have provided. + +Help for certbot itself cannot be provided until it is installed. + + --debug attempt experimental installation + -h, --help print this help + -n, --non-interactive, --noninteractive run without asking for user input + --no-self-upgrade do not download updates + --os-packages-only install OS dependencies and exit + -q, --quiet provide only update/error output + -v, --verbose provide more output + +All arguments are accepted and forwarded to the Certbot client when run." -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) for arg in "$@" ; do case "$arg" in --debug) @@ -34,25 +49,47 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --help) + HELP=1;; + --noninteractive|--non-interactive) + ASSUME_YES=1;; + --quiet) + QUIET=1;; --verbose) VERBOSE=1;; - [!-]*|-*[!v]*|-) - # Anything that isn't -v, -vv, etc.: that is, anything that does not - # start with a -, contains anything that's not a v, or is just "-" - ;; - *) # -v+ remains. - VERBOSE=1;; + -[!-]*) + while getopts ":hnvq" short_arg $arg; do + case "$short_arg" in + h) + HELP=1;; + n) + ASSUME_YES=1;; + q) + QUIET=1;; + v) + VERBOSE=1;; + esac + done;; esac done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and -# letsencrypt itself needs root access for almost all modes of operation +if [ $BASENAME = "letsencrypt-auto" ]; then + # letsencrypt-auto does not respect --help or --yes for backwards compatibility + ASSUME_YES=1 + HELP=0 +fi + +# certbot-auto needs root access to bootstrap OS dependencies, and +# certbot itself needs root access for almost all modes of operation # The "normal" case is that sudo is used for the steps that need root, but # this script *can* be run as root (not recommended), or fall back to using # `su` +SUDO_ENV="" +export CERTBOT_AUTO="$0" if test "`id -u`" -ne "0" ; then if command -v sudo 1>/dev/null 2>&1; then SUDO=sudo + SUDO_ENV="CERTBOT_AUTO=$0" else echo \"sudo\" is not available, will use \"su\" for installation steps... # Because the parameters in `su -c` has to be a string, @@ -140,7 +177,7 @@ BootstrapDebCommon() { # distro version (#346) virtualenv= - if apt-cache show virtualenv > /dev/null 2>&1; then + if apt-cache show virtualenv > /dev/null 2>&1 && ! apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then virtualenv="virtualenv" fi @@ -151,30 +188,45 @@ BootstrapDebCommon() { augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + AddBackportRepo() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" + echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + add_backports=1 + else + read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response + case $response in + [yY][eE][sS]|[yY]|"") + add_backports=1;; + *) + add_backports=0;; + esac + fi + if [ "$add_backports" = 1 ]; then + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update fi - - $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - $SUDO apt-get update fi fi - $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - + if [ "$add_backports" != 0 ]; then + $SUDO apt-get install $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + fi } @@ -186,12 +238,12 @@ BootstrapDebCommon() { AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" else echo "No libaugeas0 version is available that's new enough to run the" - echo "Let's Encrypt apache plugin..." + echo "Certbot apache plugin..." fi # XXX add a case for ubuntu PPAs fi - $SUDO apt-get install -y --no-install-recommends \ + $SUDO apt-get install $YES_FLAG --no-install-recommends \ python \ python-dev \ $virtualenv \ @@ -212,9 +264,10 @@ BootstrapDebCommon() { BootstrapRpmCommon() { # Tested with: - # - Fedora 22, 23 (x64) + # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) if type dnf 2>/dev/null then @@ -228,54 +281,62 @@ BootstrapRpmCommon() { exit 1 fi + pkgs=" + gcc + dialog + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + # Some distros and older versions of current distros use a "python27" # instead of "python" naming convention. Try both conventions. - if ! $SUDO $tool install -y \ - python \ - python-devel \ - python-virtualenv \ - python-tools \ - python-pip - then - if ! $SUDO $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv \ - python27-tools \ - python27-pip - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi + if $SUDO $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python + python-devel + python-virtualenv + python-tools + python-pip + " + else + pkgs="$pkgs + python27 + python27-devel + python27-virtualenv + python27-tools + python27-pip + " fi - if ! $SUDO $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates - then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi - - if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then - if ! $SUDO $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi + pkgs="$pkgs + mod_ssl + " + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + + if ! $SUDO $tool install $yes_flag $pkgs; then + echo "Could not install OS dependencies. Aborting bootstrap!" + exit 1 fi } BootstrapSuseCommon() { # SLE12 don't have python-virtualenv - $SUDO zypper -nq in -l \ + if [ "$ASSUME_YES" = 1 ]; then + zypper_flags="-nq" + install_flags="-l" + fi + + $SUDO zypper $zypper_flags in $install_flags \ python \ python-devel \ python-virtualenv \ @@ -310,8 +371,12 @@ BootstrapArchCommon() { # pacman -T exits with 127 if there are missing dependencies missing=$($SUDO pacman -T $deps) || true + if [ "$ASSUME_YES" = 1 ]; then + noconfirm="--noconfirm" + fi + if [ "$missing" ]; then - $SUDO pacman -S --needed $missing + $SUDO pacman -S --needed $missing $noconfirm fi } @@ -365,7 +430,8 @@ BootstrapMac() { $pkgcmd augeas $pkgcmd dialog - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" ]; then + if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ + -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. echo "Installing python..." @@ -375,7 +441,8 @@ BootstrapMac() { # Workaround for _dlopen not finding augeas on OS X if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then echo "Applying augeas workaround" - $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib + $SUDO mkdir -p /usr/local/lib/ + $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then @@ -391,12 +458,44 @@ BootstrapMac() { fi } +BootstrapSmartOS() { + pkgin update + pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' +} + +BootstrapMageiaCommon() { + if ! $SUDO urpmi --force \ + python \ + libpython-devel \ + python-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + + if ! $SUDO urpmi --force \ + git \ + gcc \ + cdialog \ + python-augeas \ + libopenssl-devel \ + libffi-devel \ + rootcerts + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} + # Install required OS packages: Bootstrap() { if [ -f /etc/debian_version ]; then echo "Bootstrapping dependencies for Debian-based OSes..." BootstrapDebCommon + elif [ -f /etc/mageia-release ] ; then + # Mageia has both /etc/mageia-release and /etc/redhat-release + ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon @@ -409,7 +508,7 @@ Bootstrap() { BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" + echo "# pacman -S certbot certbot-apache" echo echo "If you would like to use the virtualenv way, please run the script again with the" echo "--debug flag." @@ -423,14 +522,17 @@ Bootstrap() { ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd elif uname | grep -iq Darwin ; then ExperimentalBootstrap "Mac OS X" BootstrapMac - elif grep -iq "Amazon Linux" /etc/issue ; then + elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then + ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!" + echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo echo "You will need to bootstrap, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." + exit 1 fi } @@ -446,7 +548,8 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg if [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep ^letsencrypt | cut -d " " -f 2) + # grep for both certbot and letsencrypt until certbot and shim packages have been released + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) else INSTALLED_VERSION="none" fi @@ -462,11 +565,12 @@ if [ "$1" = "--le-auto-phase2" ]; then echo "Installing Python packages..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -# This is the flattened list of packages letsencrypt-auto installs. To generate -# this, do `pip install --no-cache-dir -e acme -e . -e letsencrypt-apache`, and +# This is the flattened list of packages certbot-auto installs. To generate +# this, do `pip install --no-cache-dir -e acme -e . -e certbot-apache`, and # then use `hashin` or a more secure method to gather the hashes. argparse==1.4.0 \ @@ -499,28 +603,29 @@ ConfigArgParse==0.10.0 \ --hash=sha256:3b50a83dd58149dfcee98cb6565265d10b53e9c0a2bca7eeef7fb5f5524890a7 configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==1.2.3 \ - --hash=sha256:031938f73a5c5eb3e809e18ff7caeb6865351871417be6050cb8c86a9a202b9a \ - --hash=sha256:a179a38d50f8d68b491d7a313db78f8cabe290842cecddddc7b34d408e59db0a \ - --hash=sha256:906c88b2aadcf99cfabb24098263d1bf65ab0c8688acde10dae1f09d865920f1 \ - --hash=sha256:6e706c5c6088770b1d1b634e959e21963e315b0255f5f4777125ad3d54082977 \ - --hash=sha256:f5ebf8e31c48f8707921dca0e994de77813a9c9b9bf03c119c5ddf97bdcffe73 \ - --hash=sha256:c7b89e42288cc7fbee3812e99ef5c744f22452e11d6822f6807afc6d6b3be83e \ - --hash=sha256:8408d29865947109d8b68f1837a7cde1aa4dc86e0f79ca3ba58c0c44e443d6a5 \ - --hash=sha256:c7e76cf3c3d925dd31fa238cfb806cffba718c0f08707d77a538768477969956 \ - --hash=sha256:7d8de35380f31702758b7753bb5c40723832c73006dedb2f9099bf61a37f7287 \ - --hash=sha256:5edbee71fae5469ee83fe0a37866b9398c8ce3a46325c24fcedfbf097bb48a19 \ - --hash=sha256:594edafe4801c13bdc1cc305e7704a90c19617e95936f6ab457ee4ffe000ba50 \ - --hash=sha256:b7fdb16a0a7f481be42da744bfe1ea2163025de21f90f2c688a316f3c354da9c \ - --hash=sha256:207b8bf0fe0907336df38b733b487521cf9e138189aba9234ad54fe545dd0db8 \ - --hash=sha256:509a2f05386270cf783993c90d49ffefb3dd62aee45bf1ea8ce3d2cde7271c21 \ - --hash=sha256:ac69b65dd1af0179ede40c9f15788c88f73e628ea6c0519de3838e279bb388c6 \ - --hash=sha256:8df6fad6c6ae12fd7004ea29357f0a2b4d3774eaeca7656530d08d2d90cd41aa \ - --hash=sha256:0b8b96dd81cc1533a04f30382c0fe21c1972e189f794d0c4261a18cec08fd9b5 \ - --hash=sha256:cae8fca1883f23c50ea78d89de6fe4fefdb4cea83177760f47177559414ded93 \ - --hash=sha256:1a471ca576a9cdce1b1cd9f3a22b1d09ee44d46862037557de17919c0db44425 \ - --hash=sha256:8ec4e8e3d453b3a1b63b5f57737a434dcf1ee4a2f26f6ff7c5a37c3f679104d2 \ - --hash=sha256:8eb11c77dd8e73f48df6b2f7a7e16173fe0fe8fdfe266232832e88477e08454e +cryptography==1.3.4 \ + --hash=sha256:bede00edd11a2a62c8c98c271cc103fa3a3d72acf64f6e5e4eaf251128897b17 \ + --hash=sha256:53b39e687b744bb548a98f40736cc529d9f60959b4e6cc551322cf9505d35eb3 \ + --hash=sha256:474b73ad1139b4e423e46bbd818efd0d5c0df1c65d9f7c957d64c9215d77afde \ + --hash=sha256:aaddf9592d5b99e32dd518bb4a25b147c124f9d6b4ad64b94f01b15d1666b8c8 \ + --hash=sha256:6dcad2f407db8c3cd6ecd78361439c449a4f94786b46c54507e7e68f51e1709d \ + --hash=sha256:475c153fc622e656f1f10a9c9941d0ac7ab18df7c38d35d563a437c1c0e34f24 \ + --hash=sha256:86dd61df581cba04e89e45081efbc531faff1c9d99c77b1ce97f87216c356353 \ + --hash=sha256:75cc697e4ef5fdd0102ca749114c6370dbd11db0c9132a18834858c2566247e3 \ + --hash=sha256:ea03ad5b9df6d79fc9fc1ab23729e01e1c920d2974c5e3c634ccf45a5c378452 \ + --hash=sha256:c8872b8fe4f3416d6338ab99612f49ab314f7856cb43bffab2a32d28a6267be8 \ + --hash=sha256:468fc6e16eaec6ceaa6bc341273e6e9912d01b42b740f8cf896ace7fcd6a321d \ + --hash=sha256:d6fea3c6502735011c5d61a62aef1c1d770fc6a2def45d9e6c0d94c9651e3317 \ + --hash=sha256:3cf95f179f4bead3d5649b91860ef4cf60ad4244209190fc405908272576d961 \ + --hash=sha256:141f77e60a5b9158309b2b60288c7f81d37faa15c22a69b94c190ceefaaa6236 \ + --hash=sha256:87b7a1fe703c6424451f3372d1879dae91c7fe5e13375441a72833db76fee30e \ + --hash=sha256:f5ee3cb0cf1a6550bf483ccffa6608db267a377b45f7e3a8201a86d1d8feb19f \ + --hash=sha256:4e097286651ea318300af3251375d48b71b8228481c56cd617ddd4459a1ff261 \ + --hash=sha256:1e3d3ae3f22f22d50d340f47f25227511326f3f1396c6d2446a5b45b516c4313 \ + --hash=sha256:6a057941cb64d79834ea3cf99093fcc4787c2a5d44f686c4f297361ddc419bcd \ + --hash=sha256:68b3d5390b92559ddd3353c73ab2dfcff758f9c4ec4f5d5226ccede0e5d779f4 \ + --hash=sha256:545dc003b4b6081f9c3e452da15d819b04b696f49484aff64c0a2aedf766bef8 \ + --hash=sha256:423ff890c01be7c70dbfeaa967eeef5146f1a43a5f810ffdc07b178e48a105a9 enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -580,9 +685,9 @@ pyasn1==0.1.9 \ --hash=sha256:5191ff6b9126d2c039dd87f8ff025bed274baf07fa78afa46f556b1ad7265d6e \ --hash=sha256:8323e03637b2d072cc7041300bac6ec448c3c28950ab40376036788e9a1af629 \ --hash=sha256:853cacd96d1f701ddd67aa03ecc05f51890135b7262e922710112f12a2ed2a7f -pyOpenSSL==0.15.1 \ - --hash=sha256:88e45e6bb25dfed272a1ef2e728461d44b634c2cd689e989b6e56a349c5a3ae5 \ - --hash=sha256:f0a26070d6db0881de8bcc7846934b7c3c930d8f9c79d45883ee48984bc0d672 +pyopenssl==16.0.0 \ + --hash=sha256:5add70cf00273bf957ca31fdb0df9b0ae4639e081897d5f86a0ae1f104901230 \ + --hash=sha256:363d10ee43d062285facf4e465f4f5163f9f702f9134f0a5896f134cbb92d17d pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 @@ -642,18 +747,21 @@ zope.interface==4.1.3 \ mock==1.0.1 \ --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.5.0 \ - --hash=sha256:ceb4127c13213f0006a564be82176b968c6b374d20d9fc78555d0658a252b275 \ - --hash=sha256:0605c63c656d33c883a05675f5db9cfb85d503f2771c885031800e0da7631abd -letsencrypt==0.5.0 \ - --hash=sha256:f90f883e99cdbdf8142335bdbf4f74a8af143ee4b4ec60fb49c6e47418c1114c \ - --hash=sha256:e38a2b70b82be79bc195307652244a3e012ec73d897d4dbd3f80cf698496d15a -letsencrypt-apache==0.5.0 \ - --hash=sha256:a767882164a7b09d9c12c80684a28a782135fdaf35654ef5a02c0b7b1d27ab8d \ - --hash=sha256:c20e7b9c517aa4a7d70e6bd9382da7259f00bc191b9e60d8e312e48837a00c41 +acme==0.8.1 \ + --hash=sha256:ccd7883772efbf933f91713b8241455993834f3620c8fbd459d9ed5e50bbaaca \ + --hash=sha256:d3ea4acf280bf6253ad7d641cb0970f230a19805acfed809e7a8ddcf62157d9f +certbot==0.8.1 \ + --hash=sha256:89805d9f70249ae859ec4d7a99c00b4bb7083ca90cd12d4d202b76dfc284f7c5 \ + --hash=sha256:6ca8df3d310ced6687d38aac17c0fb8c1b2ec7a3bea156a254e4cc2a1c132771 +certbot-apache==0.8.1 \ + --hash=sha256:c9e3fdc15e65589c2e39eb0e6b1f61f0c0a1db3c17b00bb337f0ff636cc61cb3 \ + --hash=sha256:0faf2879884d3b7a58b071902fba37d4b8b58a50e2c3b8ac262c0a74134045ed UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -813,7 +921,6 @@ UNLIKELY_EOF PIP_OUT=`"$VENV_BIN/pip" install --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` PIP_STATUS=$? set -e - rm -rf "$TEMP_DIR" if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while installing Python packages:" @@ -823,18 +930,34 @@ UNLIKELY_EOF fi echo "Installation succeeded." fi - echo "Requesting root privileges to run letsencrypt..." - echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" - $SUDO "$VENV_BIN/letsencrypt" "$@" + if [ -n "$SUDO" ]; then + # SUDO is su wrapper or sudo + if [ "$QUIET" != 1 ]; then + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" + fi + fi + if [ -z "$SUDO_ENV" ] ; then + # SUDO is su wrapper / noop + $SUDO "$VENV_BIN/letsencrypt" "$@" + else + # sudo + $SUDO "$SUDO_ENV" "$VENV_BIN/letsencrypt" "$@" + fi + else - # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # Phase 1: Upgrade certbot-auto if neceesary, then self-invoke. # # Each phase checks the version of only the thing it is responsible for # upgrading. Phase 1 checks the version of the latest release of - # letsencrypt-auto (which is always the same as that of the letsencrypt - # package). Phase 2 checks the version of the locally installed letsencrypt. + # certbot-auto (which is always the same as that of the certbot + # package). Phase 2 checks the version of the locally installed certbot. if [ ! -f "$VENV_BIN/letsencrypt" ]; then + if [ "$HELP" = 1 ]; then + echo "$USAGE" + exit 0 + fi # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi @@ -844,8 +967,8 @@ else fi if [ "$NO_SELF_UPGRADE" != 1 ]; then - echo "Checking for new version..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" """Do downloading and JSON parsing without additional dependencies. :: @@ -918,7 +1041,7 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/letsencrypt/json'))) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most @@ -937,7 +1060,7 @@ def verified_new_le_auto(get, tag, temp_dir): """ le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'https://raw.githubusercontent.com/certbot/certbot/%s/' 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') @@ -953,7 +1076,7 @@ def verified_new_le_auto(get, tag, temp_dir): stderr=dev_null) except CalledProcessError as exc: raise ExpectedError("Couldn't verify signature of downloaded " - "letsencrypt-auto.", exc) + "certbot-auto.", exc) def main(): @@ -978,32 +1101,28 @@ if __name__ == '__main__': UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion - REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` - if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + echo "WARNING: unable to check for updates." + elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - # Install new copy of letsencrypt-auto. + # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing letsencrypt-auto..." + echo "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: - echo " " $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" # Using mv rather than cp leaves the old file descriptor pointing to the # original copy so the shell can continue to read it unmolested. mv across # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the # cp is unlikely to fail (esp. under sudo) if the rm doesn't. - echo " " $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - # TODO: Clean up temp dir safely, even if it has quotes in its path. - rm -rf "$TEMP_DIR" fi # A newer version is available. fi # Self-upgrading is allowed. diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 36ab206aa..e3da0e0a3 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index 2c8e1ec4c..444ee7900 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -20,10 +20,25 @@ VENV_NAME="letsencrypt" VENV_PATH=${VENV_PATH:-"$XDG_DATA_HOME/$VENV_NAME"} VENV_BIN="$VENV_PATH/bin" LE_AUTO_VERSION="{{ LE_AUTO_VERSION }}" +BASENAME=$(basename $0) +USAGE="Usage: $BASENAME [OPTIONS] +A self-updating wrapper script for the Certbot ACME client. When run, updates +to both this script and certbot will be downloaded and installed. After +ensuring you have the latest versions installed, certbot will be invoked with +all arguments you have provided. + +Help for certbot itself cannot be provided until it is installed. + + --debug attempt experimental installation + -h, --help print this help + -n, --non-interactive, --noninteractive run without asking for user input + --no-self-upgrade do not download updates + --os-packages-only install OS dependencies and exit + -q, --quiet provide only update/error output + -v, --verbose provide more output + +All arguments are accepted and forwarded to the Certbot client when run." -# This script takes the same arguments as the main letsencrypt program, but it -# additionally responds to --verbose (more output) and --debug (allow support -# for experimental platforms) for arg in "$@" ; do case "$arg" in --debug) @@ -34,25 +49,47 @@ for arg in "$@" ; do # Do not upgrade this script (also prevents client upgrades, because each # copy of the script pins a hash of the python client) NO_SELF_UPGRADE=1;; + --help) + HELP=1;; + --noninteractive|--non-interactive) + ASSUME_YES=1;; + --quiet) + QUIET=1;; --verbose) VERBOSE=1;; - [!-]*|-*[!v]*|-) - # Anything that isn't -v, -vv, etc.: that is, anything that does not - # start with a -, contains anything that's not a v, or is just "-" - ;; - *) # -v+ remains. - VERBOSE=1;; + -[!-]*) + while getopts ":hnvq" short_arg $arg; do + case "$short_arg" in + h) + HELP=1;; + n) + ASSUME_YES=1;; + q) + QUIET=1;; + v) + VERBOSE=1;; + esac + done;; esac done -# letsencrypt-auto needs root access to bootstrap OS dependencies, and +if [ $BASENAME = "letsencrypt-auto" ]; then + # letsencrypt-auto does not respect --help or --yes for backwards compatibility + ASSUME_YES=1 + HELP=0 +fi + +# certbot-auto needs root access to bootstrap OS dependencies, and # certbot itself needs root access for almost all modes of operation # The "normal" case is that sudo is used for the steps that need root, but # this script *can* be run as root (not recommended), or fall back to using # `su` +SUDO_ENV="" +export CERTBOT_AUTO="$0" if test "`id -u`" -ne "0" ; then if command -v sudo 1>/dev/null 2>&1; then SUDO=sudo + SUDO_ENV="CERTBOT_AUTO=$0" else echo \"sudo\" is not available, will use \"su\" for installation steps... # Because the parameters in `su -c` has to be a string, @@ -122,12 +159,17 @@ DeterminePythonVersion() { {{ bootstrappers/gentoo_common.sh }} {{ bootstrappers/free_bsd.sh }} {{ bootstrappers/mac.sh }} +{{ bootstrappers/smartos.sh }} +{{ bootstrappers/mageia_common.sh }} # Install required OS packages: Bootstrap() { if [ -f /etc/debian_version ]; then echo "Bootstrapping dependencies for Debian-based OSes..." BootstrapDebCommon + elif [ -f /etc/mageia-release ] ; then + # Mageia has both /etc/mageia-release and /etc/redhat-release + ExperimentalBootstrap "Mageia" BootstrapMageiaCommon elif [ -f /etc/redhat-release ]; then echo "Bootstrapping dependencies for RedHat-based OSes..." BootstrapRpmCommon @@ -140,7 +182,7 @@ Bootstrap() { BootstrapArchCommon else echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S letsencrypt letsencrypt-apache" + echo "# pacman -S certbot certbot-apache" echo echo "If you would like to use the virtualenv way, please run the script again with the" echo "--debug flag." @@ -154,14 +196,17 @@ Bootstrap() { ExperimentalBootstrap "FreeBSD" BootstrapFreeBsd elif uname | grep -iq Darwin ; then ExperimentalBootstrap "Mac OS X" BootstrapMac - elif grep -iq "Amazon Linux" /etc/issue ; then + elif [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue ; then ExperimentalBootstrap "Amazon Linux" BootstrapRpmCommon + elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then + ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" echo echo "You will need to bootstrap, configure virtualenv, and run pip install manually." echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" echo "for more info." + exit 1 fi } @@ -177,7 +222,8 @@ if [ "$1" = "--le-auto-phase2" ]; then shift 1 # the --le-auto-phase2 arg if [ -f "$VENV_BIN/letsencrypt" ]; then # --version output ran through grep due to python-cryptography DeprecationWarnings - INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep ^letsencrypt | cut -d " " -f 2) + # grep for both certbot and letsencrypt until certbot and shim packages have been released + INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) else INSTALLED_VERSION="none" fi @@ -193,6 +239,7 @@ if [ "$1" = "--le-auto-phase2" ]; then echo "Installing Python packages..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" @@ -209,7 +256,6 @@ UNLIKELY_EOF PIP_OUT=`"$VENV_BIN/pip" install --no-cache-dir --require-hashes -r "$TEMP_DIR/letsencrypt-auto-requirements.txt" 2>&1` PIP_STATUS=$? set -e - rm -rf "$TEMP_DIR" if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) echo "Had a problem while installing Python packages:" @@ -219,18 +265,34 @@ UNLIKELY_EOF fi echo "Installation succeeded." fi - echo "Requesting root privileges to run certbot..." - echo " " $SUDO "$VENV_BIN/letsencrypt" "$@" - $SUDO "$VENV_BIN/letsencrypt" "$@" + if [ -n "$SUDO" ]; then + # SUDO is su wrapper or sudo + if [ "$QUIET" != 1 ]; then + echo "Requesting root privileges to run certbot..." + echo " $VENV_BIN/letsencrypt" "$@" + fi + fi + if [ -z "$SUDO_ENV" ] ; then + # SUDO is su wrapper / noop + $SUDO "$VENV_BIN/letsencrypt" "$@" + else + # sudo + $SUDO "$SUDO_ENV" "$VENV_BIN/letsencrypt" "$@" + fi + else - # Phase 1: Upgrade letsencrypt-auto if neceesary, then self-invoke. + # Phase 1: Upgrade certbot-auto if neceesary, then self-invoke. # # Each phase checks the version of only the thing it is responsible for # upgrading. Phase 1 checks the version of the latest release of - # letsencrypt-auto (which is always the same as that of the certbot + # certbot-auto (which is always the same as that of the certbot # package). Phase 2 checks the version of the locally installed certbot. if [ ! -f "$VENV_BIN/letsencrypt" ]; then + if [ "$HELP" = 1 ]; then + echo "$USAGE" + exit 0 + fi # If it looks like we've never bootstrapped before, bootstrap: Bootstrap fi @@ -240,40 +302,36 @@ else fi if [ "$NO_SELF_UPGRADE" != 1 ]; then - echo "Checking for new version..." TEMP_DIR=$(TempDir) + trap 'rm -rf "$TEMP_DIR"' EXIT # --------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/fetch.py" {{ fetch.py }} UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion - REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` - if [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading letsencrypt-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then + echo "WARNING: unable to check for updates." + elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then + echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of # future Windows compatibility. "$LE_PYTHON" "$TEMP_DIR/fetch.py" --le-auto-script "v$REMOTE_VERSION" - # Install new copy of letsencrypt-auto. + # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing letsencrypt-auto..." + echo "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on OS X or BSD, and stat -c on Linux is stat -f on OS X and BSD: - echo " " $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" - echo " " $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" $SUDO cp "$TEMP_DIR/letsencrypt-auto" "$TEMP_DIR/letsencrypt-auto.permission-clone" # Using mv rather than cp leaves the old file descriptor pointing to the # original copy so the shell can continue to read it unmolested. mv across # filesystems is non-atomic, doing `rm dest, cp src dest, rm src`, but the # cp is unlikely to fail (esp. under sudo) if the rm doesn't. - echo " " $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" $SUDO mv -f "$TEMP_DIR/letsencrypt-auto.permission-clone" "$0" - # TODO: Clean up temp dir safely, even if it has quotes in its path. - rm -rf "$TEMP_DIR" fi # A newer version is available. fi # Self-upgrading is allowed. diff --git a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh index b2fc01a14..39e2da5fe 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/arch_common.sh @@ -21,7 +21,11 @@ BootstrapArchCommon() { # pacman -T exits with 127 if there are missing dependencies missing=$($SUDO pacman -T $deps) || true + if [ "$ASSUME_YES" = 1 ]; then + noconfirm="--noconfirm" + fi + if [ "$missing" ]; then - $SUDO pacman -S --needed $missing + $SUDO pacman -S --needed $missing $noconfirm fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index 57ed11399..8eb7e16ee 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -23,7 +23,7 @@ BootstrapDebCommon() { # distro version (#346) virtualenv= - if apt-cache show virtualenv > /dev/null 2>&1; then + if apt-cache show virtualenv > /dev/null 2>&1 && ! apt-cache --quiet=0 show virtualenv 2>&1 | grep -q 'No packages found'; then virtualenv="virtualenv" fi @@ -34,30 +34,45 @@ BootstrapDebCommon() { augeas_pkg="libaugeas0 augeas-lenses" AUGVERSION=`apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` + if [ "$ASSUME_YES" = 1 ]; then + YES_FLAG="-y" + fi + AddBackportRepo() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" + echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - if echo $BACKPORT_NAME | grep -q wheezy ; then - /bin/echo '(Backports are only installed if explicitly requested via "apt-get install -t wheezy-backports")' + if [ "$ASSUME_YES" = 1 ]; then + /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." + sleep 1s + /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." + sleep 1s + /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." + sleep 1s + add_backports=1 + else + read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response + case $response in + [yY][eE][sS]|[yY]|"") + add_backports=1;; + *) + add_backports=0;; + esac + fi + if [ "$add_backports" = 1 ]; then + $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" + $SUDO apt-get update fi - - $SUDO sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - $SUDO apt-get update fi fi - $SUDO apt-get install -y --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - + if [ "$add_backports" != 0 ]; then + $SUDO apt-get install $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg + augeas_pkg= + fi } @@ -74,7 +89,7 @@ BootstrapDebCommon() { # XXX add a case for ubuntu PPAs fi - $SUDO apt-get install -y --no-install-recommends \ + $SUDO apt-get install $YES_FLAG --no-install-recommends \ python \ python-dev \ $virtualenv \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh index 79e58eb3f..2b04977c8 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/mac.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/mac.sh @@ -16,7 +16,8 @@ BootstrapMac() { $pkgcmd augeas $pkgcmd dialog - if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" ]; then + if [ "$(which python)" = "/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python" \ + -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. echo "Installing python..." @@ -26,7 +27,8 @@ BootstrapMac() { # Workaround for _dlopen not finding augeas on OS X if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then echo "Applying augeas workaround" - $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib + $SUDO mkdir -p /usr/local/lib/ + $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then diff --git a/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh new file mode 100644 index 000000000..d6651574a --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/mageia_common.sh @@ -0,0 +1,23 @@ +BootstrapMageiaCommon() { + if ! $SUDO urpmi --force \ + python \ + libpython-devel \ + python-virtualenv + then + echo "Could not install Python dependencies. Aborting bootstrap!" + exit 1 + fi + + if ! $SUDO urpmi --force \ + git \ + gcc \ + cdialog \ + python-augeas \ + libopenssl-devel \ + libffi-devel \ + rootcerts + then + echo "Could not install additional dependencies. Aborting bootstrap!" + exit 1 + fi +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh index 68a11a531..0f98b4bbc 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/rpm_common.sh @@ -1,8 +1,9 @@ BootstrapRpmCommon() { # Tested with: - # - Fedora 22, 23 (x64) + # - Fedora 20, 21, 22, 23 (x64) # - Centos 7 (x64: on DigitalOcean droplet) # - CentOS 7 Minimal install in a Hyper-V VM + # - CentOS 6 (EPEL must be installed manually) if type dnf 2>/dev/null then @@ -16,46 +17,49 @@ BootstrapRpmCommon() { exit 1 fi + pkgs=" + gcc + dialog + augeas-libs + openssl + openssl-devel + libffi-devel + redhat-rpm-config + ca-certificates + " + # Some distros and older versions of current distros use a "python27" # instead of "python" naming convention. Try both conventions. - if ! $SUDO $tool install -y \ - python \ - python-devel \ - python-virtualenv \ - python-tools \ - python-pip - then - if ! $SUDO $tool install -y \ - python27 \ - python27-devel \ - python27-virtualenv \ - python27-tools \ - python27-pip - then - echo "Could not install Python dependencies. Aborting bootstrap!" - exit 1 - fi + if $SUDO $tool list python >/dev/null 2>&1; then + pkgs="$pkgs + python + python-devel + python-virtualenv + python-tools + python-pip + " + else + pkgs="$pkgs + python27 + python27-devel + python27-virtualenv + python27-tools + python27-pip + " fi - if ! $SUDO $tool install -y \ - gcc \ - dialog \ - augeas-libs \ - openssl \ - openssl-devel \ - libffi-devel \ - redhat-rpm-config \ - ca-certificates - then - echo "Could not install additional dependencies. Aborting bootstrap!" - exit 1 - fi - - if $SUDO $tool list installed "httpd" >/dev/null 2>&1; then - if ! $SUDO $tool install -y mod_ssl - then - echo "Apache found, but mod_ssl could not be installed." - fi + pkgs="$pkgs + mod_ssl + " + fi + + if [ "$ASSUME_YES" = 1 ]; then + yes_flag="-y" + fi + + if ! $SUDO $tool install $yes_flag $pkgs; then + echo "Could not install OS dependencies. Aborting bootstrap!" + exit 1 fi } diff --git a/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh b/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh new file mode 100644 index 000000000..e721c1c0b --- /dev/null +++ b/letsencrypt-auto-source/pieces/bootstrappers/smartos.sh @@ -0,0 +1,4 @@ +BootstrapSmartOS() { + pkgin update + pkgin -y install 'gcc49' 'py27-augeas' 'py27-virtualenv' +} diff --git a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh index 46c60f992..9ac295922 100755 --- a/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/suse_common.sh @@ -1,7 +1,12 @@ BootstrapSuseCommon() { # SLE12 don't have python-virtualenv - $SUDO zypper -nq in -l \ + if [ "$ASSUME_YES" = 1 ]; then + zypper_flags="-nq" + install_flags="-l" + fi + + $SUDO zypper $zypper_flags in $install_flags \ python \ python-devel \ python-virtualenv \ diff --git a/letsencrypt-auto-source/pieces/fetch.py b/letsencrypt-auto-source/pieces/fetch.py index 39ff7777c..d11f8da61 100644 --- a/letsencrypt-auto-source/pieces/fetch.py +++ b/letsencrypt-auto-source/pieces/fetch.py @@ -68,7 +68,7 @@ def latest_stable_version(get): """Return the latest stable release of letsencrypt.""" metadata = loads(get( environ.get('LE_AUTO_JSON_URL', - 'https://pypi.python.org/pypi/letsencrypt/json'))) + 'https://pypi.python.org/pypi/certbot/json'))) # metadata['info']['version'] actually returns the latest of any kind of # release release, contrary to https://wiki.python.org/moin/PyPIJSON. # The regex is a sufficient regex for picking out prereleases for most @@ -87,7 +87,7 @@ def verified_new_le_auto(get, tag, temp_dir): """ le_auto_dir = environ.get( 'LE_AUTO_DIR_TEMPLATE', - 'https://raw.githubusercontent.com/letsencrypt/letsencrypt/%s/' + 'https://raw.githubusercontent.com/certbot/certbot/%s/' 'letsencrypt-auto-source/') % tag write(get(le_auto_dir + 'letsencrypt-auto'), temp_dir, 'letsencrypt-auto') write(get(le_auto_dir + 'letsencrypt-auto.sig'), temp_dir, 'letsencrypt-auto.sig') @@ -103,7 +103,7 @@ def verified_new_le_auto(get, tag, temp_dir): stderr=dev_null) except CalledProcessError as exc: raise ExpectedError("Couldn't verify signature of downloaded " - "letsencrypt-auto.", exc) + "certbot-auto.", exc) def main(): diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt index 27cfb3d43..0c642a33e 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt @@ -1,4 +1,4 @@ -# This is the flattened list of packages letsencrypt-auto installs. To generate +# This is the flattened list of packages certbot-auto installs. To generate # this, do `pip install --no-cache-dir -e acme -e . -e certbot-apache`, and # then use `hashin` or a more secure method to gather the hashes. @@ -32,28 +32,29 @@ ConfigArgParse==0.10.0 \ --hash=sha256:3b50a83dd58149dfcee98cb6565265d10b53e9c0a2bca7eeef7fb5f5524890a7 configobj==5.0.6 \ --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==1.2.3 \ - --hash=sha256:031938f73a5c5eb3e809e18ff7caeb6865351871417be6050cb8c86a9a202b9a \ - --hash=sha256:a179a38d50f8d68b491d7a313db78f8cabe290842cecddddc7b34d408e59db0a \ - --hash=sha256:906c88b2aadcf99cfabb24098263d1bf65ab0c8688acde10dae1f09d865920f1 \ - --hash=sha256:6e706c5c6088770b1d1b634e959e21963e315b0255f5f4777125ad3d54082977 \ - --hash=sha256:f5ebf8e31c48f8707921dca0e994de77813a9c9b9bf03c119c5ddf97bdcffe73 \ - --hash=sha256:c7b89e42288cc7fbee3812e99ef5c744f22452e11d6822f6807afc6d6b3be83e \ - --hash=sha256:8408d29865947109d8b68f1837a7cde1aa4dc86e0f79ca3ba58c0c44e443d6a5 \ - --hash=sha256:c7e76cf3c3d925dd31fa238cfb806cffba718c0f08707d77a538768477969956 \ - --hash=sha256:7d8de35380f31702758b7753bb5c40723832c73006dedb2f9099bf61a37f7287 \ - --hash=sha256:5edbee71fae5469ee83fe0a37866b9398c8ce3a46325c24fcedfbf097bb48a19 \ - --hash=sha256:594edafe4801c13bdc1cc305e7704a90c19617e95936f6ab457ee4ffe000ba50 \ - --hash=sha256:b7fdb16a0a7f481be42da744bfe1ea2163025de21f90f2c688a316f3c354da9c \ - --hash=sha256:207b8bf0fe0907336df38b733b487521cf9e138189aba9234ad54fe545dd0db8 \ - --hash=sha256:509a2f05386270cf783993c90d49ffefb3dd62aee45bf1ea8ce3d2cde7271c21 \ - --hash=sha256:ac69b65dd1af0179ede40c9f15788c88f73e628ea6c0519de3838e279bb388c6 \ - --hash=sha256:8df6fad6c6ae12fd7004ea29357f0a2b4d3774eaeca7656530d08d2d90cd41aa \ - --hash=sha256:0b8b96dd81cc1533a04f30382c0fe21c1972e189f794d0c4261a18cec08fd9b5 \ - --hash=sha256:cae8fca1883f23c50ea78d89de6fe4fefdb4cea83177760f47177559414ded93 \ - --hash=sha256:1a471ca576a9cdce1b1cd9f3a22b1d09ee44d46862037557de17919c0db44425 \ - --hash=sha256:8ec4e8e3d453b3a1b63b5f57737a434dcf1ee4a2f26f6ff7c5a37c3f679104d2 \ - --hash=sha256:8eb11c77dd8e73f48df6b2f7a7e16173fe0fe8fdfe266232832e88477e08454e +cryptography==1.3.4 \ + --hash=sha256:bede00edd11a2a62c8c98c271cc103fa3a3d72acf64f6e5e4eaf251128897b17 \ + --hash=sha256:53b39e687b744bb548a98f40736cc529d9f60959b4e6cc551322cf9505d35eb3 \ + --hash=sha256:474b73ad1139b4e423e46bbd818efd0d5c0df1c65d9f7c957d64c9215d77afde \ + --hash=sha256:aaddf9592d5b99e32dd518bb4a25b147c124f9d6b4ad64b94f01b15d1666b8c8 \ + --hash=sha256:6dcad2f407db8c3cd6ecd78361439c449a4f94786b46c54507e7e68f51e1709d \ + --hash=sha256:475c153fc622e656f1f10a9c9941d0ac7ab18df7c38d35d563a437c1c0e34f24 \ + --hash=sha256:86dd61df581cba04e89e45081efbc531faff1c9d99c77b1ce97f87216c356353 \ + --hash=sha256:75cc697e4ef5fdd0102ca749114c6370dbd11db0c9132a18834858c2566247e3 \ + --hash=sha256:ea03ad5b9df6d79fc9fc1ab23729e01e1c920d2974c5e3c634ccf45a5c378452 \ + --hash=sha256:c8872b8fe4f3416d6338ab99612f49ab314f7856cb43bffab2a32d28a6267be8 \ + --hash=sha256:468fc6e16eaec6ceaa6bc341273e6e9912d01b42b740f8cf896ace7fcd6a321d \ + --hash=sha256:d6fea3c6502735011c5d61a62aef1c1d770fc6a2def45d9e6c0d94c9651e3317 \ + --hash=sha256:3cf95f179f4bead3d5649b91860ef4cf60ad4244209190fc405908272576d961 \ + --hash=sha256:141f77e60a5b9158309b2b60288c7f81d37faa15c22a69b94c190ceefaaa6236 \ + --hash=sha256:87b7a1fe703c6424451f3372d1879dae91c7fe5e13375441a72833db76fee30e \ + --hash=sha256:f5ee3cb0cf1a6550bf483ccffa6608db267a377b45f7e3a8201a86d1d8feb19f \ + --hash=sha256:4e097286651ea318300af3251375d48b71b8228481c56cd617ddd4459a1ff261 \ + --hash=sha256:1e3d3ae3f22f22d50d340f47f25227511326f3f1396c6d2446a5b45b516c4313 \ + --hash=sha256:6a057941cb64d79834ea3cf99093fcc4787c2a5d44f686c4f297361ddc419bcd \ + --hash=sha256:68b3d5390b92559ddd3353c73ab2dfcff758f9c4ec4f5d5226ccede0e5d779f4 \ + --hash=sha256:545dc003b4b6081f9c3e452da15d819b04b696f49484aff64c0a2aedf766bef8 \ + --hash=sha256:423ff890c01be7c70dbfeaa967eeef5146f1a43a5f810ffdc07b178e48a105a9 enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 @@ -113,9 +114,9 @@ pyasn1==0.1.9 \ --hash=sha256:5191ff6b9126d2c039dd87f8ff025bed274baf07fa78afa46f556b1ad7265d6e \ --hash=sha256:8323e03637b2d072cc7041300bac6ec448c3c28950ab40376036788e9a1af629 \ --hash=sha256:853cacd96d1f701ddd67aa03ecc05f51890135b7262e922710112f12a2ed2a7f -pyOpenSSL==0.15.1 \ - --hash=sha256:88e45e6bb25dfed272a1ef2e728461d44b634c2cd689e989b6e56a349c5a3ae5 \ - --hash=sha256:f0a26070d6db0881de8bcc7846934b7c3c930d8f9c79d45883ee48984bc0d672 +pyopenssl==16.0.0 \ + --hash=sha256:5add70cf00273bf957ca31fdb0df9b0ae4639e081897d5f86a0ae1f104901230 \ + --hash=sha256:363d10ee43d062285facf4e465f4f5163f9f702f9134f0a5896f134cbb92d17d pyRFC3339==1.0 \ --hash=sha256:eea31835c56e2096af4363a5745a784878a61d043e247d3a6d6a0a32a9741f56 \ --hash=sha256:8dfbc6c458b8daba1c0f3620a8c78008b323a268b27b7359e92a4ae41325f535 @@ -175,15 +176,18 @@ zope.interface==4.1.3 \ mock==1.0.1 \ --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.5.0 \ - --hash=sha256:ceb4127c13213f0006a564be82176b968c6b374d20d9fc78555d0658a252b275 \ - --hash=sha256:0605c63c656d33c883a05675f5db9cfb85d503f2771c885031800e0da7631abd -letsencrypt==0.5.0 \ - --hash=sha256:f90f883e99cdbdf8142335bdbf4f74a8af143ee4b4ec60fb49c6e47418c1114c \ - --hash=sha256:e38a2b70b82be79bc195307652244a3e012ec73d897d4dbd3f80cf698496d15a -letsencrypt-apache==0.5.0 \ - --hash=sha256:a767882164a7b09d9c12c80684a28a782135fdaf35654ef5a02c0b7b1d27ab8d \ - --hash=sha256:c20e7b9c517aa4a7d70e6bd9382da7259f00bc191b9e60d8e312e48837a00c41 +acme==0.8.1 \ + --hash=sha256:ccd7883772efbf933f91713b8241455993834f3620c8fbd459d9ed5e50bbaaca \ + --hash=sha256:d3ea4acf280bf6253ad7d641cb0970f230a19805acfed809e7a8ddcf62157d9f +certbot==0.8.1 \ + --hash=sha256:89805d9f70249ae859ec4d7a99c00b4bb7083ca90cd12d4d202b76dfc284f7c5 \ + --hash=sha256:6ca8df3d310ced6687d38aac17c0fb8c1b2ec7a3bea156a254e4cc2a1c132771 +certbot-apache==0.8.1 \ + --hash=sha256:c9e3fdc15e65589c2e39eb0e6b1f61f0c0a1db3c17b00bb337f0ff636cc61cb3 \ + --hash=sha256:0faf2879884d3b7a58b071902fba37d4b8b58a50e2c3b8ac262c0a74134045ed diff --git a/letsencrypt-auto-source/tests/auto_test.py b/letsencrypt-auto-source/tests/auto_test.py index edb5f0c04..56023bc6f 100644 --- a/letsencrypt-auto-source/tests/auto_test.py +++ b/letsencrypt-auto-source/tests/auto_test.py @@ -11,7 +11,7 @@ from shutil import copy, rmtree import socket import ssl from stat import S_IRUSR, S_IXUSR -from subprocess import CalledProcessError, check_output, Popen, PIPE +from subprocess import CalledProcessError, Popen, PIPE import sys from tempfile import mkdtemp from threading import Thread @@ -146,7 +146,9 @@ def out_and_err(command, input=None, shell=False, env=None): out, err = process.communicate(input=input) status = process.poll() # same as in check_output(), though wait() sounds better if status: - raise CalledProcessError(status, command, output=out) + error = CalledProcessError(status, command) + error.output = out + raise error return out, err @@ -183,7 +185,7 @@ def run_le_auto(venv_dir, base_url, **kwargs): d = dict(XDG_DATA_HOME=venv_dir, # URL to PyPI-style JSON that tell us the latest released version # of LE: - LE_AUTO_JSON_URL=base_url + 'letsencrypt/json', + LE_AUTO_JSON_URL=base_url + 'certbot/json', # URL to dir containing letsencrypt-auto and letsencrypt-auto.sig: LE_AUTO_DIR_TEMPLATE=base_url + '%s/', # The public key corresponding to signing.key: @@ -231,7 +233,7 @@ class AutoTests(TestCase): * The OpenSSL sig mismatches. For tests which get to the end, we run merely ``letsencrypt --version``. - The functioning of the rest of the letsencrypt script is covered by other + The functioning of the rest of the certbot script is covered by other test suites. """ @@ -258,7 +260,7 @@ class AutoTests(TestCase): with ephemeral_dir() as venv_dir: # This serves a PyPI page with a higher version, a GitHub-alike # with a corresponding le-auto script, and a matching signature. - resources = {'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + resources = {'certbot/json': dumps({'releases': {'99.9.9': None}}), 'v99.9.9/letsencrypt-auto': NEW_LE_AUTO, 'v99.9.9/letsencrypt-auto.sig': NEW_LE_AUTO_SIG} with serving(resources) as base_url: @@ -277,7 +279,7 @@ class AutoTests(TestCase): ok_(re.match(r'letsencrypt \d+\.\d+\.\d+', err.strip().splitlines()[-1])) # Make a few assertions to test the validity of the next tests: - self.assertIn('Upgrading letsencrypt-auto ', out) + self.assertIn('Upgrading certbot-auto ', out) self.assertIn('Creating virtual environment...', out) # Now we have le-auto 99.9.9 and LE 99.9.9 installed. This @@ -286,14 +288,14 @@ class AutoTests(TestCase): # Test when neither phase-1 upgrade nor phase-2 upgrade is # needed (probably a common case): out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertNotIn('Upgrading certbot-auto ', out) self.assertNotIn('Creating virtual environment...', out) # Test when a phase-1 upgrade is not needed but a phase-2 # upgrade is: set_le_script_version(venv_dir, '0.0.1') out, err = run_letsencrypt_auto() - self.assertNotIn('Upgrading letsencrypt-auto ', out) + self.assertNotIn('Upgrading certbot-auto ', out) self.assertIn('Creating virtual environment...', out) def test_openssl_failure(self): @@ -301,8 +303,8 @@ class AutoTests(TestCase): with ephemeral_dir() as venv_dir: # Serve an unrelated hash signed with the good key (easier than # making a bad key, and a mismatch is a mismatch): - resources = {'': 'letsencrypt/', - 'letsencrypt/json': dumps({'releases': {'99.9.9': None}}), + resources = {'': 'certbot/', + 'certbot/json': dumps({'releases': {'99.9.9': None}}), 'v99.9.9/letsencrypt-auto': build_le_auto(version='99.9.9'), 'v99.9.9/letsencrypt-auto.sig': signed('something else')} with serving(resources) as base_url: @@ -312,16 +314,16 @@ class AutoTests(TestCase): except CalledProcessError as exc: eq_(exc.returncode, 1) self.assertIn("Couldn't verify signature of downloaded " - "letsencrypt-auto.", + "certbot-auto.", exc.output) else: - self.fail('Signature check on letsencrypt-auto erroneously passed.') + self.fail('Signature check on certbot-auto erroneously passed.') def test_pip_failure(self): """Make sure pip stops us if there is a hash mismatch.""" with ephemeral_dir() as venv_dir: - resources = {'': 'letsencrypt/', - 'letsencrypt/json': dumps({'releases': {'99.9.9': None}})} + resources = {'': 'certbot/', + 'certbot/json': dumps({'releases': {'99.9.9': None}})} with serving(resources) as base_url: # Build a le-auto script embedding a bad requirements file: install_le_auto( diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index b94b7f69f..25db12a47 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -16,7 +16,7 @@ here = os.path.abspath(os.path.dirname(__file__)) readme = read_file(os.path.join(here, 'README.rst')) -version = '0.6.0.dev0' +version = '0.8.0.dev0' # This package is a simple shim around certbot-nginx diff --git a/letsencrypt/setup.py b/letsencrypt/setup.py index 708c31f4b..4541e85fe 100644 --- a/letsencrypt/setup.py +++ b/letsencrypt/setup.py @@ -20,7 +20,7 @@ readme = read_file(os.path.join(here, 'README.rst')) install_requires = ['certbot'] -version = '0.6.0.dev0' +version = '0.8.0.dev0' setup( diff --git a/letsencrypt/tests/testdata/os-release b/letsencrypt/tests/testdata/os-release new file mode 100644 index 000000000..b7c3ceb1b --- /dev/null +++ b/letsencrypt/tests/testdata/os-release @@ -0,0 +1,8 @@ +NAME="SystemdOS" +VERSION="42.42.42 LTS, Unreal" +ID=systemdos +ID_LIKE=debian +PRETTY_NAME="SystemdOS 42.42.42 Unreal" +VERSION_ID="42" +HOME_URL="http://www.example.com/" +SUPPORT_URL="http://help.example.com/" diff --git a/letshelp-certbot/docs/conf.py b/letshelp-certbot/docs/conf.py index 905d70662..17d8b3ea9 100644 --- a/letshelp-certbot/docs/conf.py +++ b/letshelp-certbot/docs/conf.py @@ -307,5 +307,5 @@ texinfo_documents = [ intersphinx_mapping = { 'python': ('https://docs.python.org/', None), 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), - 'certbot': ('https://letsencrypt.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), } diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index 8359d2766..b616da688 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.6.0.dev0' +version = '0.7.0.dev0' install_requires = [ 'setuptools', # pkg_resources diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index 875c6fc92..10380c49b 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -16,7 +16,7 @@ here = os.path.abspath(os.path.dirname(__file__)) readme = read_file(os.path.join(here, 'README.rst')) -version = '0.6.0.dev0' +version = '0.7.0.dev0' # This package is a simple shim around letshelp-certbot diff --git a/pep8.travis.sh b/pep8.travis.sh index c13547a78..cadea8489 100755 --- a/pep8.travis.sh +++ b/pep8.travis.sh @@ -2,16 +2,4 @@ set -e # Fail fast -# PEP8 is not ignored in ACME pep8 --config=acme/.pep8 acme - -pep8 \ - setup.py \ - certbot \ - certbot-apache \ - certbot-nginx \ - certbot-compatibility-test \ - letshelp-certbot \ - || echo "PEP8 checking failed, but it's ignored in Travis" - -# echo exits with 0 diff --git a/setup.py b/setup.py index 4ee56576b..21cda901a 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ install_requires = [ 'configobj', 'cryptography>=0.7', # load_pem_x509_certificate 'parsedatetime>=1.3', # Calendar.parseDT - 'psutil>=2.1.0', # net_connections introduced in 2.1.0 + 'psutil>=2.2.1', # 2.1.0 for net_connections and 2.2.1 resolves #1080 'PyOpenSSL', 'pyrfc3339', 'python2-pythondialog>=3.2.2rc1', # Debian squeeze support, cf. #280 @@ -69,7 +69,6 @@ dev_extras = [ 'astroid==1.3.5', 'coverage', 'nose', - 'nosexcover', 'pep8', 'pylint==1.4.2', # upstream #248 'tox', diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 0d8a3de38..469c5cd80 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -1,40 +1,10 @@ #!/bin/bash # Download and run Boulder instance for integration testing - -# ugh, go version output is like: -# go version go1.4.2 linux/amd64 -GOVER=`go version | cut -d" " -f3 | cut -do -f2` - -# version comparison -function verlte { - #OS X doesn't support version sorting; emulate with sed - if [ `uname` == 'Darwin' ]; then - [ "$1" = "`echo -e \"$1\n$2\" | sed 's/\b\([0-9]\)\b/0\1/g' \ - | sort | sed 's/\b0\([0-9]\)/\1/g' | head -n1`" ] - else - [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] - fi -} - -if ! verlte 1.5 "$GOVER" ; then - echo "We require go version 1.5 or later; you have... $GOVER" - exit 1 -fi - set -xe -# `/...` avoids `no buildable Go source files` errors, for more info -# see `go help packages` -go get -d github.com/letsencrypt/boulder/... -cd $GOPATH/src/github.com/letsencrypt/boulder -# goose is needed for ./test/create_db.sh -wget https://github.com/jsha/boulder-tools/raw/master/goose.gz && \ - mkdir $GOPATH/bin && \ - zcat goose.gz > $GOPATH/bin/goose && \ - chmod +x $GOPATH/bin/goose -./test/create_db.sh -go run cmd/rabbitmq-setup/main.go -server amqp://localhost -# listenbuddy is needed for ./start.py -go get github.com/jsha/listenbuddy -cd - +# Check out special branch until latest docker changes land in Boulder master. +git clone -b docker-integration https://github.com/letsencrypt/boulder $BOULDERPATH +cd $BOULDERPATH +sed -i 's/FAKE_DNS: .*/FAKE_DNS: 172.17.42.1/' docker-compose.yml +docker-compose up -d diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 201343525..ab8fde5f6 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -43,8 +43,8 @@ export CSR_PATH="${root}/csr.der" KEY_PATH="${root}/key.pem" \ common auth --csr "$CSR_PATH" \ --cert-path "${root}/csr/cert.pem" \ --chain-path "${root}/csr/chain.pem" -openssl x509 -in "${root}/csr/0000_cert.pem" -text -openssl x509 -in "${root}/csr/0000_chain.pem" -text +openssl x509 -in "${root}/csr/cert.pem" -text +openssl x509 -in "${root}/csr/chain.pem" -text common --domains le3.wtf install \ --cert-path "${root}/csr/cert.pem" \ @@ -84,6 +84,24 @@ if [ "$size1" -lt 3000 ] || [ "$size2" -lt 3000 ] || [ "$size3" -gt 1800 ] ; the exit 1 fi +# ECDSA +openssl ecparam -genkey -name secp384r1 -out "${root}/privkey-p384.pem" +SAN="DNS:ecdsa.le.wtf" openssl req -new -sha256 \ + -config "${OPENSSL_CNF:-openssl.cnf}" \ + -key "${root}/privkey-p384.pem" \ + -subj "/" \ + -reqexts san \ + -outform der \ + -out "${root}/csr-p384.der" +common auth --csr "${root}/csr-p384.der" \ + --cert-path "${root}/csr/cert-p384.pem" \ + --chain-path "${root}/csr/chain-p384.pem" +openssl x509 -in "${root}/csr/cert-p384.pem" -text | grep 'ASN1 OID: secp384r1' + +# OCSP Must Staple +common auth --must-staple --domains "must-staple.le.wtf" +openssl x509 -in "${root}/conf/live/must-staple.le.wtf/cert.pem" -text | grep '1.3.6.1.5.5.7.1.24' + # revoke by account key common revoke --cert-path "$root/conf/live/le.wtf/cert.pem" # revoke renewed diff --git a/tests/letstest/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh index b5ddf2c5b..7e298783f 100755 --- a/tests/letstest/scripts/boulder_install.sh +++ b/tests/letstest/scripts/boulder_install.sh @@ -2,27 +2,8 @@ # >>>> only tested on Ubuntu 14.04LTS <<<< -# non-interactive install of mariadb and other dependencies -export DEBIAN_FRONTEND=noninteractive -sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS' -sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS' -apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server -sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');" - -# install go -wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz -tar xzvf go1.5.1.linux-amd64.tar.gz -mkdir gocode -echo "export GOROOT=/home/ubuntu/go \n\ - export GOPATH=/home/ubuntu/gocode\n\ - export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc - -# install boulder and its go dependencies -go get -d github.com/letsencrypt/boulder/... -cd $GOPATH/src/github.com/letsencrypt/boulder -wget https://github.com/jsha/boulder-tools/raw/master/goose.gz -mkdir $GOPATH/bin -zcat goose.gz > $GOPATH/bin/goose -chmod +x $GOPATH/bin/goose -./test/create_db.sh -go get github.com/jsha/listenbuddy +# Check out special branch until latest docker changes land in Boulder master. +git clone -b docker-integration https://github.com/letsencrypt/boulder $BOULDERPATH +cd $BOULDERPATH +sed -i 's/FAKE_DNS: .*/FAKE_DNS: 172.17.42.1/' docker-compose.yml +docker-compose up -d diff --git a/tests/travis-integration.sh b/tests/travis-integration.sh index 159a2ef80..b42617400 100755 --- a/tests/travis-integration.sh +++ b/tests/travis-integration.sh @@ -6,14 +6,9 @@ set -o errexit source .tox/$TOXENV/bin/activate -export CERTBOT_PATH=`pwd` +until curl http://boulder:4000/directory 2>/dev/null; do + echo waiting for boulder + sleep 1 +done -cd $GOPATH/src/github.com/letsencrypt/boulder/ - -# boulder's integration-test.py has code that knows to start and wait for the -# boulder processes to start reliably and then will run the certbot -# boulder-interation.sh on its own. The --certbot flag says to run only the -# certbot tests (instead of any other client tests it might run). We're -# going to want to define a more robust interaction point between the boulder -# and certbot tests, but that will be better built off of this. -python test/integration-test.py --certbot +./tests/boulder-integration.sh diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh index dc6ca3dd2..a121af82d 100755 --- a/tools/_venv_common.sh +++ b/tools/_venv_common.sh @@ -18,8 +18,7 @@ virtualenv --no-site-packages $VENV_NAME $VENV_ARGS # Separately install setuptools and pip to make sure following # invocations use latest pip install -U setuptools -# --force-reinstall used to fix broken pip installation on some systems -pip install --force-reinstall -U pip +pip install -U pip pip install "$@" set +x diff --git a/tools/release.sh b/tools/release.sh index d41192af9..c883e3d61 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -45,7 +45,7 @@ export GPG_TTY=$(tty) PORT=${PORT:-1234} # subpackages to be released -SUBPKGS=${SUBPKGS:-"acme certbot-apache certbot-nginx letshelp-certbot"} +SUBPKGS=${SUBPKGS:-"acme certbot-apache certbot-nginx"} subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs @@ -145,6 +145,9 @@ pip install \ kill $! cd ~- +# get a snapshot of the CLI help for the docs +certbot --help all > docs/cli-help.txt + # freeze before installing anything else, so that we know end-user KGS # make sure "twine upload" doesn't catch "kgs" if [ -d ../kgs ] ; then @@ -159,13 +162,13 @@ for module in certbot $subpkgs_modules ; do echo testing $module nosetests $module done -deactivate # pin pip hashes of the things we just built for pkg in acme certbot certbot-apache ; do echo $pkg==$version \\ pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' done > /tmp/hashes.$$ +deactivate if ! wc -l /tmp/hashes.$$ | grep -qE "^\s*9 " ; then echo Unexpected pip hash output @@ -187,10 +190,17 @@ while ! openssl dgst -sha256 -verify $RELEASE_OPENSSL_PUBKEY -signature \ read -p "Please correctly sign letsencrypt-auto with offline-signrequest.sh" done +# This signature is not quite as strong, but easier for people to verify out of band +gpg -u "$RELEASE_GPG_KEY" --detach-sign --armor --sign letsencrypt-auto-source/letsencrypt-auto +# We can't rename the openssl letsencrypt-auto.sig for compatibility reasons, +# but we can use the right name for cerbot-auto.asc from day one +mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot-auto.asc + # copy leauto to the root, overwriting the previous release version +cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto -git add letsencrypt-auto letsencrypt-auto-source +git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt git diff --cached git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version" git tag --local-user "$RELEASE_GPG_KEY" --sign --message "Release $version" "$tag" diff --git a/tox.ini b/tox.ini index 5c88dfd21..689bbf513 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = py{26,27,33,34,35},py{26,27}-oldest,cover,lint +envlist = py{26,33,34,35},cover,lint # nosetest -v => more verbose output, allows to detect busy waiting # loops, especially on Travis @@ -64,14 +64,14 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot + pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot ./pep8.travis.sh - pylint --rcfile=.pylintrc certbot - pylint --rcfile=acme/.pylintrc acme/acme - pylint --rcfile=.pylintrc certbot-apache/certbot_apache - pylint --rcfile=.pylintrc certbot-nginx/certbot_nginx - pylint --rcfile=.pylintrc certbot-compatibility-test/certbot_compatibility_test - pylint --rcfile=.pylintrc letshelp-certbot/letshelp_certbot + pylint --reports=n --rcfile=.pylintrc certbot + pylint --reports=n --rcfile=acme/.pylintrc acme/acme + pylint --reports=n --rcfile=.pylintrc certbot-apache/certbot_apache + pylint --reports=n --rcfile=.pylintrc certbot-nginx/certbot_nginx + pylint --reports=n --rcfile=.pylintrc certbot-compatibility-test/certbot_compatibility_test + pylint --reports=n --rcfile=.pylintrc letshelp-certbot/letshelp_certbot [testenv:apacheconftest] #basepython = python2.7