Create a mock OCSP server for Pebble integration tests (#7281)

* Implement a logic, miss the private key of pebble

* Complete process

* Fix nginx cert path

* Check conditionnally docker

* Update gitignore, fix apacheconftest

* Full object

* Carriage return

* Work in progress

* Move to official v2.1.0 of pebble

* Fix name

* Update acme_server.py

* Link things together with new version of pebble

* Plug the logic to tests

* Update config

* Reinitiate config

* Add OCSP config to pebble

* Working.

* Simplify logic

* Clean code

* Use forked pebble for now

# Conflicts:
#	certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

* Move full logic of mock at the acme server config

* Continue work

* Finish fixing the date parsing

* Update module name

* Use again official pebble

* Activate mock OCSP server

* Clean code

* Update pebble_artifacts.py

* Remove OCSP stale test

* Add executable permissions

* Clean code

* Update setup.py

* Simplify code

* On-demand import of pebble_ocsp_server

* Revert "Remove OCSP stale test"

This reverts commit 2e4c985b42.

# Conflicts:
#	certbot-ci/certbot_integration_tests/utils/misc.py

* Fix for virtualenv on Python 3.7.4 for Windows

* Update acme_server.py

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -4,7 +4,7 @@ import shutil
 import sys
 import tempfile
 
-from certbot_integration_tests.utils import misc, certbot_call
+from certbot_integration_tests.utils import certbot_call
 
 
 class IntegrationTestsContext(object):
@@ -19,7 +19,7 @@ class IntegrationTestsContext(object):
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist
 
-        self.acme_server =acme_xdist['acme_server']
+        self.acme_server = acme_xdist['acme_server']
         self.directory_url = acme_xdist['directory_url']
         self.tls_alpn_01_port = acme_xdist['https_port'][self.worker_id]
         self.http_01_port = acme_xdist['http_port'][self.worker_id]

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -555,11 +555,9 @@ def test_ocsp_status_stale(context):
 
 def test_ocsp_status_live(context):
     """Test retrieval of OCSP statuses for live config"""
-    if context.acme_server == 'pebble':
-        pytest.skip('Pebble does not support OCSP status requests.')
+    cert = context.get_domain('ocsp-check')
 
     # OSCP 1: Check live certificate OCSP status (VALID)
-    cert = context.get_domain('ocsp-check')
     context.certbot(['--domains', cert])
     output = context.certbot(['certificates'])
 

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -134,6 +134,13 @@ class ACMEServer(object):
             [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
              '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
 
+        # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
+        # ImportError, in the case where cryptography dependency is too old to support ocsp, but
+        # Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical
+        # situation of integration-certbot-oldest tox testenv.
+        from certbot_integration_tests.utils import pebble_ocsp_server
+        self._launch_process([sys.executable, pebble_ocsp_server.__file__])
+
         # Wait for the ACME CA server to be up.
         print('=> Waiting for pebble instance to respond...')
         misc.check_until_timeout(self.acme_xdist['directory_url'])

certbot-ci/certbot_integration_tests/utils/constants.py

@@ -5,3 +5,5 @@ CHALLTESTSRV_PORT = 8055
 BOULDER_V1_DIRECTORY_URL = 'http://localhost:4000/directory'
 BOULDER_V2_DIRECTORY_URL = 'http://localhost:4001/directory'
 PEBBLE_DIRECTORY_URL = 'https://localhost:14000/dir'
+PEBBLE_MANAGEMENT_URL = 'https://localhost:15000'
+MOCK_OCSP_SERVER_PORT = 4002

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -23,7 +23,6 @@ from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, NoEncryption
 from six.moves import socketserver, SimpleHTTPServer
 
-
 RSA_KEY_TYPE = 'rsa'
 ECDSA_KEY_TYPE = 'ecdsa'
 

certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py

@@ -6,7 +6,9 @@ import stat
 import pkg_resources
 import requests
 
-PEBBLE_VERSION = 'v2.1.0'
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT
+
+PEBBLE_VERSION = 'v2.2.1'
 ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets')
 
 
@@ -42,10 +44,12 @@ def _build_pebble_config(workspace):
         file_h.write(json.dumps({
             'pebble': {
                 'listenAddress': '0.0.0.0:14000',
+                'managementListenAddress': '0.0.0.0:15000',
                 'certificate': os.path.join(ASSETS_PATH, 'cert.pem'),
                 'privateKey': os.path.join(ASSETS_PATH, 'key.pem'),
                 'httpPort': 5002,
                 'tlsPort': 5001,
+                'ocspResponderURL': 'http://127.0.0.1:{0}'.format(MOCK_OCSP_SERVER_PORT),
             },
         }))
 

certbot-ci/certbot_integration_tests/utils/pebble_ocsp_server.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+"""
+This runnable module interfaces itself with the Pebble management interface in order
+to serve a mock OCSP responder during integration tests against Pebble.
+"""
+import datetime
+import re
+
+import requests
+from dateutil import parser
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization, hashes
+from cryptography import x509
+from cryptography.x509 import ocsp
+from six.moves import BaseHTTPServer
+
+from certbot_integration_tests.utils.misc import GracefulTCPServer
+from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT, PEBBLE_MANAGEMENT_URL
+
+
+class _ProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_POST(self):
+        request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediate-keys/0', verify=False)
+        issuer_key = serialization.load_pem_private_key(request.content, None, default_backend())
+
+        request = requests.get(PEBBLE_MANAGEMENT_URL + '/intermediates/0', verify=False)
+        issuer_cert = x509.load_pem_x509_certificate(request.content, default_backend())
+
+        try:
+            content_len = int(self.headers.getheader('content-length', 0))
+        except AttributeError:
+            content_len = int(self.headers.get('Content-Length'))
+
+        ocsp_request = ocsp.load_der_ocsp_request(self.rfile.read(content_len))
+        response = requests.get('{0}/cert-status-by-serial/{1}'.format(
+            PEBBLE_MANAGEMENT_URL, str(hex(ocsp_request.serial_number)).replace('0x', '')), verify=False)
+
+        if not response.ok:
+            ocsp_response = ocsp.OCSPResponseBuilder.build_unsuccessful(ocsp.OCSPResponseStatus.UNAUTHORIZED)
+        else:
+            data = response.json()
+
+            now = datetime.datetime.utcnow()
+            cert = x509.load_pem_x509_certificate(data['Certificate'].encode(), default_backend())
+            if data['Status'] != 'Revoked':
+                ocsp_status, revocation_time, revocation_reason = ocsp.OCSPCertStatus.GOOD, None, None
+            else:
+                ocsp_status, revocation_reason = ocsp.OCSPCertStatus.REVOKED, x509.ReasonFlags.unspecified
+                revoked_at = re.sub(r'( \+\d{4}).*$', r'\1', data['RevokedAt'])  # "... +0000 UTC" => "+0000"
+                revocation_time = parser.parse(revoked_at)
+
+            ocsp_response = ocsp.OCSPResponseBuilder().add_response(
+                cert=cert, issuer=issuer_cert, algorithm=hashes.SHA1(),
+                cert_status=ocsp_status,
+                this_update=now, next_update=now + datetime.timedelta(hours=1),
+                revocation_time=revocation_time, revocation_reason=revocation_reason
+            ).responder_id(
+                ocsp.OCSPResponderEncoding.NAME, issuer_cert
+            ).sign(issuer_key, hashes.SHA256())
+
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(ocsp_response.public_bytes(serialization.Encoding.DER))
+
+
+if __name__ == '__main__':
+    try:
+        GracefulTCPServer(('', MOCK_OCSP_SERVER_PORT), _ProxyHandler).serve_forever()
+    except KeyboardInterrupt:
+        pass

certbot-ci/setup.py

@@ -11,6 +11,7 @@ install_requires = [
     'pytest',
     'pytest-cov',
     'pytest-xdist',
+    'python-dateutil',
     'pyyaml',
     'requests',
     'six',