diff --git a/letsencrypt/client.py b/letsencrypt/client.py
index 6eb33f3fe..8a0ad6af4 100644
--- a/letsencrypt/client.py
+++ b/letsencrypt/client.py
@@ -329,9 +329,27 @@ class Client(object):
                 self.installer.save()  # needed by the Apache plugin
 
             self.installer.save("Deployed Let's Encrypt Certificate")
-            # sites may have been enabled / final cleanup
+
+        # sites may have been enabled / final cleanup
+        with error_handler.ErrorHandler(self._rollback_and_restart):
             self.installer.restart()
 
+    def _rollback_and_restart(self):
+        """Rollback the most recent checkpoint and restart the webserver"""
+        logger.critical("Rolling back to previous server configuration...")
+        try:
+            self.installer.rollback_checkpoints()
+            self.installer.restart()
+        except:
+            # TODO: suggest letshelp-letsencypt here
+            logger.critical("Failure to rollback config "
+                            "changes and restart your server")
+            logger.critical("Please submit a bug report to "
+                            "https://github.com/letsencrypt/letsencrypt")
+            raise
+        logger.critical("Rollback successful; your server has "
+                        "been restarted with your old configuration")
+
     def enhance_config(self, domains, redirect=None):
         """Enhance the configuration.
 
diff --git a/letsencrypt/tests/client_test.py b/letsencrypt/tests/client_test.py
index b2ebe74f0..4a61194f3 100644
--- a/letsencrypt/tests/client_test.py
+++ b/letsencrypt/tests/client_test.py
@@ -166,6 +166,27 @@ class ClientTest(unittest.TestCase):
         self.assertEqual(installer.save.call_count, 2)
         installer.restart.assert_called_once_with()
 
+    def test_deploy_certificate_restart_failure_with_recovery(self):
+        installer = mock.MagicMock()
+        installer.restart.side_effect = [errors.PluginError, None]
+        self.client.installer = installer
+
+        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
+                          ["foo.bar"], "key", "cert", "chain", "fullchain")
+        installer.rollback_checkpoints.assert_called_once_with()
+        self.assertEqual(installer.restart.call_count, 2)
+
+    def test_deploy_certificate_restart_failure_without_recovery(self):
+        installer = mock.MagicMock()
+        installer.restart.side_effect = errors.PluginError
+        installer.rollback_checkpoints.side_effect = errors.ReverterError
+        self.client.installer = installer
+
+        self.assertRaises(errors.PluginError, self.client.deploy_certificate,
+                          ["foo.bar"], "key", "cert", "chain", "fullchain")
+        installer.rollback_checkpoints.assert_called_once_with()
+        self.assertEqual(installer.restart.call_count, 1)
+
     @mock.patch("letsencrypt.client.enhancements")
     def test_enhance_config(self, mock_enhancements):
         self.assertRaises(errors.Error,