Lint certbot code on Python 3, and update Pylint to the latest version (#7551)

Part of #7550

This PR makes appropriate corrections to run pylint on Python 3.

Why not keeping the dependencies unchanged and just run pylint on Python 3?
Because the old version of pylint breaks horribly on Python 3 because of unsupported version of astroid.

Why updating pylint + astroid to the latest version ?
Because this version only fixes some internal errors occuring during the lint of Certbot code, and is also ready to run gracefully on Python 3.8.

Why upgrading mypy ?
Because the old version does not support the new version of astroid required to run pylint correctly.

Why not upgrading mypy to its latest version ?
Because this latest version includes a new typshed version, that adds a lot of new type definitions, and brings dozens of new errors on the Certbot codebase. I would like to fix that in a future PR.

That said so, the work has been to find the correct set of new dependency versions, then configure pylint for sane configuration errors in our situation, disable irrelevant lintings errors, then fixing (or ignoring for good reason) the remaining mypy errors.

I also made PyLint and MyPy checks run correctly on Windows.

* Start configuration

* Reconfigure travis

* Suspend a check specific to python 3. Start fixing code.

* Repair call_args

* Fix return + elif lints

* Reconfigure development to run mainly on python3

* Remove incompatible Python 3.4 jobs

* Suspend pylint in some assertions

* Remove pylint in dev

* Take first mypy that supports typed-ast>=1.4.0 to limit the migration path

* Various return + else lint errors

* Find a set of deps that is working with current mypy version

* Update local oldest requirements

* Remove all current pylint errors

* Rebuild letsencrypt-auto

* Update mypy to fix pylint with new astroid version, and fix mypy issues

* Explain type: ignore

* Reconfigure tox, fix none path

* Simplify pinning

* Remove useless directive

* Remove debugging code

* Remove continue

* Update requirements

* Disable unsubscriptable-object check

* Disable one check, enabling two more

* Plug certbot dev version for oldest requirements

* Remove useless disable directives

* Remove useless no-member disable

* Remove no-else-* checks. Use elif in symetric branches.

* Add back assertion

* Add new line

* Remove unused pylint disable

* Remove other pylint disable

acme/acme/challenges.py

@@ -54,8 +54,7 @@ class UnrecognizedChallenge(Challenge):
         object.__setattr__(self, "jobj", jobj)
 
     def to_partial_json(self):
-        # pylint: disable=no-member
-        return self.jobj
+        return self.jobj  # pylint: disable=no-member
 
     @classmethod
     def from_json(cls, jobj):
@@ -113,7 +112,7 @@ class KeyAuthorizationChallengeResponse(ChallengeResponse):
         :rtype: bool
 
         """
-        parts = self.key_authorization.split('.')  # pylint: disable=no-member
+        parts = self.key_authorization.split('.')
         if len(parts) != 2:
             logger.debug("Key authorization (%r) is not well formed",
                          self.key_authorization)

acme/acme/client.py

@@ -34,7 +34,6 @@ logger = logging.getLogger(__name__)
 # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
 if sys.version_info < (2, 7, 9):  # pragma: no cover
     try:
-        # pylint: disable=no-member
         requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()  # type: ignore
     except AttributeError:
         import urllib3.contrib.pyopenssl  # pylint: disable=import-error
@@ -280,7 +279,6 @@ class Client(ClientBase):
         assert response.status_code == http_client.CREATED
 
         # "Instance of 'Field' has no key/contact member" bug:
-        # pylint: disable=no-member
         return self._regr_from_response(response)
 
     def query_registration(self, regr):
@@ -465,7 +463,6 @@ class Client(ClientBase):
             updated[authzr] = updated_authzr
 
             attempts[authzr] += 1
-            # pylint: disable=no-member
             if updated_authzr.body.status not in (
                     messages.STATUS_VALID, messages.STATUS_INVALID):
                 if attempts[authzr] < max_attempts:
@@ -606,7 +603,6 @@ class ClientV2(ClientBase):
         if response.status_code == 200 and 'Location' in response.headers:
             raise errors.ConflictError(response.headers.get('Location'))
         # "Instance of 'Field' has no key/contact member" bug:
-        # pylint: disable=no-member
         regr = self._regr_from_response(response)
         self.net.account = regr
         return regr
@@ -730,7 +726,7 @@ class ClientV2(ClientBase):
         for authzr in responses:
             if authzr.body.status != messages.STATUS_VALID:
                 for chall in authzr.body.challenges:
-                    if chall.error != None:
+                    if chall.error is not None:
                         failed.append(authzr)
         if failed:
             raise errors.ValidationError(failed)
@@ -1125,10 +1121,9 @@ class ClientNetwork(object):
             err_regex = r".*host='(\S*)'.*Max retries exceeded with url\: (\/\w*).*(\[Errno \d+\])([A-Za-z ]*)"
             m = re.match(err_regex, str(e))
             if m is None:
-                raise # pragma: no cover
-            else:
-                host, path, _err_no, err_msg = m.groups()
-                raise ValueError("Requesting {0}{1}:{2}".format(host, path, err_msg))
+                raise  # pragma: no cover
+            host, path, _err_no, err_msg = m.groups()
+            raise ValueError("Requesting {0}{1}:{2}".format(host, path, err_msg))
 
         # If content is DER, log the base64 of it instead of raw bytes, to keep
         # binary data out of the logs.
@@ -1194,8 +1189,7 @@ class ClientNetwork(object):
             if error.code == 'badNonce':
                 logger.debug('Retrying request after error:\n%s', error)
                 return self._post_once(*args, **kwargs)
-            else:
-                raise
+            raise
 
     def _post_once(self, url, obj, content_type=JOSE_CONTENT_TYPE,
             acme_version=1, **kwargs):

acme/acme/errors.py

@@ -29,7 +29,12 @@ class NonceError(ClientError):
 class BadNonce(NonceError):
     """Bad nonce error."""
     def __init__(self, nonce, error, *args, **kwargs):
-        super(BadNonce, self).__init__(*args, **kwargs)
+        # MyPy complains here that there is too many arguments for BaseException constructor.
+        # This is an error fixed in typeshed, see https://github.com/python/mypy/issues/4183
+        # The fix is included in MyPy>=0.740, but upgrading it would bring dozen of errors due to
+        #   new types definitions. So we ignore the error until the code base is fixed to match
+        #   with MyPy>=0.740 referential.
+        super(BadNonce, self).__init__(*args, **kwargs)  # type: ignore
         self.nonce = nonce
         self.error = error
 
@@ -48,7 +53,8 @@ class MissingNonce(NonceError):
 
     """
     def __init__(self, response, *args, **kwargs):
-        super(MissingNonce, self).__init__(*args, **kwargs)
+        # See comment in BadNonce constructor above for an explanation of type: ignore here.
+        super(MissingNonce, self).__init__(*args, **kwargs)  # type: ignore
         self.response = response
 
     def __str__(self):
@@ -83,6 +89,7 @@ class PollError(ClientError):
         return '{0}(exhausted={1!r}, updated={2!r})'.format(
             self.__class__.__name__, self.exhausted, self.updated)
 
+
 class ValidationError(Error):
     """Error for authorization failures. Contains a list of authorization
     resources, each of which is invalid and should have an error field.
@@ -91,9 +98,11 @@ class ValidationError(Error):
         self.failed_authzrs = failed_authzrs
         super(ValidationError, self).__init__()
 
-class TimeoutError(Error):
+
+class TimeoutError(Error):  # pylint: disable=redefined-builtin
     """Error for when polling an authorization or an order times out."""
 
+
 class IssuanceError(Error):
     """Error sent by the server after requesting issuance of a certificate."""
 
@@ -105,6 +114,7 @@ class IssuanceError(Error):
         self.error = error
         super(IssuanceError, self).__init__()
 
+
 class ConflictError(ClientError):
     """Error for when the server returns a 409 (Conflict) HTTP status.
 

acme/acme/jws.py

@@ -40,7 +40,7 @@ class Signature(jose.Signature):
 class JWS(jose.JWS):
     """ACME-specific JWS. Includes none, url, and kid in protected header."""
     signature_cls = Signature
-    __slots__ = jose.JWS._orig_slots  # pylint: disable=no-member
+    __slots__ = jose.JWS._orig_slots
 
     @classmethod
     # pylint: disable=arguments-differ

acme/acme/messages.py

@@ -146,7 +146,7 @@ class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
         if jobj not in cls.POSSIBLE_NAMES:  # pylint: disable=unsupported-membership-test
             raise jose.DeserializationError(
                 '{0} not recognized'.format(cls.__name__))
-        return cls.POSSIBLE_NAMES[jobj]  # pylint: disable=unsubscriptable-object
+        return cls.POSSIBLE_NAMES[jobj]
 
     def __repr__(self):
         return '{0}({1})'.format(self.__class__.__name__, self.name)

acme/acme/standalone.py

@@ -44,7 +44,7 @@ class TLSServer(socketserver.TCPServer):
         return socketserver.TCPServer.server_bind(self)
 
 
-class ACMEServerMixin:  # pylint: disable=old-style-class
+class ACMEServerMixin:
     """ACME server common settings mixin."""
     # TODO: c.f. #858
     server_version = "ACME client standalone challenge solver"
@@ -105,7 +105,6 @@ class BaseDualNetworkedServers(object):
         """Wraps socketserver.TCPServer.serve_forever"""
         for server in self.servers:
             thread = threading.Thread(
-                # pylint: disable=no-member
                 target=server.serve_forever)
             thread.start()
             self.threads.append(thread)

acme/tests/challenges_test.py

@@ -4,7 +4,7 @@ import unittest
 import josepy as jose
 import mock
 import requests
-from six.moves.urllib import parse as urllib_parse  # pylint: disable=relative-import
+from six.moves.urllib import parse as urllib_parse
 
 import test_util
 
@@ -18,7 +18,6 @@ class ChallengeTest(unittest.TestCase):
         from acme.challenges import Challenge
         from acme.challenges import UnrecognizedChallenge
         chall = UnrecognizedChallenge({"type": "foo"})
-        # pylint: disable=no-member
         self.assertEqual(chall, Challenge.from_json(chall.jobj))
 
 

acme/tests/client_test.py

@@ -61,7 +61,7 @@ class ClientTestBase(unittest.TestCase):
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
             contact=self.contact, key=KEY.public_key())
-        the_arg = dict(reg) # type: Dict
+        the_arg = dict(reg)  # type: Dict
         self.new_reg = messages.NewRegistration(**the_arg)
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1')
@@ -963,8 +963,8 @@ class ClientNetworkTest(unittest.TestCase):
 
     def test_check_response_not_ok_jobj_error(self):
         self.response.ok = False
-        self.response.json.return_value = messages.Error(
-            detail='foo', typ='serverInternal', title='some title').to_json()
+        self.response.json.return_value = messages.Error.with_code(
+            'serverInternal', detail='foo', title='some title').to_json()
         # pylint: disable=protected-access
         self.assertRaises(
             messages.Error, self.net._check_response, self.response)
@@ -989,7 +989,7 @@ class ClientNetworkTest(unittest.TestCase):
         self.response.json.side_effect = ValueError
         for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']:
             self.response.headers['Content-Type'] = response_ct
-            # pylint: disable=protected-access,no-value-for-parameter
+            # pylint: disable=protected-access
             self.assertEqual(
                 self.response, self.net._check_response(self.response))
 
@@ -1003,7 +1003,7 @@ class ClientNetworkTest(unittest.TestCase):
         self.response.json.return_value = {}
         for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']:
             self.response.headers['Content-Type'] = response_ct
-            # pylint: disable=protected-access,no-value-for-parameter
+            # pylint: disable=protected-access
             self.assertEqual(
                 self.response, self.net._check_response(self.response))
 
@@ -1128,8 +1128,8 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
         self.response.headers = {}
         self.response.links = {}
         self.response.checked = False
-        self.acmev1_nonce_response = mock.MagicMock(ok=False,
-            status_code=http_client.METHOD_NOT_ALLOWED)
+        self.acmev1_nonce_response = mock.MagicMock(
+            ok=False, status_code=http_client.METHOD_NOT_ALLOWED)
         self.acmev1_nonce_response.headers = {}
         self.obj = mock.MagicMock()
         self.wrapped_obj = mock.MagicMock()

acme/tests/crypto_util_test.py

@@ -38,7 +38,6 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
         self.server = _TestServer(('', 0), socketserver.BaseRequestHandler)
         self.port = self.server.socket.getsockname()[1]
         self.server_thread = threading.Thread(
-            # pylint: disable=no-member
             target=self.server.handle_request)
 
     def tearDown(self):
@@ -65,7 +64,7 @@ class SSLSocketAndProbeSNITest(unittest.TestCase):
 
     def test_probe_connection_error(self):
         # pylint has a hard time with six
-        self.server.server_close()  # pylint: disable=no-member
+        self.server.server_close()
         original_timeout = socket.getdefaulttimeout()
         try:
             socket.setdefaulttimeout(1)

acme/tests/jose_test.py

@@ -21,11 +21,10 @@ class JoseTest(unittest.TestCase):
 
         # We use the imports below with eval, but pylint doesn't
         # understand that.
-        # pylint: disable=eval-used,unused-variable
-        import acme
-        import josepy
-        acme_jose_mod = eval(acme_jose_path)
-        josepy_mod = eval(josepy_path)
+        import acme  # pylint: disable=unused-import
+        import josepy  # pylint: disable=unused-import
+        acme_jose_mod = eval(acme_jose_path)  # pylint: disable=eval-used
+        josepy_mod = eval(josepy_path)  # pylint: disable=eval-used
         self.assertIs(acme_jose_mod, josepy_mod)
         self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute))
 

acme/tests/messages_test.py

@@ -18,8 +18,7 @@ class ErrorTest(unittest.TestCase):
 
     def setUp(self):
         from acme.messages import Error, ERROR_PREFIX
-        self.error = Error(
-            detail='foo', typ=ERROR_PREFIX + 'malformed', title='title')
+        self.error = Error.with_code('malformed', detail='foo', title='title')
         self.jobj = {
             'detail': 'foo',
             'title': 'some title',
@@ -27,7 +26,6 @@ class ErrorTest(unittest.TestCase):
         }
         self.error_custom = Error(typ='custom', detail='bar')
         self.empty_error = Error()
-        self.jobj_custom = {'type': 'custom', 'detail': 'bar'}
 
     def test_default_typ(self):
         from acme.messages import Error
@@ -42,8 +40,7 @@ class ErrorTest(unittest.TestCase):
         hash(Error.from_json(self.error.to_json()))
 
     def test_description(self):
-        self.assertEqual(
-            'The request message was malformed', self.error.description)
+        self.assertEqual('The request message was malformed', self.error.description)
         self.assertTrue(self.error_custom.description is None)
 
     def test_code(self):
@@ -53,17 +50,17 @@ class ErrorTest(unittest.TestCase):
         self.assertEqual(None, Error().code)
 
     def test_is_acme_error(self):
-        from acme.messages import is_acme_error
+        from acme.messages import is_acme_error, Error
         self.assertTrue(is_acme_error(self.error))
         self.assertFalse(is_acme_error(self.error_custom))
+        self.assertFalse(is_acme_error(Error()))
         self.assertFalse(is_acme_error(self.empty_error))
         self.assertFalse(is_acme_error("must pet all the {dogs|rabbits}"))
 
     def test_unicode_error(self):
-        from acme.messages import Error, ERROR_PREFIX, is_acme_error
-        arabic_error = Error(
-                detail=u'\u0639\u062f\u0627\u0644\u0629', typ=ERROR_PREFIX + 'malformed',
-            title='title')
+        from acme.messages import Error, is_acme_error
+        arabic_error = Error.with_code(
+            'malformed', detail=u'\u0639\u062f\u0627\u0644\u0629', title='title')
         self.assertTrue(is_acme_error(arabic_error))
 
     def test_with_code(self):
@@ -304,8 +301,7 @@ class ChallengeBodyTest(unittest.TestCase):
         from acme.messages import Error
         from acme.messages import STATUS_INVALID
         self.status = STATUS_INVALID
-        error = Error(typ='urn:ietf:params:acme:error:serverInternal',
-                      detail='Unable to communicate with DNS server')
+        error = Error.with_code('serverInternal', detail='Unable to communicate with DNS server')
         self.challb = ChallengeBody(
             uri='http://challb', chall=self.chall, status=self.status,
             error=error)

acme/tests/test_util.py

@@ -25,8 +25,7 @@ def _guess_loader(filename, loader_pem, loader_der):
         return loader_pem
     elif ext.lower() == '.der':
         return loader_der
-    else:  # pragma: no cover
-        raise ValueError("Loader could not be recognized based on extension")
+    raise ValueError("Loader could not be recognized based on extension")  # pragma: no cover
 
 
 def load_cert(*names):