Use latest version of mypy (#8992)

Fixes #8899

This PR removes the pinning upper limit of mypy currently set to <0.900 and adds the required types-* stub packages to make recent versions of mypy work.

* Unpin mypy

* Improve type in TempHandler

* Add types

certbot-ci/certbot_integration_tests/utils/misc.py

@@ -232,10 +232,15 @@ def generate_csr(domains, key_path, csr_path, key_type=RSA_KEY_TYPE):
         with warnings.catch_warnings():
             # Ignore a warning on some old versions of cryptography
             warnings.simplefilter('ignore', category=PendingDeprecationWarning)
-            key = ec.generate_private_key(ec.SECP384R1(), default_backend())
-        key = key.private_bytes(encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL,
-                                encryption_algorithm=NoEncryption())
-        key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
+            _key = ec.generate_private_key(ec.SECP384R1(), default_backend())
+        # This type ignore directive is required due to an outdated version of types-cryptography.
+        # It can be removed once package types-pyOpenSSL depends on cryptography instead of
+        # types-cryptography and so types-cryptography is not installed anymore.
+        # See https://github.com/python/typeshed/issues/5618
+        _bytes = _key.private_bytes(encoding=Encoding.PEM,  # type: ignore
+                                    format=PrivateFormat.TraditionalOpenSSL,
+                                    encryption_algorithm=NoEncryption())
+        key = crypto.load_privatekey(crypto.FILETYPE_PEM, _bytes)
     else:
         raise ValueError('Invalid key type: {0}'.format(key_type))
 

certbot-ci/setup.py

@@ -1,5 +1,4 @@
 from distutils.version import LooseVersion
-import sys
 
 from setuptools import __version__ as setuptools_version
 from setuptools import find_packages
@@ -30,6 +29,7 @@ install_requires = [
     'pywin32>=300 ; sys_platform == "win32"',
     'pyyaml',
     'requests',
+    'types-python-dateutil'
 ]
 
 setup(

certbot/certbot/_internal/log.py

@@ -29,6 +29,7 @@ import sys
 import tempfile
 import traceback
 from types import TracebackType
+from typing import IO
 
 from acme import messages
 from certbot import errors
@@ -258,6 +259,9 @@ class TempHandler(logging.StreamHandler):
         self.path = os.path.join(self._workdir, 'log')
         stream = util.safe_open(self.path, mode='w', chmod=0o600)
         super().__init__(stream)
+        # Super constructor assigns the provided stream object to self.stream.
+        # Let's help mypy be aware of this by giving a type hint.
+        self.stream: IO[str]
         self._delete = True
 
     def emit(self, record):

certbot/certbot/_internal/storage.py

@@ -1037,21 +1037,21 @@ class RenewableCert(interfaces.RenewableCert):
         archive_target = {kind: os.path.join(archive, kind + "1.pem") for kind in ALL_FOUR}
         for kind in ALL_FOUR:
             os.symlink(_relpath_from_file(archive_target[kind], target[kind]), target[kind])
-        with open(target["cert"], "wb") as f:
+        with open(target["cert"], "wb") as f_b:
             logger.debug("Writing certificate to %s.", target["cert"])
-            f.write(cert)
-        with util.safe_open(archive_target["privkey"], "wb", chmod=BASE_PRIVKEY_MODE) as f:
+            f_b.write(cert)
+        with util.safe_open(archive_target["privkey"], "wb", chmod=BASE_PRIVKEY_MODE) as f_a:
             logger.debug("Writing private key to %s.", target["privkey"])
-            f.write(privkey)
+            f_a.write(privkey)
             # XXX: Let's make sure to get the file permissions right here
-        with open(target["chain"], "wb") as f:
+        with open(target["chain"], "wb") as f_b:
             logger.debug("Writing chain to %s.", target["chain"])
-            f.write(chain)
-        with open(target["fullchain"], "wb") as f:
+            f_b.write(chain)
+        with open(target["fullchain"], "wb") as f_b:
             # assumes that OpenSSL.crypto.dump_certificate includes
             # ending newline character
             logger.debug("Writing full chain to %s.", target["fullchain"])
-            f.write(cert + chain)
+            f_b.write(cert + chain)
 
         # Write a README file to the live directory
         readme_path = os.path.join(live_dir, README)

certbot/certbot/crypto_util.py

@@ -282,7 +282,11 @@ def make_key(bits=1024, key_type="rsa", elliptic_curve=None):
             raise errors.Error("Unsupported elliptic curve: {}".format(elliptic_curve))
         except UnsupportedAlgorithm as e:
             raise e from errors.Error(str(e))
-        _key_pem = _key.private_bytes(
+        # This type ignore directive is required due to an outdated version of types-cryptography.
+        # It can be removed once package types-pyOpenSSL depends on cryptography instead of
+        # types-cryptography and so types-cryptography is not installed anymore.
+        # See https://github.com/python/typeshed/issues/5618
+        _key_pem = _key.private_bytes(  # type: ignore
             encoding=Encoding.PEM,
             format=PrivateFormat.TraditionalOpenSSL,
             encryption_algorithm=NoEncryption()

certbot/certbot/util.py

@@ -13,6 +13,7 @@ import socket
 import subprocess
 import sys
 from typing import Dict
+from typing import IO
 from typing import Text
 from typing import Tuple
 from typing import Union
@@ -207,7 +208,7 @@ def make_or_verify_dir(directory, mode=0o755, strict=False):
             raise
 
 
-def safe_open(path, mode="w", chmod=None):
+def safe_open(path: str, mode: str = "w", chmod=None) -> IO:
     """Safely open a file.
 
     :param str path: Path to a file.

certbot/setup.py

@@ -96,6 +96,13 @@ test_extras = [
     'pytest-xdist',
     'setuptools',
     'tox',
+    'types-mock',
+    'types-pyOpenSSL',
+    'types-pyRFC3339',
+    'types-pytz',
+    'types-requests',
+    'types-setuptools',
+    'types-six',
     # typing-extensions is required to import typing.Protocol and make the mypy checks
     # pass (along with pylint about non-existent objects) on Python 3.6 & 3.7
     'typing-extensions',

tools/pinning/current/pyproject.toml

@@ -51,12 +51,6 @@ awscli = ">=1.19.62"
 # as a dependency here to ensure a version of cython is pinned for extra
 # stability.
 cython = "*"
-# mypy 0.900 stopped including stubs containing type information for 3rd party
-# libraries by default. This breaks our tests so let's continue to pin it back
-# for now. See
-# https://mypy-lang.blogspot.com/2021/05/the-upcoming-switch-to-modular-typeshed.html
-# for more info.
-mypy = "<0.900"
 # We install mock in our "external-mock" tox environment to test that we didn't
 # break Certbot's test API which used to always use mock objects from the 3rd
 # party mock library. We list the mock dependency here so that is pinned, but

tools/requirements.txt

@@ -54,10 +54,10 @@ entrypoints==0.3; python_version >= "3.6" and python_version < "4.0"
 execnet==1.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 fabric==2.6.0; python_version >= "3.6"
 filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_version < "4.0"
-google-api-core==1.31.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-api-core==1.31.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 google-api-python-client==2.15.0; python_version >= "3.6"
 google-auth-httplib2==0.1.0; python_version >= "3.6"
-google-auth==1.34.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
+google-auth==1.35.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0"
 httplib2==0.19.1; python_version >= "3.6"
@@ -91,7 +91,7 @@ mock==4.0.3; python_version >= "3.6"
 msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0"
 msrest==0.6.21; python_version >= "3.6"
 mypy-extensions==0.4.3; python_version >= "3.6"
-mypy==0.812; python_version >= "3.5"
+mypy==0.910; python_version >= "3.6"
 oauth2client==4.1.3; python_version >= "3.6"
 oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
@@ -107,7 +107,7 @@ pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or pyt
 ply==3.11; python_version >= "3.6"
 poetry-core==1.1.0a6; python_version >= "3.6" and python_version < "4.0"
 poetry==1.2.0a2; python_version >= "3.6" and python_version < "4.0"
-prompt-toolkit==3.0.19; python_version == "3.6" and python_full_version >= "3.6.1" or python_version >= "3.7" and python_full_version >= "3.6.1"
+prompt-toolkit==3.0.3; python_version == "3.6" or python_version >= "3.7"
 protobuf==3.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 ptyprocess==0.7.0; python_version >= "3.6" and python_version < "4.0"
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_version >= "3.6" and python_full_version >= "3.5.0"
@@ -170,12 +170,23 @@ tox==3.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python
 tqdm==4.62.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0"
 traitlets==4.3.3
 twine==3.3.0; python_version >= "3.6"
-typed-ast==1.4.3; python_version >= "3.6" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
+typed-ast==1.4.3; python_version >= "3.6" and python_version < "3.8" or implementation_name == "cpython" and python_version < "3.8" and python_version >= "3.6"
+types-cryptography==3.3.5; python_version >= "3.6"
+types-enum34==0.1.8; python_version >= "3.6"
+types-ipaddress==0.1.5; python_version >= "3.6"
+types-mock==0.1.5; python_version >= "3.6"
+types-pyopenssl==20.0.5; python_version >= "3.6"
+types-pyrfc3339==0.1.2; python_version >= "3.6"
+types-python-dateutil==0.1.6; python_version >= "3.6"
+types-pytz==2021.1.2; python_version >= "3.6"
+types-requests==2.25.6; python_version >= "3.6"
+types-setuptools==57.0.2; python_version >= "3.6"
+types-six==1.16.0; python_version >= "3.6"
 typing-extensions==3.10.0.0; python_version >= "3.6" or python_version >= "3.6" and python_version < "3.8"
 uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 urllib3==1.26.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6"
 virtualenv==20.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.4.0" or python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"
-wcwidth==0.2.5; python_version == "3.6" and python_full_version >= "3.6.1"
+wcwidth==0.2.5; python_version == "3.6"
 webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" or python_version >= "3.6"
 websocket-client==0.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" or python_full_version >= "3.5.0" and python_version >= "3.6"
 wheel==0.37.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0"

tox.ini

@@ -132,7 +132,7 @@ commands =
 basepython = python3
 commands =
     win: {[base]pip_install} {[base]win_all_packages}
-    !win: {[base]pip_install} {[base]all_packages}
+    !win: {[base]pip_install} {[base]all_packages} certbot-ci
     mypy {[base]source_paths}
 
 [testenv:apacheconftest]