Update Certbot dependency to Lexicon to 3.x (#6593)

This PR updates Lexicon dependency to the latest version available, 3.0.6, for every lexicon-based DNS plugins. It updates also the provider construction to use the new ConfigResolverobject, and to remove the legacy configuration process.

CHANGELOG.md

@@ -11,7 +11,8 @@ Certbot adheres to [Semantic Versioning](https://semver.org/).
 
 ### Changed
 
-*
+* Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support
+  on 2.x branch is maintained).
 
 ### Fixed
 
@@ -24,6 +25,15 @@ package with changes other than its version number was:
 
 * acme
 * certbot
+* certbot-dns-cloudxns
+* certbot-dns-dnsimple
+* certbot-dns-dnsmadeeasy
+* certbot-dns-gehirn
+* certbot-dns-linode
+* certbot-dns-luadns
+* certbot-dns-nsone
+* certbot-dns-ovh
+* certbot-dns-sakuracloud
 
 More details about these changes can be found on our GitHub repo.
 

certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py

@@ -69,13 +69,15 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_key, secret_key, ttl):
         super(_CloudXNSLexiconClient, self).__init__()
 
-        self.provider = cloudxns.Provider({
+        config = dns_common_lexicon.build_lexicon_config('cloudxns', {
-            'provider_name': 'cloudxns',
+            'ttl': ttl,
+        }, {
             'auth_username': api_key,
             'auth_token': secret_key,
-            'ttl': ttl,
         })
 
+        self.provider = cloudxns.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         hint = None
         if str(e).startswith('400 Client Error:'):

certbot-dns-cloudxns/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-cloudxns/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py

@@ -65,12 +65,14 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, token, ttl):
         super(_DNSimpleLexiconClient, self).__init__()
 
-        self.provider = dnsimple.Provider({
+        config = dns_common_lexicon.build_lexicon_config('dnssimple', {
-            'provider_name': 'dnssimple',
-            'auth_token': token,
             'ttl': ttl,
+        }, {
+            'auth_token': token,
         })
 
+        self.provider = dnsimple.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         hint = None
         if str(e).startswith('401 Client Error: Unauthorized for url:'):

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -1,2 +1,3 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]
+dns-lexicon==2.2.1

certbot-dns-dnsimple/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py

@@ -71,13 +71,15 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_key, secret_key, ttl):
         super(_DNSMadeEasyLexiconClient, self).__init__()
 
-        self.provider = dnsmadeeasy.Provider({
+        config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', {
-            'provider_name': 'dnsmadeeasy',
+            'ttl': ttl,
+        }, {
             'auth_username': api_key,
             'auth_token': secret_key,
-            'ttl': ttl,
         })
 
+        self.provider = dnsmadeeasy.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         if domain_name in str(e) and str(e).startswith('404 Client Error: Not Found for url:'):
             return

certbot-dns-dnsmadeeasy/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-dnsmadeeasy/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py

@@ -72,13 +72,15 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_token, api_secret, ttl):
         super(_GehirnLexiconClient, self).__init__()
 
-        self.provider = gehirn.Provider({
+        config = dns_common_lexicon.build_lexicon_config('gehirn', {
-            'provider_name': 'gehirn',
+            'ttl': ttl,
+        }, {
             'auth_token': api_token,
             'auth_secret': api_secret,
-            'ttl': ttl,
         })
 
+        self.provider = gehirn.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')):
             return  # Expected errors when zone name guess is wrong

certbot-dns-gehirn/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+-e acme[dev]
+-e .[dev]

certbot-dns-gehirn/setup.py

@@ -6,8 +6,8 @@ version = '0.31.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
     'dns-lexicon>=2.1.22',
     'mock',
     'setuptools',

certbot-dns-linode/certbot_dns_linode/dns_linode.py

@@ -54,6 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator):
     def _get_linode_client(self):
         return _LinodeLexiconClient(self.credentials.conf('key'))
 
+
 class _LinodeLexiconClient(dns_common_lexicon.LexiconClient):
     """
     Encapsulates all communication with the Linode API.
@@ -61,11 +62,13 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient):
 
     def __init__(self, api_key):
         super(_LinodeLexiconClient, self).__init__()
-        self.provider = linode.Provider({
+
-            'provider_name': 'linode',
+        config = dns_common_lexicon.build_lexicon_config('linode', {}, {
-            'auth_token': api_key
+            'auth_token': api_key,
         })
 
+        self.provider = linode.Provider(config)
+
     def _handle_general_error(self, e, domain_name):
         if not str(e).startswith('Domain not found'):
             return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}'

certbot-dns-linode/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-linode/setup.py

@@ -5,8 +5,8 @@ version = '0.31.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
     'dns-lexicon>=2.2.1',
     'mock',
     'setuptools',

certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py

@@ -68,13 +68,15 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, email, token, ttl):
         super(_LuaDNSLexiconClient, self).__init__()
 
-        self.provider = luadns.Provider({
+        config = dns_common_lexicon.build_lexicon_config('luadns', {
-            'provider_name': 'luadns',
+            'ttl': ttl,
+        }, {
             'auth_username': email,
             'auth_token': token,
-            'ttl': ttl,
         })
 
+        self.provider = luadns.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         hint = None
         if str(e).startswith('401 Client Error: Unauthorized for url:'):

certbot-dns-luadns/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-luadns/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py

@@ -65,12 +65,14 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_key, ttl):
         super(_NS1LexiconClient, self).__init__()
 
-        self.provider = nsone.Provider({
+        config = dns_common_lexicon.build_lexicon_config('nsone', {
-            'provider_name': 'nsone',
-            'auth_token': api_key,
             'ttl': ttl,
+        }, {
+            'auth_token': api_key,
         })
 
+        self.provider = nsone.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:') or \
                                       str(e).startswith("400 Client Error: Bad Request for url:")):

certbot-dns-nsone/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-nsone/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.2.1', # Support for >1 TXT record per name
+    'dns-lexicon>=2.2.1',  # Support for >1 TXT record per name
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py

@@ -77,15 +77,17 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl):
         super(_OVHLexiconClient, self).__init__()
 
-        self.provider = ovh.Provider({
+        config = dns_common_lexicon.build_lexicon_config('ovh', {
-            'provider_name': 'ovh',
+            'ttl': ttl,
+        }, {
             'auth_entrypoint': endpoint,
             'auth_application_key': application_key,
             'auth_application_secret': application_secret,
             'auth_consumer_key': consumer_key,
-            'ttl': ttl,
         })
 
+        self.provider = ovh.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         hint = None
         if str(e).startswith('400 Client Error:'):

certbot-dns-ovh/local-oldest-requirements.txt

@@ -1,2 +1,2 @@
-acme[dev]==0.21.1
+-e acme[dev]
-certbot[dev]==0.21.1
+-e .[dev]

certbot-dns-ovh/setup.py

@@ -7,9 +7,9 @@ version = '0.31.0.dev0'
 # Remember to update local-oldest-requirements.txt when changing the minimum
 # acme/certbot version.
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
-    'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider
+    'dns-lexicon>=2.7.14',  # Correct proxy use on OVH provider
     'mock',
     'setuptools',
     'zope.interface',

certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py

@@ -75,13 +75,15 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient):
     def __init__(self, api_token, api_secret, ttl):
         super(_SakuraCloudLexiconClient, self).__init__()
 
-        self.provider = sakuracloud.Provider({
+        config = dns_common_lexicon.build_lexicon_config('sakuracloud', {
-            'provider_name': 'sakuracloud',
+            'ttl': ttl,
+        }, {
             'auth_token': api_token,
             'auth_secret': api_secret,
-            'ttl': ttl,
         })
 
+        self.provider = sakuracloud.Provider(config)
+
     def _handle_http_error(self, e, domain_name):
         if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')):
             return  # Expected errors when zone name guess is wrong

certbot-dns-sakuracloud/local-oldest-requirements.txt

@@ -0,0 +1,2 @@
+-e acme[dev]
+-e .[dev]

certbot-dns-sakuracloud/setup.py

@@ -6,8 +6,8 @@ version = '0.31.0.dev0'
 
 # Please update tox.ini when modifying dependency version requirements
 install_requires = [
-    'acme>=0.21.1',
+    'acme>=0.31.0.dev0',
-    'certbot>=0.21.1',
+    'certbot>=0.31.0.dev0',
     'dns-lexicon>=2.1.23',
     'mock',
     'setuptools',

certbot/plugins/dns_common_lexicon.py

@@ -1,12 +1,22 @@
 """Common code for DNS Authenticator Plugins built on Lexicon."""
-
 import logging
 
 from requests.exceptions import HTTPError, RequestException
 
+from acme.magic_typing import Union, Dict, Any  # pylint: disable=unused-import,no-name-in-module
 from certbot import errors
 from certbot.plugins import dns_common
 
+# Lexicon is not declared as a dependency in Certbot itself,
+# but in the Certbot plugins backed by Lexicon.
+# So we catch import error here to allow this module to be
+# always importable, even if it does not make sense to use it
+# if Lexicon is not available, obviously.
+try:
+    from lexicon.config import ConfigResolver
+except ImportError:
+    ConfigResolver = None  # type: ignore
+
 logger = logging.getLogger(__name__)
 
 
@@ -100,3 +110,28 @@ class LexiconClient(object):
         if not str(e).startswith('No domain found'):
             return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}'
                                       .format(domain_name, e))
+
+
+def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options):
+    # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict]
+    """
+    Convenient function to build a Lexicon 2.x/3.x config object.
+    :param str lexicon_provider_name: the name of the lexicon provider to use
+    :param dict lexicon_options: options specific to lexicon
+    :param dict provider_options: options specific to provider
+    :return: configuration to apply to the provider
+    :rtype: ConfigurationResolver or dict
+    """
+    config = {'provider_name': lexicon_provider_name}  # type: Dict[str, Any]
+    config.update(lexicon_options)
+    if not ConfigResolver:
+        # Lexicon 2.x
+        config.update(provider_options)
+    else:
+        # Lexicon 3.x
+        provider_config = {}
+        provider_config.update(provider_options)
+        config[lexicon_provider_name] = provider_config
+        config = ConfigResolver().with_dict(config).with_env()
+
+    return config

tools/dev_constraints.txt

@@ -12,7 +12,7 @@ botocore==1.12.36
 cloudflare==1.5.1
 coverage==4.4.2
 decorator==4.1.2
-dns-lexicon==2.7.14
+dns-lexicon==3.0.8
 dnspython==1.15.0
 docutils==0.12
 execnet==1.5.0

tools/oldest_constraints.txt

@@ -50,6 +50,12 @@ ConfigArgParse==0.10.0
 funcsigs==0.4
 zope.hookable==4.0.4
 
+# Ubuntu Bionic constraints
+# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some
+# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure
+# that Lexicon 2.x works with Certbot.
+dns-lexicon==2.7.14
+
 # Plugin constraints
 # These aren't necessarily the oldest versions we need to support
 # Tracking at https://github.com/certbot/certbot/issues/6473