Add --use-pep517 flag to pip to silence warning in tools/venv.py, and switch codebase to src-layout (#10249)

Fixes #10252.

See further discussion here: https://github.com/pypa/pip/issues/11457

We are doing option:

> Alternatively, enable the --use-pep517 pip option, possibly with
--no-build-isolation. The --use-pip517 flag will force pip to use the
modern mechanism for editable installs. --no-build-isolation may be
needed if your project has build-time requirements beyond setuptools and
wheel. By passing this flag, you are responsible for making sure your
environment already has the required dependencies to build your package.
Once the legacy mechanism is removed, --use-pep517 will have no effect
and will essentially be enabled by default in this context.

Major changes made here include:
- Add `--use-pep517` to use the modern mechanism, which will be the only
mechanism in future pip releases
- Change to `/src` layout to appease mypy, and because for editable
installs that really is the normal way these days.
  - `cd acme && mkdir src && mv acme src/` etc.
- add `where='src'` argument to `find_packages` and add
`package_dir={'': 'src'},` in `setup.py`s
  - update `MANIFEST.in` files with new path locations 
- Update our many hardcoded filepaths
- Update `importlib-metadata` requirement to fix
double-plugin-entry-point problem in oldest tests

certbot-dns-nsone/src/certbot_dns_nsone/__init__.py

@@ -0,0 +1,89 @@
+"""
+The `~certbot_dns_nsone.dns_nsone` plugin automates the process of completing
+a ``dns-01`` challenge (`~acme.challenges.DNS01`) by creating, and subsequently
+removing, TXT records using the NS1 API.
+
+.. note::
+   The plugin is not installed by default. It can be installed by heading to
+   `certbot.eff.org <https://certbot.eff.org/instructions#wildcard>`_, choosing your system and
+   selecting the Wildcard tab.
+
+Named Arguments
+---------------
+
+========================================  =====================================
+``--dns-nsone-credentials``               NS1 credentials_ INI file.
+                                          (Required)
+``--dns-nsone-propagation-seconds``       The number of seconds to wait for DNS
+                                          to propagate before asking the ACME
+                                          server to verify the DNS record.
+                                          (Default: 30)
+========================================  =====================================
+
+
+Credentials
+-----------
+
+Use of this plugin requires a configuration file containing NS1 API credentials,
+obtained from your NS1
+`account page <https://my.nsone.net/#/account/settings>`_.
+
+.. code-block:: ini
+   :name: credentials.ini
+   :caption: Example credentials file:
+
+   # NS1 API credentials used by Certbot
+   dns_nsone_api_key = MDAwMDAwMDAwMDAwMDAw
+
+The path to this file can be provided interactively or using the
+``--dns-nsone-credentials`` command-line argument. Certbot records the path
+to this file for use during renewal, but does not store the file's contents.
+
+.. caution::
+   You should protect these API credentials as you would the password to your
+   NS1 account. Users who can read this file can use these credentials to issue
+   arbitrary API calls on your behalf. Users who can cause Certbot to run using
+   these credentials can complete a ``dns-01`` challenge to acquire new
+   certificates or revoke existing certificates for associated domains, even if
+   those domains aren't being managed by this server.
+
+Certbot will emit a warning if it detects that the credentials file can be
+accessed by other users on your system. The warning reads "Unsafe permissions
+on credentials configuration file", followed by the path to the credentials
+file. This warning will be emitted each time Certbot uses the credentials file,
+including for renewal, and cannot be silenced except by addressing the issue
+(e.g., by using a command like ``chmod 600`` to restrict access to the file).
+
+
+Examples
+--------
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``
+
+   certbot certonly \\
+     --dns-nsone \\
+     --dns-nsone-credentials ~/.secrets/certbot/nsone.ini \\
+     -d example.com
+
+.. code-block:: bash
+   :caption: To acquire a single certificate for both ``example.com`` and
+             ``www.example.com``
+
+   certbot certonly \\
+     --dns-nsone \\
+     --dns-nsone-credentials ~/.secrets/certbot/nsone.ini \\
+     -d example.com \\
+     -d www.example.com
+
+.. code-block:: bash
+   :caption: To acquire a certificate for ``example.com``, waiting 60 seconds
+             for DNS propagation
+
+   certbot certonly \\
+     --dns-nsone \\
+     --dns-nsone-credentials ~/.secrets/certbot/nsone.ini \\
+     --dns-nsone-propagation-seconds 60 \\
+     -d example.com
+
+"""

certbot-dns-nsone/src/certbot_dns_nsone/_internal/__init__.py

@@ -0,0 +1 @@
+"""Internal implementation of `~certbot_dns_nsone.dns_nsone` plugin."""

certbot-dns-nsone/src/certbot_dns_nsone/_internal/dns_nsone.py

@@ -0,0 +1,55 @@
+"""DNS Authenticator for NS1 DNS."""
+import logging
+from typing import Any
+from typing import Callable
+from typing import Optional
+
+from requests import HTTPError
+
+from certbot import errors
+from certbot.plugins import dns_common_lexicon
+
+logger = logging.getLogger(__name__)
+
+ACCOUNT_URL = 'https://my.nsone.net/#/account/settings'
+
+
+class Authenticator(dns_common_lexicon.LexiconDNSAuthenticator):
+    """
+    DNS Authenticator for NS1
+    This Authenticator uses the NS1 API to fulfill a dns-01 challenge.
+    """
+
+    description = 'Obtain certificates using a DNS TXT record (if you are using NS1 for DNS).'
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._add_provider_option('api-key',
+                                  f'API key for NS1 API, obtained from {ACCOUNT_URL}',
+                                  'auth_token')
+
+    @classmethod
+    def add_parser_arguments(cls, add: Callable[..., None],
+                             default_propagation_seconds: int = 30) -> None:
+        super().add_parser_arguments(add, default_propagation_seconds)
+        add('credentials', help='NS1 credentials file.')
+
+    def more_info(self) -> str:
+        return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \
+               'the NS1 API.'
+
+    @property
+    def _provider_name(self) -> str:
+        return 'nsone'
+
+    def _handle_http_error(self, e: HTTPError, domain_name: str) -> Optional[errors.PluginError]:
+        if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:') or
+                                      str(e).startswith("400 Client Error: Bad Request for url:")):
+            return None  # Expected errors when zone name guess is wrong
+        hint = None
+        if str(e).startswith('401 Client Error: Unauthorized for url:'):
+            hint = 'Is your API key correct?'
+
+        hint_disp = f' ({hint})' if hint else ''
+
+        return errors.PluginError(f'Error determining zone identifier: {e}.{hint_disp}')

certbot-dns-nsone/src/certbot_dns_nsone/_internal/tests/__init__.py

@@ -0,0 +1 @@
+"""certbot-dns-nsone tests"""

certbot-dns-nsone/src/certbot_dns_nsone/_internal/tests/dns_nsone_test.py

@@ -0,0 +1,39 @@
+"""Tests for certbot_dns_nsone._internal.dns_nsone."""
+import sys
+from unittest import mock
+
+import pytest
+from requests import Response
+from requests.exceptions import HTTPError
+
+from certbot.compat import os
+from certbot.plugins import dns_test_common
+from certbot.plugins import dns_test_common_lexicon
+from certbot.plugins.dns_test_common import DOMAIN
+from certbot.tests import util as test_util
+
+API_KEY = 'foo'
+
+
+class AuthenticatorTest(test_util.TempDirTestCase,
+                        dns_test_common_lexicon.BaseLexiconDNSAuthenticatorTest):
+
+    DOMAIN_NOT_FOUND = HTTPError(f'404 Client Error: Not Found for url: {DOMAIN}.', response=Response())
+    LOGIN_ERROR = HTTPError(f'401 Client Error: Unauthorized for url: {DOMAIN}.', response=Response())
+
+    def setUp(self):
+        super().setUp()
+
+        from certbot_dns_nsone._internal.dns_nsone import Authenticator
+
+        path = os.path.join(self.tempdir, 'file.ini')
+        dns_test_common.write({"nsone_api_key": API_KEY}, path)
+
+        self.config = mock.MagicMock(nsone_credentials=path,
+                                     nsone_propagation_seconds=0)  # don't wait during tests
+
+        self.auth = Authenticator(self.config, "nsone")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(sys.argv[1:] + [__file__]))  # pragma: no cover