Add --use-pep517 flag to pip to silence warning in tools/venv.py, and switch codebase to src-layout (#10249)

Fixes #10252.

See further discussion here: https://github.com/pypa/pip/issues/11457

We are doing option:

> Alternatively, enable the --use-pep517 pip option, possibly with
--no-build-isolation. The --use-pip517 flag will force pip to use the
modern mechanism for editable installs. --no-build-isolation may be
needed if your project has build-time requirements beyond setuptools and
wheel. By passing this flag, you are responsible for making sure your
environment already has the required dependencies to build your package.
Once the legacy mechanism is removed, --use-pep517 will have no effect
and will essentially be enabled by default in this context.

Major changes made here include:
- Add `--use-pep517` to use the modern mechanism, which will be the only
mechanism in future pip releases
- Change to `/src` layout to appease mypy, and because for editable
installs that really is the normal way these days.
  - `cd acme && mkdir src && mv acme src/` etc.
- add `where='src'` argument to `find_packages` and add
`package_dir={'': 'src'},` in `setup.py`s
  - update `MANIFEST.in` files with new path locations 
- Update our many hardcoded filepaths
- Update `importlib-metadata` requirement to fix
double-plugin-entry-point problem in oldest tests

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -155,5 +155,5 @@ jobs:
         displayName: Prepare Certbot-CI
       - script: |
           set -e
-          sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
+          sudo -E venv/bin/pytest certbot-ci/src/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps

.isort.cfg

@@ -4,4 +4,4 @@ force_sort_within_sections=True
 force_single_line=True
 order_by_type=False
 line_length=400
-src_paths=acme/acme,acme/tests,certbot*/certbot*,certbot*/tests
+src_paths=acme/src,acme/tests,certbot*/tests,certbot/src,certbot*/src/certbot*

acme/MANIFEST.in

@@ -3,7 +3,7 @@ include README.rst
 include pytest.ini
 recursive-include docs *
 recursive-include examples *
-recursive-include acme/_internal/tests/testdata *
+recursive-include src/acme/_internal/tests/testdata *
-include acme/py.typed
+include src/acme/py.typed
 global-exclude __pycache__
 global-exclude *.py[cod]

acme/setup.py

@@ -49,7 +49,8 @@ setup(
         'Topic :: Security',
     ],
 
-    packages=find_packages(),
+    packages=find_packages(where='src'),
+    package_dir={'': 'src'},
     include_package_data=True,
     install_requires=install_requires,
     extras_require={

certbot-apache/MANIFEST.in

@@ -1,8 +1,8 @@
 include LICENSE.txt
 include README.rst
-recursive-include certbot_apache/_internal/augeas_lens *.aug
+recursive-include src/certbot_apache/_internal/augeas_lens *.aug
-recursive-include certbot_apache/_internal/tls_configs *.conf
+recursive-include src/certbot_apache/_internal/tls_configs *.conf
-recursive-include certbot_apache/_internal/tests/testdata *
+recursive-include src/certbot_apache/_internal/tests/testdata *
-include certbot_apache/py.typed
+include src/certbot_apache/py.typed
 global-exclude __pycache__
 global-exclude *.py[cod]

certbot-apache/setup.py

@@ -50,7 +50,8 @@ setup(
         'Topic :: Utilities',
     ],
 
-    packages=find_packages(),
+    packages=find_packages(where='src'),
+    package_dir={'': 'src'},
     include_package_data=True,
     install_requires=install_requires,
     extras_require={