From c3c587001faa516ee375ad52cb95cbdecacb4a25 Mon Sep 17 00:00:00 2001 From: Will Greenberg Date: Thu, 8 Aug 2024 16:24:11 -0700 Subject: [PATCH] Update python version to 3.12 and base to core24 in snaps (#9983) Fixes #9872, originally merged in #9956. To upgrade to python3.12 as 3.8 is reaching EOL, we need to upgrade the core snap that certbot is based on. The latest version is core24, so we're going with that for longevity. We will want to notify third party snaps to make changes as well. They can release their snaps to a version higher than certbot's, and their users will not be upgraded until the matching (or greater) version of certbot is released. They should do this as otherwise including these changes will break their plugins. Key documents for this migration are https://snapcraft.io/docs/migrate-core22 and https://snapcraft.io/docs/migrate-core24. The discussion at https://forum.snapcraft.io/t/upgrading-classic-snap-to-core24-using-snapcraft-8-3-causes-python-3-12-errors-at-runtime/ is also relevant to understanding some changes, which may become unnecessary in future versions of snapcraft. * Migrate primary certbot snap to core24 and python 3.12 * Migrate plugin snaps to core24 and python 3.12 * Migrate to core24 in build_remote * Run snap tests using python 3.12 * Unstage pyvenv.cfg and set PYTHONPATH --------- Co-authored-by: Erica Portnoy Co-authored-by: Erica Portnoy --- .../templates/jobs/packaging-jobs.yml | 6 +- certbot/certbot/_internal/snap_config.py | 2 +- certbot/docs/contributing.rst | 4 +- snap/snapcraft.yaml | 39 ++++++------ tools/snap/README.md | 2 +- tools/snap/build_remote.py | 60 ++++++++++--------- tools/snap/generate_dnsplugins_snapcraft.sh | 10 ++-- 7 files changed, 67 insertions(+), 56 deletions(-) diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml index cdf8393c2..ebf6b44e5 100644 --- a/.azure-pipelines/templates/jobs/packaging-jobs.yml +++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml @@ -76,7 +76,7 @@ jobs: displayName: Install dependencies - task: UsePythonVersion@0 inputs: - versionSpec: 3.8 + versionSpec: 3.12 addToPath: true - task: DownloadSecureFile@1 name: credentials @@ -107,7 +107,7 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: 3.8 + versionSpec: 3.12 addToPath: true - script: | set -e @@ -141,7 +141,7 @@ jobs: displayName: Install dependencies - task: UsePythonVersion@0 inputs: - versionSpec: 3.8 + versionSpec: 3.12 addToPath: true - task: DownloadPipelineArtifact@2 inputs: diff --git a/certbot/certbot/_internal/snap_config.py b/certbot/certbot/_internal/snap_config.py index 62f0043fa..e07ee6c08 100644 --- a/certbot/certbot/_internal/snap_config.py +++ b/certbot/certbot/_internal/snap_config.py @@ -71,7 +71,7 @@ def prepare_env(cli_args: List[str]) -> List[str]: raise e data = response.json() - connections = ['/snap/{0}/current/lib/python3.8/site-packages/'.format(item['slot']['snap']) + connections = ['/snap/{0}/current/lib/python3.12/site-packages/'.format(item['slot']['snap']) for item in data.get('result', {}).get('established', []) if item.get('plug', {}).get('plug') == 'plugin' and item.get('plug-attrs', {}).get('content') == 'certbot-1'] diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 3b6ad8649..7122f77eb 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -376,8 +376,8 @@ Certbot plugin snaps expose their Python modules to the Certbot snap via a `snap content interface`_ where ``certbot-1`` is the value for the ``content`` attribute. The Certbot snap only uses this to find the names of connected plugin snaps and it expects to find the Python modules to be loaded under -``lib/python3.8/site-packages/`` in the plugin snap. This location is the -default when using the ``core20`` `base snap`_ and the `python snapcraft +``lib/python3.12/site-packages/`` in the plugin snap. This location is the +default when using the ``core24`` `base snap`_ and the `python snapcraft plugin`_. The Certbot snap also provides a separate content interface which diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 5fdde3124..cc811632a 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -14,10 +14,13 @@ description: | - Keep track of when your certificate is going to expire, and renew it - Help you revoke the certificate if that ever becomes necessary. confinement: classic -base: core20 +base: core24 grade: stable adopt-info: certbot +environment: + PYTHONPATH: "$SNAP/lib/python3.12/site-packages:${PYTHONPATH}" + apps: certbot: command: bin/python3 -s $SNAP/bin/certbot @@ -47,7 +50,8 @@ parts: - ./certbot-apache - ./certbot-nginx stage: - - -usr/lib/python3.8/sitecustomize.py # maybe unnecessary + - -usr/lib/python3.12/sitecustomize.py # maybe unnecessary + - -pyvenv.cfg # Old versions of this file used to unstage # lib/python3.8/site-packages/augeas.py to avoid conflicts between # python-augeas 0.5.0 which was pinned in snap-constraints.txt and @@ -57,18 +61,17 @@ parts: # effect so we now stage the file to keep the auto-generated cffi file. stage-packages: - libaugeas0 - - libpython3.8-dev + - libpython3.12-dev # added to stage python: - libpython3-stdlib - - libpython3.8-stdlib - - libpython3.8-minimal + - libpython3.12-stdlib + - libpython3.12-minimal - python3-pip - python3-wheel - python3-venv - python3-minimal - - python3-distutils - python3-pkg-resources - - python3.8-minimal + - python3.12-minimal # To build cryptography and cffi if needed build-packages: - gcc @@ -85,28 +88,30 @@ parts: # stability of fetching the rust crates needed to build the cryptography # library. - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - - SNAPCRAFT_PYTHON_VENV_ARGS: --upgrade + - PARTS_PYTHON_VENV_ARGS: --upgrade # Constraints are passed through the environment variable PIP_CONSTRAINTS instead of using the # parts.[part_name].constraints option available in snapcraft.yaml when the Python plugin is # used. This is done to let these constraints be applied not only on the certbot package # build, but also on any isolated build that pip could trigger when building wheels for # dependencies. See https://github.com/certbot/certbot/pull/8443 for more info. - - PIP_CONSTRAINT: $SNAPCRAFT_PART_SRC/snap-constraints.txt + - PIP_CONSTRAINT: $CRAFT_PART_SRC/snap-constraints.txt override-build: | - python3 -m venv "${SNAPCRAFT_PART_INSTALL}" - "${SNAPCRAFT_PART_INSTALL}/bin/python3" "${SNAPCRAFT_PART_SRC}/tools/pipstrap.py" - snapcraftctl build + python3 -m venv "${CRAFT_PART_INSTALL}" + "${CRAFT_PART_INSTALL}/bin/python3" "${CRAFT_PART_SRC}/tools/pipstrap.py" + craftctl default override-pull: | - snapcraftctl pull - grep -v python-augeas "${SNAPCRAFT_PART_SRC}/tools/requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` + craftctl default + grep -v python-augeas "${CRAFT_PART_SRC}/tools/requirements.txt" >> "${CRAFT_PART_SRC}/snap-constraints.txt" + craftctl set version=$(grep -oP "__version__ = '\K.*(?=')" "${CRAFT_PART_SRC}/certbot/certbot/__init__.py") + build-attributes: + - enable-patchelf shared-metadata: plugin: dump source: . override-pull: | - snapcraftctl pull + craftctl default mkdir -p certbot-metadata - grep -oP "__version__ = '\K.*(?=')" $SNAPCRAFT_PART_SRC/certbot/certbot/__init__.py > certbot-metadata/certbot-version.txt + grep -oP "__version__ = '\K.*(?=')" $CRAFT_PART_SRC/certbot/certbot/__init__.py > certbot-metadata/certbot-version.txt stage: [certbot-metadata/certbot-version.txt] plugs: diff --git a/tools/snap/README.md b/tools/snap/README.md index f6354f748..c4ae0558d 100644 --- a/tools/snap/README.md +++ b/tools/snap/README.md @@ -12,7 +12,7 @@ These steps need to be done once to set up your VM and do not need to be run aga 1. Start with a Focal VM. You need a full virtual machine using something like DigitalOcean, EC2, or VirtualBox. Docker won't work. Another version of Ubuntu can probably be used, but Focal was used when writing these instructions. 2. Set up a user other than root with sudo privileges for use with snapcraft and run all of the following commands with it. A command to do this for a user named certbot looks like `adduser certbot && usermod -aG sudo certbot && su - certbot`. 3. Install git and python with `sudo apt update && sudo apt install -y git python`. - 4. Set up lxd for use with snapcraft by running `sudo snap install lxd && sudo /snap/bin/lxd.migrate -yes; sudo /snap/bin/lxd waitready && sudo /snap/bin/lxd init --auto` (errors here are ok; it may already + 4. Set up lxd for use with snapcraft by running `sudo snap install lxd; sudo /snap/bin/lxd waitready && sudo /snap/bin/lxd init --auto` (errors here are ok; it may already have been installed on your system). 5. Add your current user to the lxd group and update your shell to have the new assignment by running `sudo usermod -a -G lxd ${USER} && newgrp lxd`. 6. Install snapcraft with `sudo snap install --classic snapcraft`. diff --git a/tools/snap/build_remote.py b/tools/snap/build_remote.py index 717eabd07..9935d7ab4 100755 --- a/tools/snap/build_remote.py +++ b/tools/snap/build_remote.py @@ -49,26 +49,32 @@ def _snap_log_name(target: str, arch: str): def _execute_build( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], workspace: str, output_lock: Lock) -> Tuple[int, List[str]]: - - # snapcraft remote-build accepts a --build-id flag with snapcraft version - # 5.0+. We make use of this feature to set a unique build ID so a fresh - # build is started for each run instead of potentially reusing an old - # build. See https://github.com/certbot/certbot/pull/8719 and - # https://github.com/snapcore/snapcraft/pull/3554 for more info. + # The implementation of remote-build recovery has changed over time. + # Currently, you cannot set a build-id, and the build-id is instead derived + # from a hash of the contents of the files in the directory: + # https://github.com/canonical/craft-application/blob/5b09ab3d9152a2b61ffcdf57691289023ed6ba26/craft_application/remote/utils.py#L64 # - # This random string was chosen because snapcraft uses a MD5 hash - # represented as a 32 character hex string by default, so we use the same - # length but from a larger character set just because we can. + # We want a unique build ID so a fresh build is started for each run instead + # of potentially reusing an old build. See https://github.com/certbot/certbot/pull/8719 + # and https://github.com/snapcore/snapcraft/pull/3554 for more info. + # + # In the hope that one day you can again set a build ID, we will modify + # the directory by creating a file containing a build ID that conforms + # to the shape of snapcraft's build ID: using a MD5 hash represented as a + # 32 character hex string (we use a larger character set). + random_string = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(32)) - build_id = f'snapcraft-{target}-{random_string}' + # place random string in build_id file inside `workspace` directory + with open(join(workspace, 'build_id'), 'w') as build_id_file: + build_id_file.write(random_string) with tempfile.TemporaryDirectory() as tempdir: environ = os.environ.copy() environ['XDG_CACHE_HOME'] = tempdir process = subprocess.Popen([ 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', - '--build-for', ','.join(archs), '--build-id', build_id], + '--build-for', ','.join(archs)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, env=environ, cwd=workspace) @@ -103,6 +109,10 @@ def _build_snap( workspace = CERTBOT_DIR else: workspace = join(CERTBOT_DIR, target) + # init and commit git repo in workspace + subprocess.run(['git', 'init'], capture_output=True, check=True, cwd=workspace) + subprocess.run(['git', 'add', '-A'], capture_output=True, check=True, cwd=workspace) + subprocess.run(['git', 'commit', '-m', 'init'], capture_output=True, check=True, cwd=workspace) build_success = False retry = 3 @@ -115,7 +125,7 @@ def _build_snap( print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with ' f'exit code {exit_code}.') - failed_archs = [arch for arch in archs if status[target][arch] != 'Successfully built'] + failed_archs = [arch for arch in archs if status[target][arch] != 'Succeeded'] # If the command failed or any architecture wasn't built # successfully, let's try to print all the output about the problem # that we can. @@ -147,17 +157,17 @@ def _build_snap( def _extract_state(project: str, output: str, status: Dict[str, Dict[str, str]]) -> None: state = status[project] - if "Sending build data to Launchpad..." in output: + if "Starting new build" in output: for arch in state.keys(): - state[arch] = "Sending build data" + state[arch] = "Starting new build" - match = re.match(r'^.*arch=(\w+)\s+state=([\w ]+).*$', output) + match = re.match(r'^(\w+): (\w+)$', output) if match: - arch = match.group(1) - state[arch] = match.group(2) + arch = match.group(2) + state[arch] = match.group(1) # You need to reassign the value of status[project] here (rather than doing - # something like status[project][arch] = match.group(2)) for the state change + # something like status[project][arch] = match.group(1)) for the state change # to propagate to other processes. See # https://docs.python.org/3.8/library/multiprocessing.html#proxy-objects for # more info. @@ -187,18 +197,18 @@ def _dump_status( def _dump_failed_build_logs( target: str, archs: Set[str], status: Dict[str, Dict[str, str]], workspace: str) -> None: + logs_list = glob.glob(join(workspace, f'snapcraft-{target}-*.txt')) for arch in archs: result = status[target][arch] - if result != 'Successfully built': + if result != 'Succeeded': failures = True - build_output_name = _snap_log_name(target, arch) - build_output_path = join(workspace, build_output_name) - if not exists(build_output_path): + build_output_path = [log_name for log_name in logs_list if arch in log_name] + if not build_output_path: build_output = f'No output has been dumped by snapcraft remote-build.' else: - with open(build_output_path) as file_h: + with open(build_output_path[0]) as file_h: build_output = file_h.read() print(f'Output for failed build target={target} arch={arch}') @@ -240,10 +250,6 @@ def main(): subprocess.run(['tools/snap/generate_dnsplugins_all.sh'], check=True, cwd=CERTBOT_DIR) - # Use the legacy remote launchpad build until - # https://github.com/certbot/certbot/issues/9890 is fixed - os.environ['SNAPCRAFT_REMOTE_BUILD_STRATEGY'] = 'force-fallback' - print('Start remote snap builds...') print(f' - archs: {", ".join(archs)}') print(f' - projects: {", ".join(sorted(targets))}') diff --git a/tools/snap/generate_dnsplugins_snapcraft.sh b/tools/snap/generate_dnsplugins_snapcraft.sh index 1de5e584d..3be6ffff1 100755 --- a/tools/snap/generate_dnsplugins_snapcraft.sh +++ b/tools/snap/generate_dnsplugins_snapcraft.sh @@ -16,7 +16,7 @@ summary: ${DESCRIPTION} description: ${DESCRIPTION} confinement: strict grade: stable -base: core20 +base: core24 adopt-info: ${PLUGIN} parts: @@ -24,8 +24,8 @@ parts: plugin: python source: . override-pull: | - snapcraftctl pull - snapcraftctl set-version \`grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]"\` + craftctl default + craftctl set version=\$(grep ^version \$SNAPCRAFT_PART_SRC/setup.py | cut -f2 -d= | tr -d "'[:space:]") build-environment: # We set this environment variable while building to try and increase the # stability of fetching the rust crates needed to build the cryptography @@ -53,7 +53,7 @@ parts: source: . stage: [setup.py, certbot-shared] override-pull: | - snapcraftctl pull + craftctl default mkdir -p \$SNAPCRAFT_PART_SRC/certbot-shared slots: @@ -61,7 +61,7 @@ slots: interface: content content: certbot-1 read: - - \$SNAP/lib/python3.8/site-packages + - \$SNAP/lib/python3.12/site-packages plugs: certbot-metadata: