From 29e13edc769e8bf98bd2f394a582626939f9a4dd Mon Sep 17 00:00:00 2001 From: Erica Portnoy Date: Thu, 11 Jun 2026 10:18:54 -0700 Subject: [PATCH] Use build_remote_single.py instead of build_remote.py --- .github/workflows/snap_packaging_jobs.yml | 7 +- tools/snap/build_remote_single.py | 209 ++++++++++++++++++++++ 2 files changed, 211 insertions(+), 5 deletions(-) create mode 100755 tools/snap/build_remote_single.py diff --git a/.github/workflows/snap_packaging_jobs.yml b/.github/workflows/snap_packaging_jobs.yml index b8855a674..481702c1b 100644 --- a/.github/workflows/snap_packaging_jobs.yml +++ b/.github/workflows/snap_packaging_jobs.yml @@ -12,9 +12,6 @@ on: permissions: contents: read -env: - SNAP_BUILD_TIMEOUT: ${{ inputs.snapBuildTimeout }} - jobs: gha_build_snap: name: Build certbot ${{ matrix.build-for }} @@ -114,7 +111,7 @@ jobs: - uses: actions/setup-python@v6.2.0 with: python-version: '3.12' - - name: Build snaps + - name: Build snap env: LAUNCHPAD_CREDS: "${{ secrets.LAUNCHPAD_CREDENTIALS }}" run: |- @@ -122,7 +119,7 @@ jobs: git config --global user.name "41898282+github-actions[bot]@users.noreply.github.com" mkdir -p ~/.local/share/snapcraft/ echo "$LAUNCHPAD_CREDS" > ~/.local/share/snapcraft/launchpad-credentials - python3 tools/snap/build_remote.py ${{ matrix.target }} --archs "$SNAP_ARCH" --timeout "$SNAP_BUILD_TIMEOUT" + python3 tools/snap/build_remote_single.py ${{ matrix.target }} --arch "$SNAP_ARCH" - name: Flatten snaps layout run: |- mv *.snap ${{ runner.temp }} || true diff --git a/tools/snap/build_remote_single.py b/tools/snap/build_remote_single.py new file mode 100755 index 000000000..80e85448a --- /dev/null +++ b/tools/snap/build_remote_single.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +import argparse +import functools +import glob +import os +from os.path import basename +from os.path import dirname +from os.path import join +from os.path import realpath +import random +import re +import string +import subprocess +import sys +import tempfile +from typing import List +from typing import Tuple + +CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) +PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] + + +# In Python, stdout and stderr are buffered in each process by default. When +# printing output from multiple processes, this can cause delays in printing +# output with lines from different processes being interleaved depending +# on when the output for that process is flushed. To prevent this, we override +# print so that it always flushes its output. Disabling output buffering can +# also be done through command line flags or environment variables set when the +# Python process starts, but this approach was taken instead to ensure +# consistent behavior regardless of how the script is invoked. +print = functools.partial(print, flush=True) + + +def _execute_build( + target: str, arch: str, + workspace: str) -> Tuple[int, List[str], str]: + # The implementation of remote-build recovery has changed over time. + # Currently, you cannot set a build-id, and the build-id is instead derived + # from a hash of the contents of the files in the directory: + # https://github.com/canonical/craft-application/blob/5b09ab3d9152a2b61ffcdf57691289023ed6ba26/craft_application/remote/utils.py#L64 + # + # We want a unique build ID so a fresh build is started for each run instead + # of potentially reusing an old build. See https://github.com/certbot/certbot/pull/8719 + # and https://github.com/snapcore/snapcraft/pull/3554 for more info. + # + # In the hope that one day you can again set a build ID, we will modify + # the directory by creating a file containing a build ID that conforms + # to the shape of snapcraft's build ID: using a MD5 hash represented as a + # 32 character hex string (we use a larger character set). + + status: str = "..." + + random_string = ''.join(random.choice(string.ascii_lowercase + string.digits) + for _ in range(32)) + # place random string in build_id file inside `workspace` directory + with open(join(workspace, 'build_id'), 'w') as build_id_file: + build_id_file.write(random_string) + + with tempfile.TemporaryDirectory() as tempdir: + environ = os.environ.copy() + environ['XDG_CACHE_HOME'] = tempdir + process = subprocess.Popen([ + 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', + '--build-for', arch], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, env=environ, cwd=workspace, bufsize=1) + + killed = False + process_output: List[str] = [] + for line in process.stdout: + print(line.rstrip()) + process_output.append(line.rstrip()) + status = _extract_state(target, line, status) + + if not killed and status == 'Chroot problem': + # On this error the snapcraft process hangs. Let's finish it. + # + # killed is used to stop us from executing this code path + # multiple times per build that encounters "Chroot problem". + print('Chroot problem encountered for build ' + f'{target} for {arch}.\n' + 'Launchpad seems to be unable to recover from this ' + 'state so we are terminating the build.') + process.kill() + killed = True + + process_state = process.wait() + + return process_state, process_output, status + + +def _extract_state(project: str, output: str, state: str) -> str: + # This output may change, and is set by + # https://github.com/canonical/snapcraft/blob/8ab7fd0c8a1d3f13045bec41a6e0158c063faa9b/snapcraft/commands/remote.py#L218 + if "Starting new build" in output: + state = "Starting new build" + + match = re.match(r'^(\w+): (\w+)$', output) + if match: + state = match.group(1) + + return state + + +def build_snap(target: str, arch: str) -> None: + if target == 'certbot': + workspace = CERTBOT_DIR + else: + workspace = join(CERTBOT_DIR, target) + # Init and commit git repo in workspace. This is necessary starting in core24 + # as "Projects must be at the top level of a git repository" + # https://snapcraft.io/docs/migrate-core24#remote-build + subprocess.run(['git', 'init'], capture_output=True, check=True, cwd=workspace) + subprocess.run(['git', 'add', '-A'], capture_output=True, check=True, cwd=workspace) + subprocess.run(['git', 'commit', '-m', 'init'], capture_output=True, check=True, cwd=workspace) + + exit_code, process_output, status = _execute_build(target, arch, workspace) + print(f'Build {target} for {arch} ended with ' + f'exit code {exit_code}.') + + # This output may change, and is set by + # https://github.com/canonical/snapcraft/blob/8ab7fd0c8a1d3f13045bec41a6e0158c063faa9b/snapcraft/commands/remote.py#L278 + failed = status != 'Succeeded' + + # If the command failed, let's try to print all the output about the problem + # that we can. + failed = exit_code != 0 or failed + + # Check that snap file exists + # We expect to have the target snap available, or something bad happened. + if not failed: + snap_path_list = glob.glob(join(workspace, f'{target}_*_{arch}.snap')) + if not len(snap_path_list) == 1: + print('The expected snap is missing.') + failed = True + + # Check if the snap file just contains html + if not failed: + with open(snap_path_list[0], 'r') as f: + try: + first_line = f.readline().rstrip() + except UnicodeDecodeError: + first_line = '' + if first_line == "": + failed = True + print(f'The {target} {arch} snap file contains html instead of a snap') + + if failed: + print('Dumping snapcraft remote-build logs:') + log_location = _extract_log_location(process_output[-1]) + _dump_failed_build_logs(log_location) + print('Build failed.') + raise ValueError("There were failures during the build!") + else: + print('Build succeeded.') + + +def _extract_log_location(line: str) -> str: + print(f'Final line: {line}') # for testing, should be removed + result = "" + match = re.match(r"^Full execution log: '(.+)'$", line) + + if match: + result = match.group(1) + print(f'Log location: {result}') # for testing, should be removed + return result + + +def _dump_failed_build_logs(build_output_path: str) -> None: + if not build_output_path: + build_output = 'Log location not extracted from output.' + else: + with open(build_output_path) as file_h: + build_output = file_h.read() + + print('Output for failed build') + print('-------------------------------------------') + print(build_output) + print('-------------------------------------------') + print() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('target', choices=['certbot', *PLUGINS], + help='the snap to build') + parser.add_argument('--arch', choices=['amd64', 'arm64', 'armhf'], + default='amd64', help='the architecture for which snap is built') + args = parser.parse_args() + + arch = args.arch + target = args.target + + # If we're building anything other than just Certbot, we need to + # generate the snapcraft files for the DNS plugins. + if target != 'certbot': + subprocess.run(['tools/snap/generate_dnsplugins_all.sh'], + check=True, cwd=CERTBOT_DIR) + + print('Start remote snap build...') + print(f' - arch: {arch}') + print(f' - project: {target}') + print() + + build_snap(target, arch) + + +if __name__ == '__main__': + sys.exit(main())