Use build_remote_single.py instead of build_remote.py

2026-07-28 08:05:31 +02:00 · 2026-06-11 11:12:41 -07:00
parent f7b5c61f04
commit 29e13edc76
2 changed files with 211 additions and 5 deletions
@@ -12,9 +12,6 @@ on:
 permissions:
  contents: read

-env:
-  SNAP_BUILD_TIMEOUT: ${{ inputs.snapBuildTimeout }}
-
 jobs:
  gha_build_snap:
    name: Build certbot ${{ matrix.build-for }}
@@ -114,7 +111,7 @@ jobs:
    - uses: actions/setup-python@v6.2.0
      with:
        python-version: '3.12'
-    - name: Build snaps
+    - name: Build snap
      env:
        LAUNCHPAD_CREDS: "${{ secrets.LAUNCHPAD_CREDENTIALS }}"
      run: |-
@@ -122,7 +119,7 @@ jobs:
        git config --global user.name "41898282+github-actions[bot]@users.noreply.github.com"
        mkdir -p ~/.local/share/snapcraft/
        echo "$LAUNCHPAD_CREDS" > ~/.local/share/snapcraft/launchpad-credentials
-        python3 tools/snap/build_remote.py ${{ matrix.target }} --archs "$SNAP_ARCH" --timeout "$SNAP_BUILD_TIMEOUT"
+        python3 tools/snap/build_remote_single.py ${{ matrix.target }} --arch "$SNAP_ARCH"
    - name: Flatten snaps layout
      run: |-
        mv *.snap ${{ runner.temp }} || true
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+import argparse
+import functools
+import glob
+import os
+from os.path import basename
+from os.path import dirname
+from os.path import join
+from os.path import realpath
+import random
+import re
+import string
+import subprocess
+import sys
+import tempfile
+from typing import List
+from typing import Tuple
+
+CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__))))
+PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))]
+
+
+# In Python, stdout and stderr are buffered in each process by default. When
+# printing output from multiple processes, this can cause delays in printing
+# output with lines from different processes being interleaved depending
+# on when the output for that process is flushed. To prevent this, we override
+# print so that it always flushes its output. Disabling output buffering can
+# also be done through command line flags or environment variables set when the
+# Python process starts, but this approach was taken instead to ensure
+# consistent behavior regardless of how the script is invoked.
+print = functools.partial(print, flush=True)
+
+
+def _execute_build(
+        target: str, arch: str,
+        workspace: str) -> Tuple[int, List[str], str]:
+    # The implementation of remote-build recovery has changed over time.
+    # Currently, you cannot set a build-id, and the build-id is instead derived
+    # from a hash of the contents of the files in the directory:
+    # https://github.com/canonical/craft-application/blob/5b09ab3d9152a2b61ffcdf57691289023ed6ba26/craft_application/remote/utils.py#L64
+    #
+    # We want a unique build ID so a fresh build is started for each run instead
+    # of potentially reusing an old build. See https://github.com/certbot/certbot/pull/8719
+    # and https://github.com/snapcore/snapcraft/pull/3554 for more info.
+    #
+    # In the hope that one day you can again set a build ID, we will modify
+    # the directory by creating a file containing a build ID that conforms
+    # to the shape of snapcraft's build ID: using a MD5 hash represented as a
+    # 32 character hex string (we use a larger character set).
+
+    status: str = "..."
+
+    random_string = ''.join(random.choice(string.ascii_lowercase + string.digits)
+                            for _ in range(32))
+    # place random string in build_id file inside `workspace` directory
+    with open(join(workspace, 'build_id'), 'w') as build_id_file:
+        build_id_file.write(random_string)
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        environ = os.environ.copy()
+        environ['XDG_CACHE_HOME'] = tempdir
+        process = subprocess.Popen([
+            'snapcraft', 'remote-build', '--launchpad-accept-public-upload',
+            '--build-for', arch],
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            universal_newlines=True, env=environ, cwd=workspace, bufsize=1)
+
+    killed = False
+    process_output: List[str] = []
+    for line in process.stdout:
+        print(line.rstrip())
+        process_output.append(line.rstrip())
+        status = _extract_state(target, line, status)
+
+        if not killed and status == 'Chroot problem':
+            # On this error the snapcraft process hangs. Let's finish it.
+            #
+            # killed is used to stop us from executing this code path
+            # multiple times per build that encounters "Chroot problem".
+            print('Chroot problem encountered for build '
+                  f'{target} for {arch}.\n'
+                  'Launchpad seems to be unable to recover from this '
+                  'state so we are terminating the build.')
+            process.kill()
+            killed = True
+
+    process_state = process.wait()
+
+    return process_state, process_output, status
+
+
+def _extract_state(project: str, output: str, state: str) -> str:
+    # This output may change, and is set by
+    # https://github.com/canonical/snapcraft/blob/8ab7fd0c8a1d3f13045bec41a6e0158c063faa9b/snapcraft/commands/remote.py#L218
+    if "Starting new build" in output:
+        state = "Starting new build"
+
+    match = re.match(r'^(\w+): (\w+)$', output)
+    if match:
+        state = match.group(1)
+
+    return state
+
+
+def build_snap(target: str, arch: str) -> None:
+    if target == 'certbot':
+        workspace = CERTBOT_DIR
+    else:
+        workspace = join(CERTBOT_DIR, target)
+        # Init and commit git repo in workspace. This is necessary starting in core24
+        # as "Projects must be at the top level of a git repository"
+        # https://snapcraft.io/docs/migrate-core24#remote-build
+        subprocess.run(['git', 'init'], capture_output=True, check=True, cwd=workspace)
+        subprocess.run(['git', 'add', '-A'], capture_output=True, check=True, cwd=workspace)
+        subprocess.run(['git', 'commit', '-m', 'init'], capture_output=True, check=True, cwd=workspace)
+
+    exit_code, process_output, status = _execute_build(target, arch, workspace)
+    print(f'Build {target} for {arch} ended with '
+          f'exit code {exit_code}.')
+
+    # This output may change, and is set by
+    # https://github.com/canonical/snapcraft/blob/8ab7fd0c8a1d3f13045bec41a6e0158c063faa9b/snapcraft/commands/remote.py#L278
+    failed = status != 'Succeeded'
+
+    # If the command failed, let's try to print all the output about the problem
+    # that we can.
+    failed = exit_code != 0 or failed
+
+    # Check that snap file exists
+    # We expect to have the target snap available, or something bad happened.
+    if not failed:
+        snap_path_list = glob.glob(join(workspace, f'{target}_*_{arch}.snap'))
+        if not len(snap_path_list) == 1:
+            print('The expected snap is missing.')
+            failed = True
+
+    # Check if the snap file just contains html
+    if not failed:
+        with open(snap_path_list[0], 'r') as f:
+            try:
+                first_line = f.readline().rstrip()
+            except UnicodeDecodeError:
+                first_line = ''
+            if first_line == "<!DOCTYPE html>":
+                failed = True
+                print(f'The {target} {arch} snap file contains html instead of a snap')
+
+    if failed:
+        print('Dumping snapcraft remote-build logs:')
+        log_location = _extract_log_location(process_output[-1])
+        _dump_failed_build_logs(log_location)
+        print('Build failed.')
+        raise ValueError("There were failures during the build!")
+    else:
+        print('Build succeeded.')
+
+
+def _extract_log_location(line: str) -> str:
+    print(f'Final line: {line}') # for testing, should be removed
+    result = ""
+    match = re.match(r"^Full execution log: '(.+)'$", line)
+
+    if match:
+        result = match.group(1)
+    print(f'Log location: {result}') # for testing, should be removed
+    return result
+
+
+def _dump_failed_build_logs(build_output_path: str) -> None:
+    if not build_output_path:
+        build_output = 'Log location not extracted from output.'
+    else:
+        with open(build_output_path) as file_h:
+            build_output = file_h.read()
+
+    print('Output for failed build')
+    print('-------------------------------------------')
+    print(build_output)
+    print('-------------------------------------------')
+    print()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('target', choices=['certbot', *PLUGINS],
+                        help='the snap to build')
+    parser.add_argument('--arch', choices=['amd64', 'arm64', 'armhf'],
+                        default='amd64', help='the architecture for which snap is built')
+    args = parser.parse_args()
+
+    arch = args.arch
+    target = args.target
+
+    # If we're building anything other than just Certbot, we need to
+    # generate the snapcraft files for the DNS plugins.
+    if target != 'certbot':
+        subprocess.run(['tools/snap/generate_dnsplugins_all.sh'],
+                       check=True, cwd=CERTBOT_DIR)
+
+    print('Start remote snap build...')
+    print(f' - arch: {arch}')
+    print(f' - project: {target}')
+    print()
+
+    build_snap(target, arch)
+
+
+if __name__ == '__main__':
+    sys.exit(main())