starred/certbot
1
0
Fork 0
mirror of https://github.com/certbot/certbot.git synced 2026-08-01 18:56:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Setup a timeout to the remote snap build process (#8484)

Browse Source 
This PR adds a `--timeout` flag to `tools/snap/build_remote.py` in order to fail the process if the time execution reaches the provided timeout. It is set to 5h30 on the relevant Azure job, while the job itself has a timeout of 6h managed on Azure side. This allows a slightly better output for these jobs when the snapcraft build stales for any reason.
This commit is contained in:
Adrien Ferrand
2020-12-11 12:33:11 -08:00
committed by GitHub
parent 6d71378c05
commit 3889311557
2 changed files with 44 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.azure-pipelines/templates/jobs/packaging-jobs.yml
+1 -1
View File
@@ -144,7 +144,7 @@ jobs:
git config --global user.name "$(Build.RequestedFor)" git config --global user.name "$(Build.RequestedFor)"
mkdir -p ~/.local/share/snapcraft/provider/launchpad mkdir -p ~/.local/share/snapcraft/provider/launchpad
cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials cp $(credentials.secureFilePath) ~/.local/share/snapcraft/provider/launchpad/credentials
python3 tools/snap/build_remote.py ALL --archs ${ARCHS} python3 tools/snap/build_remote.py ALL --archs ${ARCHS} --timeout 19800
displayName: Build snaps displayName: Build snaps
- script: | - script: |
set -e set -e
tools/snap/build_remote.py
+43 -32
View File
@@ -1,22 +1,22 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import glob
import datetime import datetime
from multiprocessing import Pool, Process, Manager, Event import glob
import re import re
import subprocess import subprocess
import sys import sys
import tempfile import time
from multiprocessing import Pool, Process, Manager
from os.path import join, realpath, dirname, basename, exists from os.path import join, realpath, dirname, basename, exists
CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__)))) CERTBOT_DIR = dirname(dirname(dirname(realpath(__file__))))
PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))] PLUGINS = [basename(path) for path in glob.glob(join(CERTBOT_DIR, 'certbot-dns-*'))]
def _execute_build(target, archs, status, workspace): def _execute_build(target, archs, status, workspace):
process = subprocess.Popen([ process = subprocess.Popen([
'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover', '--build-on', ','.join(archs) 'snapcraft', 'remote-build', '--launchpad-accept-public-upload', '--recover',
'--build-on', ','.join(archs)
], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, cwd=workspace)
process_output = [] process_output = []
@@ -27,7 +27,7 @@ def _execute_build(target, archs, status, workspace):
return process.wait(), process_output return process.wait(), process_output
def _build_snap(target, archs, status, lock): def _build_snap(target, archs, status, running, lock):
status[target] = {arch: '...' for arch in archs} status[target] = {arch: '...' for arch in archs}
if target == 'certbot': if target == 'certbot':
@@ -39,7 +39,8 @@ def _build_snap(target, archs, status, lock):
while retry: while retry:
exit_code, process_output = _execute_build(target, archs, status, workspace) exit_code, process_output = _execute_build(target, archs, status, workspace)
print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with exit code {exit_code}.') print(f'Build {target} for {",".join(archs)} (attempt {4-retry}/3) ended with '
f'exit code {exit_code}.')
sys.stdout.flush() sys.stdout.flush()
with lock: with lock:
@@ -49,7 +50,8 @@ def _build_snap(target, archs, status, lock):
# We expect to have all target snaps available, or something bad happened. # We expect to have all target snaps available, or something bad happened.
snaps_list = glob.glob(join(workspace, '*.snap')) snaps_list = glob.glob(join(workspace, '*.snap'))
if not len(snaps_list) == len(archs): if not len(snaps_list) == len(archs):
print(f'Some of the expected snaps for a successful build are missing (current list: {snaps_list}).') print('Some of the expected snaps for a successful build are missing '
f'(current list: {snaps_list}).')
dump_output = True dump_output = True
else: else:
break break
@@ -63,9 +65,12 @@ def _build_snap(target, archs, status, lock):
print(f'Dumping snapcraft remote-build output build for {target}:') print(f'Dumping snapcraft remote-build output build for {target}:')
print('\n'.join(process_output)) print('\n'.join(process_output))
# Retry the remote build if it has been interrupted (non zero status code) or if some builds have failed. # Retry the remote build if it has been interrupted (non zero status code)
# or if some builds have failed.
retry = retry - 1 retry = retry - 1
running[target] = False
return {target: workspace} return {target: workspace}
@@ -96,15 +101,11 @@ def _dump_status_helper(archs, status):
sys.stdout.flush() sys.stdout.flush()
def _dump_status(archs, status, stop_event): def _dump_status(archs, status, running):
while not stop_event.wait(10): while any(running.values()):
print('Remote build status at {0}'.format(datetime.datetime.now())) print(f'Remote build status at {datetime.datetime.now()}')
_dump_status_helper(archs, status) _dump_status_helper(archs, status)
time.sleep(10)
def _dump_status_final(archs, status):
print('Results for remote build finished at {0}'.format(datetime.datetime.now()))
_dump_status_helper(archs, status)
def _dump_results(targets, archs, status, workspaces): def _dump_results(targets, archs, status, workspaces):
@@ -120,10 +121,10 @@ def _dump_results(targets, archs, status, workspaces):
if not exists(build_output_path): if not exists(build_output_path):
build_output = f'No output has been dumped by snapcraft remote-build.' build_output = f'No output has been dumped by snapcraft remote-build.'
else: else:
with open(join(workspaces[target], '{0}_{1}.txt'.format(target, arch))) as file_h: with open(join(workspaces[target], f'{target}_{arch}.txt')) as file_h:
build_output = file_h.read() build_output = file_h.read()
print('Output for failed build target={0} arch={1}'.format(target, arch)) print(f'Output for failed build target={target} arch={arch}')
print('-------------------------------------------') print('-------------------------------------------')
print(build_output) print(build_output)
print('-------------------------------------------') print('-------------------------------------------')
@@ -134,6 +135,10 @@ def _dump_results(targets, archs, status, workspaces):
else: else:
print('Some builds failed.') print('Some builds failed.')
print()
print(f'Results for remote build finished at {datetime.datetime.now()}')
_dump_status_helper(archs, status)
return failures return failures
@@ -143,6 +148,8 @@ def main():
help='the list of snaps to build') help='the list of snaps to build')
parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'], parser.add_argument('--archs', nargs='+', choices=['amd64', 'arm64', 'armhf'], default=['amd64'],
help='the architectures for which snaps are built') help='the architectures for which snaps are built')
parser.add_argument('--timeout', type=int, default=None,
help='build process will fail after the provided timeout (in seconds)')
args = parser.parse_args() args = parser.parse_args()
archs = set(args.archs) archs = set(args.archs)
@@ -158,7 +165,7 @@ def main():
# If we're building anything other than just Certbot, we need to # If we're building anything other than just Certbot, we need to
# generate the snapcraft files for the DNS plugins. # generate the snapcraft files for the DNS plugins.
if targets != set(('certbot',)): if targets != {'certbot'}:
subprocess.run(['tools/snap/generate_dnsplugins_all.sh'], subprocess.run(['tools/snap/generate_dnsplugins_all.sh'],
check=True, cwd=CERTBOT_DIR) check=True, cwd=CERTBOT_DIR)
@@ -169,25 +176,29 @@ def main():
with Manager() as manager, Pool(processes=len(targets)) as pool: with Manager() as manager, Pool(processes=len(targets)) as pool:
status = manager.dict() status = manager.dict()
running = manager.dict({target: True for target in targets})
lock = manager.Lock() lock = manager.Lock()
stop_event = Event() async_results = [pool.apply_async(_build_snap, (target, archs, status, running, lock))
state_process = Process(target=_dump_status, args=(archs, status, stop_event)) for target in targets]
state_process.start()
async_results = [pool.apply_async(_build_snap, (target, archs, status, lock)) for target in targets] process = Process(target=_dump_status, args=(archs, status, running))
process.start()
workspaces = {} try:
for async_result in async_results: process.join(args.timeout)
workspaces.update(async_result.get())
stop_event.set() if process.is_alive():
state_process.join() raise ValueError(f"Timeout out reached ({args.timeout} seconds) during the build!")
failures = _dump_results(targets, archs, status, workspaces) workspaces = {}
_dump_status_final(archs, status) for async_result in async_results:
workspaces.update(async_result.get())
return 1 if failures else 0 if _dump_results(targets, archs, status, workspaces):
raise ValueError("There were failures during the build!")
finally:
process.terminate()
if __name__ == '__main__': if __name__ == '__main__':