From 9ec4105ff3c00582b628de4761260b0b0e643da8 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 28 Apr 2026 12:34:42 -0700 Subject: [PATCH] Migrate PR tests from azure pipelines to github actions (#10622) Related to https://github.com/certbot/certbot/issues/10581 This is the first step of migrating to github actions. Nightly and full tests have been converted on branch `convert-all-pipelines`; you can see additional changes to do those for context [here](https://github.com/certbot/certbot/compare/convert-pr-tests...convert-all-pipelines). Some notes: - All github workflows must be flat in the `.github/workflows/` directory. - Github actions doesn't have a concept of "stages." Instead, it generates a dependency graph, which is kind of nice. You can see an example of a more complicated one [here](https://github.com/certbot/certbot/actions/runs/24580625688). - I don't know why the actions in the left bar (under Actions tab --> All workflows) are using the path instead of the listed name. I suspect it has something to do with not being run on main. Once it's merged, if the name doesn't change, we can delete previous runs and that will clear the entry on the left. - "permissions" is for the fine-grained github PAT. contents: read is needed for the "checkout" action, which basically everything uses. it's still best practice to define per-workflow. it can also be defined per-job, but per-workflow seemed nicer to me. [This](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#permissions) is the best permissions explanation I've found; [some actions](https://github.com/actions/checkout) mention what permissions they need. - For definitions of the keywords to `on`, see [here](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows). - Some of the potential inputs in tox steps are not used in this PR because we're not running the AWS tests. It seemed messier to take them out here and put them back later when the extended tests need them, but I can do that on request. We currently have a `main` [protection rule](https://github.com/certbot/certbot/settings/branch_protection_rules/5466) set that Azure pipelines PR test suite must pass before merging. Obviously I don't want to turn that off before this PR is reviewed. In github actions, it can only require a specific job to pass, though you can have multiple. To address this, I've created a job that requires all other jobs to pass, and that can be set at the required job. We probably do not want to list every individual job, as that includes every job generated by a matrix strategy. To find it in the protection rules page, start typing "PR test suite success" and it will show up. --------- Co-authored-by: Brad Warren Co-authored-by: Will Greenberg --- .azure-pipelines/main.yml | 18 ------ .github/workflows/pr-test-suite.yml | 27 +++++++++ .github/workflows/sphinx_steps.yml | 42 ++++++++++++++ .github/workflows/standard_tests_jobs.yml | 43 ++++++++++++++ .github/workflows/tox_steps.yml | 68 +++++++++++++++++++++++ .gitignore | 4 ++ 6 files changed, 184 insertions(+), 18 deletions(-) delete mode 100644 .azure-pipelines/main.yml create mode 100644 .github/workflows/pr-test-suite.yml create mode 100644 .github/workflows/sphinx_steps.yml create mode 100644 .github/workflows/standard_tests_jobs.yml create mode 100644 .github/workflows/tox_steps.yml diff --git a/.azure-pipelines/main.yml b/.azure-pipelines/main.yml deleted file mode 100644 index 315b0d47f..000000000 --- a/.azure-pipelines/main.yml +++ /dev/null @@ -1,18 +0,0 @@ -# We run the test suite on commits to main so codecov gets coverage data -# about the main branch and can use it to track coverage changes. -trigger: - - main -pr: - - main - - '*.x' - -variables: - # We set this here to avoid coverage data being uploaded from things like our - # nightly pipeline. This is done because codecov (helpfully) keeps track of - # the number of coverage uploads for a commit and displays a warning when - # comparing two commits with an unequal number of uploads. Only uploading - # coverage here should keep the number of uploads it sees consistent. - uploadCoverage: true - -jobs: - - template: templates/jobs/standard-tests-jobs.yml diff --git a/.github/workflows/pr-test-suite.yml b/.github/workflows/pr-test-suite.yml new file mode 100644 index 000000000..0003a2d19 --- /dev/null +++ b/.github/workflows/pr-test-suite.yml @@ -0,0 +1,27 @@ +# We run the test suite on commits to main to double check that everything is good after merging +name: PR test suite +on: + push: + branches: + - main + pull_request: + branches: + - main + - "*.x" + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + standard_tests_jobs: + name: Standard tests + uses: "./.github/workflows/standard_tests_jobs.yml" + all_success: + name: PR test suite success + needs: standard_tests_jobs + runs-on: ubuntu-latest + steps: + - run: true diff --git a/.github/workflows/sphinx_steps.yml b/.github/workflows/sphinx_steps.yml new file mode 100644 index 000000000..73b2b2166 --- /dev/null +++ b/.github/workflows/sphinx_steps.yml @@ -0,0 +1,42 @@ +name: Sphinx steps +on: + workflow_call: +permissions: + contents: read + +jobs: + test_sphinx_builds: + name: build sphinx documentation + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + - name: Build Sphinx Documentation + run: |- + echo "::group::Install dependencies" + sudo apt-get update + sudo apt-get install -y --no-install-recommends libaugeas-dev + echo "::endgroup::" + FINAL_STATUS=0 + declare -a FAILED_BUILDS + echo "::group::Setup venv" + tools/venv.py + source venv/bin/activate + echo "::endgroup::" + for doc_path in */docs + do + echo "" + echo "::group::Building $doc_path" + if ! sphinx-build -W --keep-going -b html "$doc_path" "$doc_path/_build/html"; then + FINAL_STATUS=1 + FAILED_BUILDS[${#FAILED_BUILDS[@]}]="${doc_path%/docs}" + fi + echo "::endgroup::" + done + if [[ $FINAL_STATUS -ne 0 ]]; then + echo "::error::The following builds failed: ${FAILED_BUILDS[*]}" + exit 1 + fi + shell: bash diff --git a/.github/workflows/standard_tests_jobs.yml b/.github/workflows/standard_tests_jobs.yml new file mode 100644 index 000000000..2e29a1d10 --- /dev/null +++ b/.github/workflows/standard_tests_jobs.yml @@ -0,0 +1,43 @@ +name: Standard tests jobs +on: + workflow_call: +permissions: + contents: read + +jobs: + test: + name: '-' # Makes results viewing cleaner + strategy: + fail-fast: false + matrix: + PYTHON_VERSION: ['3.14'] + IMAGE_NAME: ['ubuntu-22.04'] + TOXENV: + - 'cover' + - 'lint-posix' + - 'mypy' + - 'integration' + - 'apache_compat' + - 'apacheconftest-with-pebble' + - 'nginxroundtrip' + - 'validate-changelog' + include: + - IMAGE_NAME: macOS-15 + PYTHON_VERSION: '3.14' + TOXENV: cover + PIP_USE_PEP517: 'true' + - IMAGE_NAME: ubuntu-22.04 + PYTHON_VERSION: '3.10' + TOXENV: oldest + - IMAGE_NAME: ubuntu-22.04 + PYTHON_VERSION: '3.10' + TOXENV: py310 + uses: "./.github/workflows/tox_steps.yml" + with: + PYTHON_VERSION: "${{ matrix.PYTHON_VERSION }}" + PIP_USE_PEP517: "${{ matrix.PIP_USE_PEP517 }}" + TOXENV: "${{ matrix.TOXENV }}" + IMAGE_NAME: "${{ matrix.IMAGE_NAME }}" + test_sphinx_builds: + name: '-' # Makes results viewing cleaner + uses: "./.github/workflows/sphinx_steps.yml" diff --git a/.github/workflows/tox_steps.yml b/.github/workflows/tox_steps.yml new file mode 100644 index 000000000..8f17d5d12 --- /dev/null +++ b/.github/workflows/tox_steps.yml @@ -0,0 +1,68 @@ +name: Tox steps +on: + workflow_call: + inputs: + PYTHON_VERSION: + type: string + IMAGE_NAME: + type: string + TOXENV: + type: string + PIP_USE_PEP517: + type: string +permissions: + contents: read + +jobs: + tox_all: + name: ${{ inputs.TOXENV }} ${{ inputs.IMAGE_NAME }} + runs-on: + - "${{ inputs.IMAGE_NAME }}" + steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + with: + persist-credentials: false + # We run brew update because we've seen attempts to install an older version + # of a package fail. See + # https://github.com/actions/virtual-environments/issues/3165. + # + # We untap homebrew/core and homebrew/cask and unset HOMEBREW_NO_INSTALL_FROM_API (which + # is maybe set by the CI macOS env) because GitHub has been having issues, making these jobs + # fail on git clones: https://github.com/orgs/Homebrew/discussions/4612. + - name: Install MacOS dependencies + if: runner.os == 'macOS' + run: |- + unset HOMEBREW_NO_INSTALL_FROM_API + brew untap homebrew/core homebrew/cask + brew update + brew install augeas + BREW_PREFIX="$(brew --prefix)" + CFLAGS="$CFLAGS -I$BREW_PREFIX/include -L$BREW_PREFIX/lib" + echo "CFLAGS=$CFLAGS" >> "$GITHUB_ENV" + shell: bash + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: |- + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libaugeas-dev \ + nginx-light + sudo systemctl stop nginx + sudo sysctl net.ipv4.ip_unprivileged_port_start=0 + shell: bash + - uses: actions/setup-python@v6.2.0 + with: + python-version: "${{ inputs.PYTHON_VERSION }}" + - name: Install runtime dependencies + run: |- + python3 tools/pip_install.py tox + shell: bash + - name: Run tox + env: + PIP_USE_PEP517: "${{ inputs.PIP_USE_PEP517 }}" + TOXENV: "${{ inputs.TOXENV }}" + run: |- + env + python3 -m tox run + shell: bash diff --git a/.gitignore b/.gitignore index dcd20b9bd..49fcff369 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,7 @@ snapcraft.cfg # macOS files .DS_Store + +# azure workflow migration +.env.local +tmp