diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml
index b1c5ea6b1..d4550aad0 100644
--- a/.azure-pipelines/templates/installer-tests.yml
+++ b/.azure-pipelines/templates/installer-tests.yml
@@ -1,5 +1,5 @@
 jobs:
-  - job: installer
+  - job: installer_build
     pool:
       vmImage: vs2017-win2016
     steps:
@@ -20,10 +20,32 @@ jobs:
           path: $(Build.ArtifactStagingDirectory)
           artifact: windows-installer
         displayName: Publish Windows installer
-      - script: $(Build.ArtifactStagingDirectory)\certbot-beta-installer-win32.exe /S
+  - job: installer_run
+    dependsOn: installer_build
+    strategy:
+      matrix:
+        win2019:
+          imageName: windows-2019
+        win2016:
+          imageName: vs2017-win2016
+        win2012r2:
+          imageName: vs2015-win2012r2
+    pool:
+      vmImage: $(imageName)
+    steps:
+      - task: DownloadPipelineArtifact@2
+        inputs:
+          artifact: windows-installer
+          path: $(Build.SourcesDirectory)/bin
+        displayName: Retrieve Windows installer
+      - script: $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe /S
         displayName: Install Certbot
+      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.0/python-3.8.0-amd64-webinstall.exe -OutFile C:\py3-setup.exe
+        displayName: Get Python
+      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=0 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
+        displayName: Install Python
       - script: |
-          python -m venv venv
+          py -3 -m venv venv
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |