dnf/setup_rpm_repo: simplify dummy testing repos (#83624)

This change simplifies construction and footprint of testing rpm
repository created by rpmfluff:
* all packages default to noarch
* only when necessary build package for a specific architecture(s)
* do not build several repositories each for specific arch
* remove duplicate "incompatible arch" test
* skip_broken_and_nobest: move testing packages from an external repo
  into our dummy repo for transparency
* remove compatibility code from create_repo.py for versions no longer
  needed
* remove support for old OS versions from setup_rpm_repo
* simplify representation of an rpm in create_repo.py to allow easier
  changes

Notes
* there is one more external testing repo used at
https://ci-files.testing.ansible.com/test/integration/targets/setup_rpm_repo/repo-with-updateinfo

test/integration/targets/dnf/tasks/main.yml

@@ -30,6 +30,7 @@
     - include_tasks: dnfinstallroot.yml
     - include_tasks: logging.yml
     - include_tasks: cacheonly.yml
+    - include_tasks: multilib.yml
   when: ansible_distribution in ['Fedora', 'RedHat']
 
 # Attempting to install a different RHEL release in a tmpdir doesn't work (rhel8 beta)

test/integration/targets/dnf/tasks/multilib.yml

@@ -0,0 +1,70 @@
+- name: create conf file that forces x86_64 arch
+  copy:
+    content: |
+      [main]
+      arch=x86_64
+      ignorearch=true
+    dest: "{{ remote_tmp_dir }}/dnf-multilib.conf"
+
+- name: setting arch works differently in dnf5
+  copy:
+    content: |
+      x86_64
+    dest: /etc/dnf/vars/arch
+  when: dnf5|default(false)
+
+- block:
+    - name: test that only evr is compared, avoiding a situation when a specific arch would be considered as a "newer" package
+      dnf:
+        name: "{{ item }}"
+        state: present
+      loop:
+        - "multilib-dinginessentail-1.0-1.x86_64"
+        - "multilib-dinginessentail-1.0-1.i686"
+      register: dnf_results
+
+    - assert:
+        that:
+          - dnf_results["results"][0] is changed
+          - dnf_results["results"][1] is changed
+
+    - name: make sure multilib-dinginessentail is not installed
+      dnf:
+        name: multilib-dinginessentail
+        state: absent
+
+    - name: install multilib-dinginessentail both archs
+      dnf:
+        name:
+          - "{{ repodir }}/multilib-dinginessentail-1.1-1.x86_64.rpm"
+          - "{{ repodir }}/multilib-dinginessentail-1.1-1.i686.rpm"
+        state: present
+        disable_gpg_check: true
+
+    - name: try to install lower version of multilib-dinginessentail from rpm file, without allow_downgrade, just one arch
+      dnf:
+        name: "{{ repodir }}/multilib-dinginessentail-1.0-1.i686.rpm"
+        state: present
+      register: dnf_result
+
+    - name: check multilib-dinginessentail with rpm
+      shell: rpm -q multilib-dinginessentail
+      register: rpm_result
+
+    - name: verify installation
+      assert:
+        that:
+            - "not dnf_result.changed"
+            - "rpm_result.stdout_lines[0].startswith('multilib-dinginessentail-1.1-1')"
+            - "rpm_result.stdout_lines[1].startswith('multilib-dinginessentail-1.1-1')"
+  always:
+    - name: Clean up
+      dnf:
+        name: multilib-dinginessentail
+        state: absent
+    - file:
+        name: /etc/dnf/vars/arch
+        state: absent
+  module_defaults:
+    dnf:
+      conf_file: "{{ remote_tmp_dir }}/dnf-multilib.conf"

test/integration/targets/dnf/tasks/repo.yml

@@ -61,7 +61,7 @@
     # ============================================================================
     - name: Install dinginessentail-1:1.0-2
       dnf:
-        name: "dinginessentail-1:1.0-2.{{ ansible_architecture }}"
+        name: "dinginessentail-1:1.0-2.noarch"
         state: present
       register: dnf_result
 
@@ -103,7 +103,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-1 from a file (higher version is already installed)
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-1.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-1.noarch.rpm"
         state: present
         disable_gpg_check: True
       register: dnf_result
@@ -127,7 +127,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-1 from a file (downgrade)
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-1.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-1.noarch.rpm"
         state: present
         allow_downgrade: True
         disable_gpg_check: True
@@ -155,7 +155,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-1 from a file
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-1.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-1.noarch.rpm"
         state: present
         disable_gpg_check: True
       register: dnf_result
@@ -177,7 +177,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-1 from a file again
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-1.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-1.noarch.rpm"
         state: present
         disable_gpg_check: True
       register: dnf_result
@@ -194,7 +194,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-2 from a file
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-2.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-2.noarch.rpm"
         state: present
         disable_gpg_check: True
       register: dnf_result
@@ -216,7 +216,7 @@
     # ============================================================================
     - name: Install dinginessentail-1.0-2 from a file again
       dnf:
-        name: "{{ repodir }}/dinginessentail-1.0-2.{{ ansible_architecture }}.rpm"
+        name: "{{ repodir }}/dinginessentail-1.0-2.noarch.rpm"
         state: present
         disable_gpg_check: True
       register: dnf_result
@@ -231,31 +231,6 @@
             - "not dnf_result.changed"
             - "rpm_result.stdout.startswith('dinginessentail-1.0-2')"
     # ============================================================================
-    - name: Remove dinginessentail
-      dnf:
-        name: dinginessentail
-        state: absent
-
-    - name: Try to install incompatible arch
-      dnf:
-        name: "{{ repodir_ppc64 }}/dinginessentail-1.0-1.ppc64.rpm"
-        state: present
-      register: dnf_result
-      ignore_errors: yes
-
-    - name: Check dinginessentail with rpm
-      shell: rpm -q dinginessentail
-      register: rpm_result
-      ignore_errors: yes
-
-    - name: Verify installation
-      assert:
-        that:
-            - "rpm_result.rc == 1"
-            - "not dnf_result.changed"
-            - "dnf_result is failed"
-    # ============================================================================
-
     # Should install dinginessentail-with-weak-dep and dinginessentail-weak-dep
     - name: Install package with defaults
       dnf:
@@ -542,60 +517,3 @@
       dnf:
         name: provides_foo*
         state: absent
-
-- name: test that only evr is compared, avoiding a situation when a specific arch would be considered as a "newer" package
-  block:
-    - dnf:
-        name: "{{ item }}"
-        state: present
-      loop:
-        - "dinginessentail-1.0-1.x86_64"
-        - "dinginessentail-1.0-1.i686"
-      register: dnf_results
-
-    - assert:
-        that:
-          - dnf_results["results"][0] is changed
-          - dnf_results["results"][1] is changed
-  always:
-    - name: Clean up
-      dnf:
-        name: dinginessentail
-        state: absent
-
-- block:
-  - name: make sure dinginessentail is not installed
-    dnf:
-      name: dinginessentail
-      state: absent
-
-  - name: install dinginessentail both archs
-    dnf:
-      name:
-        - "{{ repodir }}/dinginessentail-1.1-1.x86_64.rpm"
-        - "{{ repodir_i686 }}/dinginessentail-1.1-1.i686.rpm"
-      state: present
-      disable_gpg_check: true
-
-  - name: try to install lower version of dinginessentail from rpm file, without allow_downgrade, just one arch
-    dnf:
-      name: "{{ repodir_i686 }}/dinginessentail-1.0-1.i686.rpm"
-      state: present
-    register: dnf_result
-
-  - name: check dinginessentail with rpm
-    shell: rpm -q dinginessentail
-    register: rpm_result
-
-  - name: verify installation
-    assert:
-      that:
-          - "not dnf_result.changed"
-          - "rpm_result.stdout_lines[0].startswith('dinginessentail-1.1-1')"
-          - "rpm_result.stdout_lines[1].startswith('dinginessentail-1.1-1')"
-  always:
-    - name: Clean up
-      dnf:
-        name: dinginessentail
-        state: absent
-  when: ansible_architecture == "x86_64"

test/integration/targets/dnf/tasks/skip_broken_and_nobest.yml

@@ -3,20 +3,9 @@
 #
 # There are a lot of fairly complex, corner cases we test here especially towards the end.
 #
-# The test repo is generated from the "skip-broken" repo in this repository:
-# https://github.com/relrod/ansible-ci-contrived-yum-repos
+# The test repo is generated by the setup_rpm_repo role.
 #
-# It is laid out like this:
-#
-# There are three packages, `broken-a`, `broken-b`, `broken-c`.
-#
-# * broken-a has three versions: 1.2.3, 1.2.3.4, 1.2.4, 2.0.0.
-#   * 1.2.3 and 1.2.4 have no dependencies
-#   * 1.2.3.4 and 2.0.0 both depend on a non-existent package (to break depsolving)
-#
-# * broken-b depends on broken-a-1.2.3
-# * broken-c depends on broken-a-1.2.4
-# * broken-d depends on broken-a (no version constraint)
+# See test/integration/targets/setup_rpm_repo/library/create_repo.py for how the repo is laid out.
 #
 # This allows us to test various upgrades, downgrades, and installs with broken dependencies.
 # skip_broken should usually be successful in the upgrade/downgrade case, it will just do nothing.
@@ -26,14 +15,6 @@
 # will try to install 2.0.0 which is broken. With nobest=true, it will fall back to 1.2.4. Similar
 # for upgrading.
 - block:
-    - name: Set up test yum repo
-      yum_repository:
-        name: skip-broken
-        description: ansible-test skip-broken test repo
-        baseurl: "{{ skip_broken_repo_baseurl }}"
-        gpgcheck: no
-        repo_gpgcheck: no
-
     - name: Install two packages
       dnf:
         name:
@@ -240,8 +221,7 @@
     - name: Do an "upgrade" to an older version of broken-a, allow_downgrade=false
       dnf:
         name:
-          #- broken-a-1.2.3-1*
-          - broken-a-1.2.3-1.el7.x86_64
+          - broken-a-1.2.3-1.noarch
         state: latest
         allow_downgrade: false
       check_mode: true
@@ -327,11 +307,6 @@
       when: not dnf5|default(false)
 
   always:
-    - name: Remove test yum repo
-      yum_repository:
-        name: skip-broken
-        state: absent
-
     - name: Remove all test packages installed
       dnf:
         name:

test/integration/targets/dnf/vars/main.yml

@@ -2,5 +2,3 @@ dnf_log_files:
     - /var/log/dnf.log
     - /var/log/dnf.rpm.log
     - /var/log/dnf.librepo.log
-
-skip_broken_repo_baseurl: "https://ci-files.testing.ansible.com/test/integration/targets/dnf/skip-broken/RPMS/"

test/integration/targets/setup_rpm_repo/library/create_repo.py

@@ -4,71 +4,74 @@ from __future__ import annotations
 
 import tempfile
 
-from collections import namedtuple
+from dataclasses import dataclass
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.common.respawn import has_respawned, probe_interpreters_for_module, respawn_module
 
 HAS_RPMFLUFF = True
-can_use_rpm_weak_deps = None
+
 try:
-    from rpmfluff import SimpleRpmBuild, GeneratedSourceFile, make_gif
-    from rpmfluff import YumRepoBuild
+    from rpmfluff.make import make_gif
+    from rpmfluff.sourcefile import GeneratedSourceFile
+    from rpmfluff.rpmbuild import SimpleRpmBuild
+    from rpmfluff.yumrepobuild import YumRepoBuild
 except ImportError:
-    try:
-        from rpmfluff.make import make_gif
-        from rpmfluff.sourcefile import GeneratedSourceFile
-        from rpmfluff.rpmbuild import SimpleRpmBuild
-        from rpmfluff.yumrepobuild import YumRepoBuild
-    except ImportError:
-        HAS_RPMFLUFF = False
-
-can_use_rpm_weak_deps = None
-if HAS_RPMFLUFF:
-    try:
-        from rpmfluff import can_use_rpm_weak_deps
-    except ImportError:
-        try:
-            from rpmfluff.utils import can_use_rpm_weak_deps
-        except ImportError:
-            pass
+    HAS_RPMFLUFF = False
 
 
-RPM = namedtuple('RPM', ['name', 'version', 'release', 'epoch', 'recommends', 'file', 'arch'])
+@dataclass
+class RPM:
+    name: str
+    version: str
+    release: str = '1'
+    epoch: int = 0
+    arch: list[str] | None = None
+    recommends: list[str] | None = None
+    requires: list[str] | None = None
+    file: str | None = None
+
 
 SPECS = [
-    RPM('dinginessentail', '1.0', '1', None, None, None, None),
-    RPM('dinginessentail', '1.0', '2', '1', None, None, None),
-    RPM('dinginessentail', '1.1', '1', '1', None, None, None),
-    RPM('dinginessentail-olive', '1.0', '1', None, None, None, None),
-    RPM('dinginessentail-olive', '1.1', '1', None, None, None, None),
-    RPM('landsidescalping', '1.0', '1', None, None, None, None),
-    RPM('landsidescalping', '1.1', '1', None, None, None, None),
-    RPM('dinginessentail-with-weak-dep', '1.0', '1', None, ['dinginessentail-weak-dep'], None, None),
-    RPM('dinginessentail-weak-dep', '1.0', '1', None, None, None, None),
-    RPM('noarchfake', '1.0', '1', None, None, None, 'noarch'),
-    RPM('provides_foo_a', '1.0', '1', None, None, 'foo.gif', 'noarch'),
-    RPM('provides_foo_b', '1.0', '1', None, None, 'foo.gif', 'noarch'),
-    RPM('number-11-name', '11.0', '1', None, None, None, None),
-    RPM('number-11-name', '11.1', '1', None, None, None, None),
-    RPM('epochone', '1.0', '1', '1', None, None, "noarch"),
-    RPM('epochone', '1.1', '1', '1', None, None, "noarch"),
+    RPM(name='dinginessentail', version='1.0'),
+    RPM(name='dinginessentail', version='1.0', release='2', epoch=1),
+    RPM(name='dinginessentail', version='1.1', epoch=1),
+    RPM(name='dinginessentail-olive', version='1.0'),
+    RPM(name='dinginessentail-olive', version='1.1'),
+    RPM(name='multilib-dinginessentail', version='1.0', arch=['i686', 'x86_64']),
+    RPM(name='multilib-dinginessentail', version='1.1', arch=['i686', 'x86_64']),
+    RPM(name='landsidescalping', version='1.0',),
+    RPM(name='landsidescalping', version='1.1',),
+    RPM(name='dinginessentail-with-weak-dep', version='1.0', recommends=['dinginessentail-weak-dep']),
+    RPM(name='dinginessentail-weak-dep', version='1.0',),
+    RPM(name='noarchfake', version='1.0'),
+    RPM(name='provides_foo_a', version='1.0', file='foo.gif'),
+    RPM(name='provides_foo_b', version='1.0', file='foo.gif'),
+    RPM(name='number-11-name', version='11.0',),
+    RPM(name='number-11-name', version='11.1',),
+    RPM(name='epochone', version='1.0', epoch=1),
+    RPM(name='epochone', version='1.1', epoch=1),
+    RPM(name='broken-a', version='1.2.3',),
+    RPM(name='broken-a', version='1.2.3.4', requires=['dinginessentail-doesnotexist']),
+    RPM(name='broken-a', version='1.2.4',),
+    RPM(name='broken-a', version='2.0.0', requires=['dinginessentail-doesnotexist']),
+    RPM(name='broken-b', version='1.0', requires=['broken-a = 1.2.3-1']),
+    RPM(name='broken-c', version='1.0', requires=['broken-c = 1.2.4-1']),
+    RPM(name='broken-d', version='1.0', requires=['broken-a']),
 ]
 
 
-def create_repo(arch='x86_64'):
+def create_repo():
     pkgs = []
     for spec in SPECS:
-        pkg = SimpleRpmBuild(spec.name, spec.version, spec.release, [spec.arch or arch])
+        pkg = SimpleRpmBuild(spec.name, spec.version, spec.release, spec.arch or ['noarch'])
         pkg.epoch = spec.epoch
 
-        if spec.recommends:
-            # Skip packages that require weak deps but an older version of RPM is being used
-            if not can_use_rpm_weak_deps or not can_use_rpm_weak_deps():
-                continue
+        for requires in spec.requires or []:
+            pkg.add_requires(requires)
 
-            for recommend in spec.recommends:
-                pkg.add_recommends(recommend)
+        for recommend in spec.recommends or []:
+            pkg.add_recommends(recommend)
 
         if spec.file:
             pkg.add_installed_file(
@@ -81,7 +84,7 @@ def create_repo(arch='x86_64'):
         pkgs.append(pkg)
 
     repo = YumRepoBuild(pkgs)
-    repo.make(arch, 'noarch')
+    repo.make('noarch', 'i686', 'x86_64')
 
     for pkg in pkgs:
         pkg.clean()
@@ -92,7 +95,6 @@ def create_repo(arch='x86_64'):
 def main():
     module = AnsibleModule(
         argument_spec={
-            'arch': {'required': True},
             'tempdir': {'type': 'path'},
         }
     )
@@ -107,7 +109,6 @@ def main():
 
         respawn_module(interpreter)
 
-    arch = module.params['arch']
     tempdir = module.params['tempdir']
 
     # Save current temp dir so we can set it back later
@@ -115,7 +116,7 @@ def main():
     tempfile.tempdir = tempdir
 
     try:
-        repo_dir = create_repo(arch)
+        repo_dir = create_repo()
     finally:
         tempfile.tempdir = original_tempdir
 

test/integration/targets/setup_rpm_repo/tasks/main.yml

@@ -1,47 +1,24 @@
 - block:
-  - name: Include distribution specific variables
-    include_vars: "{{ lookup('first_found', params) }}"
-    vars:
-      params:
-        files:
-          - "{{ ansible_facts.distribution }}-{{ ansible_facts.distribution_version }}.yml"
-          - "{{ ansible_facts.os_family }}-{{ ansible_facts.distribution_major_version }}.yml"
-          - "{{ ansible_facts.distribution }}.yml"
-          - "{{ ansible_facts.os_family }}.yml"
-          - default.yml
-        paths:
-          - "{{ role_path }}/vars"
-
-  - name: Install rpmfluff and deps
-    action: "{{ ansible_facts.pkg_mgr }}"
-    args:
-      name: "{{ rpm_repo_packages }}"
+  - name: Install deps
+    dnf:
+       name:
+         - python3-pip
+         - createrepo_c
+         - rpm-build
 
   - name: Install rpmfluff via pip, ensure it is installed with default python as python3-rpm may not exist for other versions
-    block:
-      - action: "{{ ansible_facts.pkg_mgr }}"
-        args:
-           name:
-             - python3-pip
-             - python3
-           state: latest
-
-      - pip:
-          name: rpmfluff
-          executable: pip3
-    when: ansible_facts.os_family == 'RedHat' and ansible_distribution_major_version is version('9', '==')
+    pip:
+      name: rpmfluff
+      executable: pip3
 
   - set_fact:
       repos:
-        - "fake-{{ ansible_architecture }}"
-        - "fake-i686"
-        - "fake-ppc64"
+        - "dummy-repo"
     changed_when: yes
     notify: remove repos
 
   - name: Create RPMs and put them into a repo
     create_repo:
-      arch: "{{ ansible_architecture }}"
       tempdir: "{{ remote_tmp_dir }}"
     register: repo
 
@@ -50,8 +27,8 @@
 
   - name: Install the repo
     yum_repository:
-      name: "fake-{{ ansible_architecture }}"
-      description: "fake-{{ ansible_architecture }}"
+      name: "dummy-repo"
+      description: "dummy-repo"
       baseurl: "file://{{ repodir }}"
       gpgcheck: no
     when: install_repos | bool
@@ -65,38 +42,4 @@
   - name: Register comps.xml on repo
     command: createrepo -g {{ repodir_comps.dest | quote }} {{ repodir | quote }}
 
-  - name: Create RPMs and put them into a repo (i686)
-    create_repo:
-      arch: i686
-      tempdir: "{{ remote_tmp_dir }}"
-    register: repo_i686
-
-  - set_fact:
-      repodir_i686: "{{ repo_i686.repo_dir }}"
-
-  - name: Install the repo (i686)
-    yum_repository:
-      name: "fake-i686"
-      description: "fake-i686"
-      baseurl: "file://{{ repodir_i686 }}"
-      gpgcheck: no
-    when: install_repos | bool
-
-  - name: Create RPMs and put them into a repo (ppc64)
-    create_repo:
-      arch: ppc64
-      tempdir: "{{ remote_tmp_dir }}"
-    register: repo_ppc64
-
-  - set_fact:
-      repodir_ppc64: "{{ repo_ppc64.repo_dir }}"
-
-  - name: Install the repo (ppc64)
-    yum_repository:
-      name: "fake-ppc64"
-      description: "fake-ppc64"
-      baseurl: "file://{{ repodir_ppc64 }}"
-      gpgcheck: no
-    when: install_repos | bool
-
-  when: ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux', 'Fedora']
+  when: ansible_distribution in ['RedHat', 'Fedora']

test/integration/targets/setup_rpm_repo/vars/Fedora.yml

@@ -1,4 +0,0 @@
-rpm_repo_packages:
-  - "{{ 'python' ~ rpm_repo_python_major_version ~ '-rpmfluff' }}"
-  - createrepo
-  - rpm-build

test/integration/targets/setup_rpm_repo/vars/RedHat-6.yml

@@ -1,5 +0,0 @@
-rpm_repo_packages:
-  - rpm-build
-  - python-rpmfluff
-  - createrepo_c
-  - createrepo

test/integration/targets/setup_rpm_repo/vars/RedHat-7.yml

@@ -1,5 +0,0 @@
-rpm_repo_packages:
-  - rpm-build
-  - python-rpmfluff
-  - createrepo_c
-  - createrepo

test/integration/targets/setup_rpm_repo/vars/RedHat-8.yml

@@ -1,5 +0,0 @@
-rpm_repo_packages:
-  - rpm-build
-  - createrepo_c
-  - createrepo
-  - python3-rpmfluff

test/integration/targets/setup_rpm_repo/vars/RedHat-9.yml

@@ -1,4 +0,0 @@
-rpm_repo_packages:
-  - rpm-build
-  - createrepo_c
-  - createrepo

test/integration/targets/setup_rpm_repo/vars/main.yml

@@ -1 +0,0 @@
-rpm_repo_python_major_version: "{{ ansible_facts.python_version.split('.')[0] }}"