Populate optional comps.xml unit fields [RHELDST-25995]

src/pubtools/pulplib/_impl/comps.py

@@ -5,6 +5,34 @@
 from io import StringIO
 
 
+# All optional fields in the pulp-rpm comps.xml units
+COMMON_FIELDS = [
+    "description",
+    "translated_description",
+    "display_order",
+    "translated_name",
+    "name",
+]
+UNIT_FIELD_DEFAULTS = {
+    "package_group": (
+        {"default": False, "user_visible": False},
+        [
+            "default_package_names",
+            "optional_package_names",
+            "mandatory_package_names",
+            "default",
+            "user_visible",
+            "langonly",
+            "conditional_package_names",
+        ]
+        + COMMON_FIELDS,
+    ),
+    "package_category": ({}, ["packagegroupids"] + COMMON_FIELDS),
+    "package_environment": ({}, ["group_ids", "options"] + COMMON_FIELDS),
+    "package_langpacks": ({}, ["matches"]),
+}
+
+
 class BooleanStringIO(StringIO):
     """A StringIO which coerces the output value into a boolean."""
 
@@ -356,4 +384,21 @@ def units_for_xml(io):
     modules.
     """
     parser = CompsParser()
-    return parser.parse(io)
+    return fill_unit_field_defaults(parser.parse(io))
+
+
+def fill_unit_field_defaults(units):
+    """
+    Fill in missing comp unit fields with default values.
+
+    Arguments:
+    units (dict)
+        Parsed comps.xml file.
+    """
+
+    for unit in units:
+        defaults, fields = UNIT_FIELD_DEFAULTS[unit["_content_type_id"]]
+        for field in fields:
+            unit[field] = unit.get(field, defaults.get(field))
+
+    return units

tests/comps/test_comps_parse.py

@@ -2,7 +2,7 @@
 
 import os
 
-from pubtools.pulplib._impl.comps import units_for_xml
+from pubtools.pulplib._impl.comps import units_for_xml, fill_unit_field_defaults
 
 # Don't use autoformatting in this file because we use u'' string literals
 # at least until py2 support is dropped, and black wants to remove them...
@@ -38,6 +38,10 @@ def test_can_parse_units(data_path):
         },
         "translated_name": {"af": "3D-drukwerk", "bg": u"3D Печатане"},
         "user_visible": True,
+        "optional_package_names": None,
+        "langonly": None,
+        "mandatory_package_names": None,
+        "display_order": None,
     }
 
     assert units[1] == {
@@ -53,6 +57,11 @@ def test_can_parse_units(data_path):
         "default": False,
         "user_visible": True,
         "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"],
+        "conditional_package_names": None,
+        "default_package_names": None,
+        "display_order": None,
+        "langonly": None,
+        "optional_package_names": None,
     }
 
     assert units[2] == {
@@ -115,6 +124,7 @@ def test_can_parse_units(data_path):
             "es": u"Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.",
         },
         "display_order": 22,
+        "description": None,
         "group_ids": ["input-methods", "multimedia"],
         "options": [{"default": False, "group": "libreoffice"}],
     }
@@ -128,3 +138,113 @@ def test_can_parse_units(data_path):
             {"install": "tkgate-%s", "name": "tkgate"},
         ],
     }
+
+
+def test_parse_populates_default_fields(data_path):
+    """units_for_xml parses comps.xml data and can populate fields with
+       default values."""
+
+    xml_path = os.path.join(data_path, "sparse-comps.xml")
+
+    with open(xml_path, "rb") as f:
+        units = units_for_xml(f)
+
+    assert len(units) == 8
+
+    assert units[0] == {"_content_type_id": "package_group", "id": "3d-printing",
+            "name": "3D Printing",
+            "translated_name": {"af": "3D-drukwerk", "bg": "3D Печатане"},
+            "user_visible": True, "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None}
+    assert units[1] == {"_content_type_id": "package_group", "id": "admin-tools",
+            "name": "Administration Tools",
+            "translated_name": {"af": "Administrasienutsgoed",
+                                "am": "የአስተዳደሩ መሣሪያዎች"},
+            "description": "This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.",
+            "translated_description": {
+                "sr": "Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.",
+                "sr@Latn": "Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu."},
+            "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"],
+            "default_package_names": None, "optional_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "display_order": None}
+    assert units[2] == {"_content_type_id": "package_category",
+            "id": "kde-desktop-environment", "name": "KDE Desktop",
+            "translated_name": {"af": "KDE-werkskerm", "as": "KDE ডেস্কটপ"},
+            "packagegroupids": ["kde-office", "kde-telepathy"],
+            "description": None, "translated_description": None,
+            "display_order": None}
+    assert units[3] == {
+        "_content_type_id": "package_category",
+        "id": "xfce-desktop-environment",
+        "name": "Xfce Desktop",
+        "translated_name": {"uk": u"Графічне середовище Xfce", "zh_CN": u"Xfce 桌面环境"},
+        "description": "A lightweight desktop environment that works well on low end machines.",
+        "translated_description": {
+            "as": u"এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।",
+            "ast": u"Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.",
+        },
+        "display_order": 15,
+        "packagegroupids": ["xfce-apps", "xfce-desktop"],
+    }
+    assert units[4] == {"_content_type_id": "package_environment",
+            "id": "basic-desktop-environment", "name": "Basic Desktop",
+            "description": "X Window System with a choice of window manager.",
+            "translated_description": {
+                "af": "X Window-stelsel met ’n keuse van vensterbestuurder.",
+                "bg": "X Window система с избор на мениджър на прозорци."},
+            "display_order": None,
+            "group_ids": ["networkmanager-submodules", "standard"],
+            "options": None, "translated_name": None}
+
+    assert units[5] == {"_content_type_id": "package_environment",
+            "id": "cinnamon-desktop-environment", "name": "Cinnamon Desktop",
+            "translated_name": {"en_GB": "Cinnamon Desktop",
+                                "fr": "Bureau Cinnamon"},
+            "translated_description": {
+                "ca": "Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.",
+                "es": "Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible."},
+            "display_order": 22, "group_ids": ["input-methods", "multimedia"],
+            "options": [{"group": "libreoffice", "default": False}],
+            "description": None}
+
+    assert units[6] == {"_content_type_id": "package_langpacks",
+            "matches": [{"install": "stardict-dic-%s", "name": "stardict"},
+                        {"install": "tagainijisho-dic-%s",
+                         "name": "tagainijisho-common"},
+                        {"install": "tesseract-langpack-%s",
+                         "name": "tesseract"},
+                        {"install": "tkgate-%s", "name": "tkgate"}]}
+
+    assert units[7] ==  {"_content_type_id": "package_langpacks", "matches": None}
+
+
+
+def test_fill_unit_field_defaults():
+    """Missing fields are filled in with expected values."""
+    units = [{"_content_type_id": "package_group"},
+             {"_content_type_id": "package_category"},
+             {"_content_type_id": "package_environment"},
+             {"_content_type_id": "package_langpacks"}]
+    expected = [{"_content_type_id": "package_group", "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_category", "packagegroupids": None,
+            "description": None, "translated_description": None,
+            "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_environment", "group_ids": None,
+            "options": None, "description": None,
+            "translated_description": None,
+            "display_order": None, "translated_name": None, "name": None},
+           {"_content_type_id": "package_langpacks", "matches": None}]
+
+    units = fill_unit_field_defaults(units)
+
+    assert units == expected

tests/data/sparse-comps.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE comps
+  PUBLIC '-//Red Hat, Inc.//DTD Comps info//EN'
+  'comps.dtd'>
+
+<!--
+  A comps file for testing.
+
+  This was originally copied from sample-comps.xml, but trimmed down to
+  test field population
+-->
+
+<comps>
+
+
+  <group>
+    <id>3d-printing</id>
+    <name>3D Printing</name>
+    <name xml:lang="af">3D-drukwerk</name>
+    <name xml:lang="bg">3D Печатане</name>
+    <uservisible>true</uservisible>
+  </group>
+  <group>
+    <id>admin-tools</id>
+    <name>Administration Tools</name>
+    <name xml:lang="af">Administrasienutsgoed</name>
+    <name xml:lang="am">የአስተዳደሩ መሣሪያዎች</name>
+    <description>This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.</description>
+    <description xml:lang="sr">Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.</description>
+    <description xml:lang="sr@Latn">Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu.</description>
+    <packagelist>
+      <packagereq type="mandatory">abrt-desktop</packagereq>
+      <packagereq type="mandatory">gnome-disk-utility</packagereq>
+    </packagelist>
+  </group>
+
+
+  <category>
+    <id>kde-desktop-environment</id>
+    <name>KDE Desktop</name>
+    <name xml:lang="af">KDE-werkskerm</name>
+    <name xml:lang="as">KDE ডেস্কটপ</name>
+    <grouplist>
+      <groupid>kde-office</groupid>
+      <groupid>kde-telepathy</groupid>
+    </grouplist>
+  </category>
+  <category>
+    <id>xfce-desktop-environment</id>
+    <name>Xfce Desktop</name>
+    <name xml:lang="uk">Графічне середовище Xfce</name>
+    <name xml:lang="zh_CN">Xfce 桌面环境</name>
+    <description>A lightweight desktop environment that works well on low end machines.</description>
+    <description xml:lang="as">এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।</description>
+    <description xml:lang="ast">Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.</description>
+    <display_order>15</display_order>
+    <grouplist>
+      <groupid>xfce-apps</groupid>
+      <groupid>xfce-desktop</groupid>
+    </grouplist>
+  </category>
+
+
+  <environment>
+    <id>basic-desktop-environment</id>
+    <name>Basic Desktop</name>
+    <description>X Window System with a choice of window manager.</description>
+    <description xml:lang="af">X Window-stelsel met ’n keuse van vensterbestuurder.</description>
+    <description xml:lang="bg">X Window система с избор на мениджър на прозорци.</description>
+    <display_order><!-- tweaked: try empty display order --></display_order>
+    <grouplist>
+      <groupid>networkmanager-submodules</groupid>
+      <groupid>standard</groupid>
+    </grouplist>
+  </environment>
+  <environment>
+    <id>cinnamon-desktop-environment</id>
+    <name>Cinnamon Desktop</name>
+    <name xml:lang="en_GB">Cinnamon Desktop</name>
+    <name xml:lang="fr">Bureau Cinnamon</name>
+    <description xml:lang="ca">Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.</description>
+    <description xml:lang="es">Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.</description>
+    <display_order>22</display_order>
+    <grouplist>
+      <groupid>input-methods</groupid>
+      <groupid>multimedia</groupid>
+    </grouplist>
+    <optionlist>
+      <groupid>libreoffice</groupid>
+    </optionlist>
+  </environment>
+
+  <langpacks>
+    <match install="stardict-dic-%s" name="stardict"/>
+    <match install="tagainijisho-dic-%s" name="tagainijisho-common"/>
+    <match install="tesseract-langpack-%s" name="tesseract"/>
+    <match install="tkgate-%s" name="tkgate"/>
+  </langpacks>
+  <langpacks>
+    <!-- An empty langpack doesn't really make much sense hey-->
+  </langpacks>
+</comps>

tests/repository/test_upload_comps.py

@@ -136,37 +136,50 @@ def test_upload_comps_xml(client, requests_mocker):
     imported_units.sort(key=lambda u: u["_content_type_id"])
 
     # Did we import exactly the expected units?
+
     assert imported_units == [
         {
             "_content_type_id": "package_category",
+            "description": None,
+            "display_order": 10,
             "id": "kde-desktop-environment",
             "name": "KDE Desktop",
-            "display_order": 10,
             "packagegroupids": ["kde-office", "kde-telepathy"],
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": None,
         },
         {
             "_content_type_id": "package_environment",
-            "id": "basic-desktop-environment",
-            "name": "Basic Desktop",
-            "translated_name": {"af": "Basiese werkskerm"},
             "description": "X Window System with a choice of window manager.",
+            "display_order": None,
             "group_ids": ["networkmanager-submodules", "standard"],
+            "id": "basic-desktop-environment",
+            "name": "Basic Desktop",
             "options": [
-                {"group": "xmonad", "default": True},
-                {"group": "xmonad-mate", "default": False},
+                {"default": True, "group": "xmonad"},
+                {"default": False, "group": "xmonad-mate"},
             ],
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": {"af": "Basiese werkskerm"},
         },
         {
             "_content_type_id": "package_group",
-            "id": "3d-printing",
-            "name": "3D Printing",
-            "translated_name": {"af": "3D-drukwerk"},
-            "description": "3D printing software",
+            "conditional_package_names": None,
+            "default": False,
             "default_package_names": ["admesh"],
+            "description": "3D printing software",
+            "display_order": None,
+            "id": "3d-printing",
+            "langonly": None,
             "mandatory_package_names": ["blender"],
+            "name": "3D Printing",
+            "optional_package_names": None,
             "repo_id": "repo1",
+            "translated_description": None,
+            "translated_name": {"af": "3D-drukwerk"},
+            "user_visible": False,
         },
         {
             "_content_type_id": "package_langpacks",