Add type to addstream, ingest addons and allow to query them

app/apps.py

@@ -10,32 +10,44 @@ def load_appstream():
 
     current_apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
     current_categories = db.redis_conn.smembers("categories:index")
+    current_types = db.redis_conn.smembers("types:index")
 
     db.initialize()
     with db.redis_conn.pipeline() as p:
         p.delete("categories:index", *current_categories)
+        p.delete("types:index", *current_types)
 
         for appid in apps:
             redis_key = f"apps:{appid}"
 
-            clean_html_re = re.compile("<.*?>")
-            search_description = re.sub(clean_html_re, "", apps[appid]["description"])
+            if apps[appid].get("type") == "desktop":
+                clean_html_re = re.compile("<.*?>")
+                search_description = re.sub(
+                    clean_html_re, "", apps[appid]["description"]
+                )
+
+                if search_keywords := apps[appid].get("keywords"):
+                    search_keywords = " ".join(search_keywords)
+                else:
+                    search_keywords = ""
+
+                fts = {
+                    "id": appid,
+                    "name": apps[appid]["name"],
+                    "summary": apps[appid]["summary"],
+                    "description": search_description,
+                    "keywords": search_keywords,
+                }
+                p.hset(f"fts:{appid}", mapping=fts)
 
-            if search_keywords := apps[appid].get("keywords"):
-                search_keywords = " ".join(search_keywords)
-            else:
-                search_keywords = ""
+            p.set(f"apps:{appid}", json.dumps(apps[appid]))
 
-            fts = {
-                "id": appid,
-                "name": apps[appid]["name"],
-                "summary": apps[appid]["summary"],
-                "description": search_description,
-                "keywords": search_keywords,
-            }
+            if type := apps[appid].get("type"):
+                p.sadd("types:index", type)
+                p.sadd(f"types:{type}", redis_key)
 
-            p.set(f"apps:{appid}", json.dumps(apps[appid]))
-            p.hset(f"fts:{appid}", mapping=fts)
+            if extends := apps[appid].get("extends"):
+                p.sadd(f"addons:{extends}", redis_key)
 
             if categories := apps[appid].get("categories"):
                 for category in categories:
@@ -48,6 +60,7 @@ def load_appstream():
                 f"fts:{appid}",
                 f"summary:{appid}",
                 f"app_stats:{appid}",
+                f"addons:{appid}",
             )
             db.redis_search.delete_document(f"fts:{appid}")
 
@@ -62,8 +75,8 @@ def load_appstream():
     return new_apps
 
 
-def list_appstream(repo: str = "stable"):
-    apps = {app[5:] for app in db.redis_conn.smembers("apps:index")}
+def list_appstream(repo: str = "stable", type: str = "desktop"):
+    apps = {app[5:] for app in db.redis_conn.smembers(f"types:{type}")}
     return sorted(apps)
 
 
@@ -82,6 +95,16 @@ def get_category(category: str):
         return []
 
 
+def get_addons(appid: str):
+    if index := db.redis_conn.smembers(f"addons:{appid}"):
+        json_appdata = db.redis_conn.mget(index)
+        appdata = [json.loads(app) for app in json_appdata]
+
+        return [(app["id"]) for app in appdata]
+    else:
+        return []
+
+
 def search(query: str):
     if results := db.search(query):
         appids = tuple(doc_id.replace("fts", "apps") for doc_id in results)

app/db.py

@@ -127,4 +127,4 @@ def get_json_key(key: str):
 
 
 def get_app_count():
-    return redis_conn.scard("apps:index")
+    return redis_conn.scard("types:desktop")

app/main.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from functools import lru_cache
 
 import sentry_sdk
@@ -53,6 +54,21 @@ def get_category(category: schemas.Category):
     return sorted_ids
 
 
+@app.get("/addon/{appid}")
+def get_addons(appid: str):
+    ids = apps.get_addons(appid)
+
+    addon_appstreams = defaultdict()
+    for addonid in ids:
+        addon_appstreams[addonid] = db.get_json_key(f"apps:{addonid}")
+    sorted_ids = sorted(
+        ids,
+        key=lambda appid: addon_appstreams[appid].get("name", "Unknown"),
+    )
+
+    return sorted_ids
+
+
 @app.get("/appstream")
 def list_appstream():
     return apps.list_appstream()

app/utils.py

@@ -35,9 +35,14 @@ def appstream2dict(reponame: str):
     for component in root:
         app = {}
 
-        if component.attrib.get("type") != "desktop":
+        if (
+            component.attrib.get("type") != "desktop"
+            and component.attrib.get("type") != "addon"
+        ):
             continue
 
+        app["type"] = component.attrib.get("type")
+
         descriptions = component.findall("description")
         if len(descriptions):
             for desc in descriptions:

tests/results/test_appstream_by_appid.json

@@ -1,4 +1,5 @@
 {
+  "type": "desktop",
   "description": "<p>Good at finding your way out of hard situations? Let's see! Try to make your way through an increasingly difficult path.</p>",
   "screenshots": [
     {
@@ -42,8 +43,12 @@
   "name": "Maze",
   "summary": "A simple maze game",
   "developer_name": "Sugar Labs Community",
-  "categories": ["Game"],
-  "kudos": ["HiDpiIcon"],
+  "categories": [
+    "Game"
+  ],
+  "kudos": [
+    "HiDpiIcon"
+  ],
   "project_license": "GPL-3.0-or-later",
   "launchable": {
     "value": "org.sugarlabs.Maze.desktop",
@@ -55,4 +60,4 @@
     "runtime": "org.gnome.Platform/x86_64/3.36",
     "sdk": "org.gnome.Sdk/x86_64/3.36"
   }
-}
+}