From abffe2f3e040b1a8727a4ebd4321811d0ecbb826 Mon Sep 17 00:00:00 2001
From: jarbasal <jarbasai@mailfence.com>
Date: Wed, 8 Jul 2020 02:44:21 +0100
Subject: [PATCH] appstores #0

---
 examples/appstores.py                | 26 +++++++++++
 msm/appstores/__init__.py            |  8 ++++
 msm/appstores/mycroft_marketplace.py | 14 ++++++
 msm/appstores/pling.py               | 63 +++++++++++++++++++++++++++
 msm/skill_entry.py                   | 33 +++++++++++---
 msm/util.py                          | 65 +++++++++++++++++++++++++++-
 requirements.txt                     |  2 +
 setup.py                             |  2 +-
 8 files changed, 205 insertions(+), 8 deletions(-)
 create mode 100644 examples/appstores.py
 create mode 100644 msm/appstores/__init__.py
 create mode 100644 msm/appstores/mycroft_marketplace.py
 create mode 100644 msm/appstores/pling.py

diff --git a/examples/appstores.py b/examples/appstores.py
new file mode 100644
index 0000000..381dd29
--- /dev/null
+++ b/examples/appstores.py
@@ -0,0 +1,26 @@
+from msm.appstores import pling, mycroft_marketplace
+from msm import appstores
+from msm import MycroftSkillsManager
+
+msm = MycroftSkillsManager()
+
+
+print("Searching pling appstore for hivemind")
+for s in pling.search("hivemind", msm=msm):
+    print(s.name)  # pling appstore only
+
+print("Searching mycroft marketplace for jokes")
+for s in mycroft_marketplace.search("jokes", msm=msm):
+    print(s.name)  # mycroft appstore only
+
+print("Searching everywhere for node red")
+for s in appstores.search("node red", min_conf=0.4, msm=msm):
+    print(s.name)  # All appstores
+
+print("Listing pling appstore skills")
+for s in pling.list_skills(msm=msm):
+    print(s.name)
+
+print("Listing mycroft marketplace skills")
+for s in mycroft_marketplace.list_skills(msm=msm):
+    print(s.name)
diff --git a/msm/appstores/__init__.py b/msm/appstores/__init__.py
new file mode 100644
index 0000000..1f1375f
--- /dev/null
+++ b/msm/appstores/__init__.py
@@ -0,0 +1,8 @@
+from msm.appstores import pling, mycroft_marketplace
+
+
+def search(*args, **kwargs):
+    for skill in mycroft_marketplace.search(*args, **kwargs):
+        yield skill
+    for skill in pling.search(*args, **kwargs):
+        yield skill
diff --git a/msm/appstores/mycroft_marketplace.py b/msm/appstores/mycroft_marketplace.py
new file mode 100644
index 0000000..21ea724
--- /dev/null
+++ b/msm/appstores/mycroft_marketplace.py
@@ -0,0 +1,14 @@
+from msm import MycroftSkillsManager
+
+
+def search(name, author=None, msm=None, min_conf=0.3):
+    for skill in list_skills(msm):
+        if skill.match(name, author) >= min_conf:
+            yield skill
+
+
+def list_skills(msm=None):
+    msm = msm or MycroftSkillsManager()
+    for skill in msm.list():
+        yield skill
+
diff --git a/msm/appstores/pling.py b/msm/appstores/pling.py
new file mode 100644
index 0000000..e734b91
--- /dev/null
+++ b/msm/appstores/pling.py
@@ -0,0 +1,63 @@
+import requests
+from msm import MycroftSkillsManager
+from msm.util import xml2dict
+from msm.skill_entry import SkillEntry
+import json
+
+
+def _parse_pling(skill, msm):
+
+    if isinstance(skill, str):
+        json_data = json.loads(skill)
+    else:
+        json_data = skill
+
+    # TODO is it a safe assumption downloadlink1 is always the skill.json ?
+    # this can be made smarter
+    url = json_data["downloadlink1"]
+    skill_json = requests.get(url).json()
+
+    # save useful data to skill.meta_info
+    skill_json["category"] = json_data['typename']
+    skill_json["created"] = json_data['created']
+    skill_json["modified"] = json_data['changed']
+    skill_json["description"] = json_data["description"]
+    skill_json["tags"] = json_data['tags'].split(",")
+    skill_json["author"] = json_data['personid']
+    skill_json["version"] = json_data["version"]
+
+    # appstore data
+    # TODO also provide this from mycroft appstore
+    skill_json["appstore"] = "pling.opendesktop"
+    skill_json["appurl"] = json_data["detailpage"]
+
+    return SkillEntry.from_json(skill_json, msm)
+
+
+def list_skills(msm=None):
+    msm = msm or MycroftSkillsManager()
+
+    url = "https://api.kde-look.org/ocs/v1/content/data"
+    params = {"categories": "608", "page": 0}
+    xml = requests.get(url, params=params).text
+
+    data = xml2dict(xml)
+    meta = data["ocs"]["meta"]
+    n_pages = int(meta["totalitems"]) // int(meta["itemsperpage"])
+
+    for skill in data["ocs"]["data"]["content"]:
+        yield _parse_pling(skill, msm)
+
+    for n in range(1, n_pages + 1):
+        params = {"categories": "608", "page": n}
+        xml = requests.get(url, params=params).text
+
+        for skill in xml2dict(xml)["ocs"]["data"]["content"]:
+            yield _parse_pling(skill, msm)
+
+
+def search(name, author=None, msm=None, min_conf=0.3):
+    msm = msm or MycroftSkillsManager()
+    for skill in list_skills(msm):
+        if skill.match(name, author) >= min_conf:
+            yield skill
diff --git a/msm/skill_entry.py b/msm/skill_entry.py
index 3170656..540bc52 100644
--- a/msm/skill_entry.py
+++ b/msm/skill_entry.py
@@ -20,7 +20,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import sys
-
+import json
 import logging
 import os
 import shutil
@@ -101,7 +101,7 @@ def wrapper(self, *args, **kwargs):
     return wrapper
 
 
-class SkillEntry(object):
+class SkillEntry:
     pip_lock = Lock()
     manifest_yml_format = {
         'dependencies': {
@@ -112,18 +112,18 @@ class SkillEntry(object):
         }
     }
 
-    def __init__(self, name, path, url='', sha='', msm=None):
+    def __init__(self, name, path, url='', sha='', msm=None, meta_info=None):
         url = url.rstrip('/')
         url = url[:-len('.git')] if url.endswith('.git') else url
         self.path = path
         self.url = url
         self.sha = sha
         self.msm = msm
-        if msm:
+        if msm and meta_info is None:
             u = url.lower()
             self.meta_info = msm.repo.skills_meta_info.get(u, {})
         else:
-            self.meta_info = {}
+            self.meta_info = meta_info or {}
         if name is not None:
             self.name = name
         elif 'name' in self.meta_info:
@@ -216,6 +216,29 @@ def from_folder(cls, path, msm=None, use_cache=True):
                 return skills[path]
         return cls(None, path, cls.find_git_url(path), msm=msm)
 
+    @classmethod
+    def from_json(cls, json_data, msm, use_cache=True):
+        """Find or create skill entry from folder path.
+
+        Arguments:
+            json_data:  skill.json  str or dict
+            msm:        msm instance to use for caching and extended information
+                        retrieval.
+            use_cache:  Enable/Disable cache usage. defaults to True
+        """
+        if isinstance(json_data, str):
+            json_data = json.loads(json_data)
+
+        name = json_data["skillname"]
+        url = json_data["url"]
+        path = cls.create_path(msm.skills_dir, url, name)
+
+        if use_cache:
+            skills = {skill.path: skill for skill in msm.local_skills.values()}
+            if path in skills:
+                return skills[path]
+        return cls(name, path, url, msm=msm, meta_info=json_data)
+
     @classmethod
     def create_path(cls, folder, url, name=''):
         return join(folder, '{}.{}'.format(
diff --git a/msm/util.py b/msm/util.py
index 9eed351..2202530 100644
--- a/msm/util.py
+++ b/msm/util.py
@@ -20,11 +20,12 @@
 # specific language governing permissions and limitations
 # under the License.
 import time
-
 import git
 from os.path import exists
 from os import chmod
 from fasteners.process_lock import InterProcessLock
+from collections import defaultdict
+from xml.etree import cElementTree as ET
 
 
 class Git(git.cmd.Git):
@@ -49,13 +50,14 @@ def __init__(self):
             chmod(lock_path, 0o777)
         super().__init__(lock_path)
 
+
 # The cached_property class defined below was copied from the
 # PythonDecoratorLibrary at:
 #   https://wiki.python.org/moin/PythonDecoratorLibrary/#Cached_Properties
 #
 # © 2011 Christopher Arndt, MIT License
 #
-class cached_property(object):
+class cached_property:
     """Decorator for read-only properties evaluated only once within TTL period.
 
     It can be used to create a cached property like this::
@@ -86,6 +88,7 @@ def randint(self):
         del instance._cache[<property name>]
 
     """
+
     def __init__(self, ttl=300):
         self.ttl = ttl
 
@@ -110,3 +113,61 @@ def __get__(self, inst, owner):
                 cache = inst._cache = {}
             cache[self.__name__] = (value, now)
         return value
+
+
+#  xml2dict and dict2xml copied from jarbas_utils
+#    https://github.com/OpenJarbas/jarbas_utils/
+#
+#   Apache License, Version 2.0
+#
+
+def xml2dict(xml_string):
+
+    e = ET.XML(xml_string)
+
+    def etree2dict(t):
+        d = {t.tag: {} if t.attrib else None}
+        children = list(t)
+        if children:
+            dd = defaultdict(list)
+            for dc in map(etree2dict, children):
+                for k, v in dc.items():
+                    dd[k].append(v)
+            d = {t.tag: {k: v[0] if len(v) == 1 else v for k, v in
+                         dd.items()}}
+        if t.attrib:
+            d[t.tag].update((k, v) for k, v in t.attrib.items())
+        if t.text:
+            text = t.text.strip()
+            if children or t.attrib:
+                if text:
+                    d[t.tag]['text'] = text
+            else:
+                d[t.tag] = text
+        return d
+
+    return etree2dict(e)
+
+
+def dict2xml(d, root="xml"):
+    xml = "<" + root
+    for k in d:
+        if isinstance(d[k], str):
+            if k == "text":
+                pass
+            else:
+                xml += " " + k + ' ="' + d[k] + '"'
+    xml += ">"
+    for k in d:
+        if isinstance(d[k], dict):
+            xml += dict2xml(d[k], k)
+        if isinstance(d[k], list):
+            for e in d[k]:
+                if isinstance(e, dict):
+                    xml += dict2xml(e, k)
+        if isinstance(d[k], str):
+            if k == "text":
+                xml += d[k]
+
+    xml += "</" + root + ">"
+    return xml
diff --git a/requirements.txt b/requirements.txt
index 4c184b0..d6d83a4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,5 @@ requests
 GitPython
 fasteners
 lazy
+pyyaml
+pako
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 1bfa674..1829c45 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@
 setup(
     name='msm',
     version='0.8.7',
-    packages=['msm'],
+    packages=['msm', 'msm.appstores'],
     install_requires=[
         'GitPython', 'fasteners', 'pyyaml', 'pako',
         'lazy'