Merge branch 'main' into draft-core

docs/source/managers/analyzer.rst

@@ -0,0 +1,13 @@
+AnalyzerManager
+===========================================
+
+
+.. autodata:: duetector.managers.analyzer.PROJECT_NAME
+.. autofunction:: duetector.managers.analyzer.init_analyzer
+
+
+.. autoclass:: duetector.managers.analyzer.AnalyzerManager
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:

docs/source/managers/collector.rst

@@ -6,7 +6,7 @@ CollectorManager
 .. autofunction:: duetector.managers.collector.init_collector
 
 
-.. autoclass:: duetector.managers.CollectorManager
+.. autoclass:: duetector.managers.collector.CollectorManager
    :members:
    :undoc-members:
    :inherited-members:

docs/source/managers/filter.rst

@@ -6,7 +6,7 @@ FilterManager
 .. autofunction:: duetector.managers.filter.init_filter
 
 
-.. autoclass:: duetector.managers.FilterManager
+.. autoclass:: duetector.managers.filter.FilterManager
    :members:
    :undoc-members:
    :inherited-members:

docs/source/managers/index.rst

@@ -4,7 +4,7 @@ Managers
 Managers provides a way to get instances of both built-in implementations and extensions.
 
 
-.. autoclass:: duetector.managers.Manager
+.. autoclass:: duetector.managers.base.Manager
    :members:
    :undoc-members:
    :inherited-members:
@@ -20,3 +20,4 @@ Avaliable Manager
    Collector Manager <collector>
    Filter Manager <filter>
    Tracer Manager <tracer>
+   Analyzer Manager <analyzer>

docs/source/managers/tracer.rst

@@ -6,7 +6,7 @@ TracerManager
 .. autofunction:: duetector.managers.tracer.init_tracer
 
 
-.. autoclass:: duetector.managers.TracerManager
+.. autoclass:: duetector.managers.tracer.TracerManager
    :members:
    :undoc-members:
    :inherited-members:

duetector/analyzer/base.py

@@ -10,17 +10,28 @@ class Analyzer(Configuable):
     A base class for all analyzers.
     """
 
-    default_config = {}
+    default_config = {
+        "disabled": False,
+    }
     """
     Default config for ``Analyzer``.
     """
 
-    config_scope = "analyzer"
-    """
-    Config scope for this analyzer.
+    @property
+    def disabled(self) -> bool:
+        """
+        Weather this analyzer is disabled.
+        """
+        return self.config.disabled
 
-    Subclasses cloud override this.
-    """
+    @property
+    def config_scope(self):
+        """
+        Config scope for this analyzer.
+
+        Subclasses cloud override this.
+        """
+        return self.__class__.__name__.lower()
 
     def get_all_tracers(self) -> List[str]:
         """

duetector/analyzer/db.py

@@ -6,6 +6,7 @@
 from duetector.analyzer.base import Analyzer
 from duetector.analyzer.models import AnalyzerBrief, Brief, Tracking
 from duetector.db import SessionManager
+from duetector.extension.analyzer import hookimpl
 from duetector.log import logger
 
 
@@ -68,11 +69,6 @@ class DBAnalyzer(Analyzer):
     Default config for ``DBAnalyzer``.
     """
 
-    config_scope = "db_analyzer"
-    """
-    Config scope for this analyzer.
-    """
-
     def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         # Init as a submodel
@@ -299,3 +295,8 @@ def brief(
             collector_ids=set([brief.collector_id for brief in briefs]),
             briefs={f"{brief.tracer}@{brief.collector_id}": brief for brief in briefs},
         )
+
+
+@hookimpl
+def init_analyzer(config):
+    return DBAnalyzer(config)

duetector/analyzer/register.py

@@ -0,0 +1,4 @@
+# Expose for plugin system
+from . import db
+
+registers = [db]

duetector/cli/main.py

@@ -10,6 +10,7 @@
 from duetector.analyzer.db import DBAnalyzer
 from duetector.config import CONFIG_PATH, ConfigLoader
 from duetector.log import logger
+from duetector.managers.analyzer import AnalyzerManager
 from duetector.monitors import BccMonitor, ShMonitor
 from duetector.monitors.base import Monitor
 from duetector.tools.config_generator import ConfigGenerator
@@ -170,7 +171,9 @@ def _shutdown(sig=None, frame=None):
         if brief:
             try:
                 logger.info("Generating brief...")
-                logger.info(str(DBAnalyzer(c).brief(inspect_type=False)))
+                analyzers = AnalyzerManager(c).init()
+                for a in analyzers:
+                    logger.info(str(a.brief(inspect_type=False)))
             except Exception as e:
                 logger.error("Exception when generating brief")
                 logger.exception(e)

duetector/extension/analyzer/__init__.py

@@ -0,0 +1,4 @@
+import pluggy
+
+project_name = "duetector.analyzer"
+hookimpl = pluggy.HookimplMarker(project_name)

duetector/managers/__init__.py

@@ -1,6 +0,0 @@
-from .base import Manager
-from .collector import CollectorManager
-from .filter import FilterManager
-from .tracer import TracerManager
-
-__all__ = ["Manager", "CollectorManager", "FilterManager", "TracerManager"]

duetector/managers/analyzer.py

@@ -0,0 +1,70 @@
+import sys
+from typing import Any, Dict, List, Optional
+
+import pluggy
+
+import duetector.analyzer.register
+from duetector.analyzer.base import Analyzer
+from duetector.extension.analyzer import project_name
+from duetector.log import logger
+from duetector.managers.base import Manager
+
+PROJECT_NAME = project_name  #: Default project name for pluggy
+hookspec = pluggy.HookspecMarker(PROJECT_NAME)
+
+
+@hookspec
+def init_analyzer(config) -> Optional[Analyzer]:
+    """
+    Initialize analyzer from config
+    None means the analyzer is not available
+    Also the analyzer can be disabled by config, Manager will discard disabled analyzer
+    """
+
+
+class AnalyzerManager(Manager):
+    """
+    Manager for all analyzers.
+
+    Analyzers are initialized from config, and can be ``disabled`` by config.
+    """
+
+    config_scope = "analyzer"
+    """
+    Config scope for ``AnalyzerManager``.
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs):
+        super().__init__(config, *args, **kwargs)
+
+        self.pm = pluggy.PluginManager(PROJECT_NAME)
+        self.pm.add_hookspecs(sys.modules[__name__])
+        self.pm.load_setuptools_entrypoints(PROJECT_NAME)
+        self.register(duetector.analyzer.register)
+
+    def init(self, analyzer_type=Analyzer, ignore_disabled=True) -> List[Analyzer]:
+        """
+        Initialize all analyzers from config.
+
+        Args:
+            analyzer_type: Only return analyzers of this type
+            ignore_disabled: Ignore disabled analyzers
+        """
+        if self.disabled:
+            logger.info("AnalyzerManager disabled.")
+            return []
+
+        objs = []
+        for f in self.pm.hook.init_analyzer(config=self.config._config_dict):
+            if not f or (f.disabled and ignore_disabled):
+                logger.debug(f"Analyzer {f.__class__.__name__} is not available (None or Disabled)")
+                continue
+            if not isinstance(f, analyzer_type):
+                logger.debug(
+                    f"Skip Analyzer {f.__class__.__name__} (Not a instance of {analyzer_type})"
+                )
+                continue
+
+            objs.append(f)
+
+        return objs

duetector/managers/collector.py

@@ -7,7 +7,7 @@
 from duetector.collectors.base import Collector
 from duetector.extension.collector import project_name
 from duetector.log import logger
-from duetector.managers import Manager
+from duetector.managers.base import Manager
 
 PROJECT_NAME = project_name  #: Default project name for pluggy
 hookspec = pluggy.HookspecMarker(PROJECT_NAME)

duetector/managers/filter.py

@@ -7,7 +7,7 @@
 from duetector.extension.filter import project_name
 from duetector.filters.base import Filter
 from duetector.log import logger
-from duetector.managers import Manager
+from duetector.managers.base import Manager
 
 PROJECT_NAME = project_name  #: Default project name for pluggy
 hookspec = pluggy.HookspecMarker(PROJECT_NAME)

duetector/managers/tracer.py

@@ -6,7 +6,7 @@
 import duetector.tracers.register
 from duetector.extension.tracer import project_name
 from duetector.log import logger
-from duetector.managers import Manager
+from duetector.managers.base import Manager
 from duetector.tracers.base import Tracer
 
 PROJECT_NAME = project_name  #: Default project name for pluggy

duetector/monitors/bcc_monitor.py

@@ -3,7 +3,9 @@
 
 from duetector.collectors.base import Collector
 from duetector.log import logger
-from duetector.managers import CollectorManager, FilterManager, TracerManager
+from duetector.managers.collector import CollectorManager
+from duetector.managers.filter import FilterManager
+from duetector.managers.tracer import TracerManager
 from duetector.monitors.base import Monitor
 from duetector.tracers import BccTracer
 

duetector/monitors/sh_monitor.py

@@ -3,7 +3,9 @@
 
 from duetector.collectors.base import Collector
 from duetector.log import logger
-from duetector.managers import CollectorManager, FilterManager, TracerManager
+from duetector.managers.collector import CollectorManager
+from duetector.managers.filter import FilterManager
+from duetector.managers.tracer import TracerManager
 from duetector.monitors.base import Monitor
 from duetector.tracers.base import ShellTracer
 

duetector/service/base.py

@@ -1,5 +1,10 @@
 from typing import Any, Dict, Optional
 
+try:
+    from functools import cache
+except ImportError:
+    from functools import lru_cache as cache
+
 from fastapi import Depends
 
 from duetector.config import Configuable
@@ -15,6 +20,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
 
 
+@cache
 def get_controller(controller_type: type):
     def _(config: dict = Depends(get_config)) -> Controller:
         return controller_type(config)

duetector/service/query/controller.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Optional
 
 from duetector.analyzer.base import Analyzer
-from duetector.analyzer.db import DBAnalyzer
+from duetector.managers.analyzer import AnalyzerManager
 from duetector.service.base import Controller
 from duetector.service.exceptions import NotFoundError
 
@@ -11,11 +11,10 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
 
         # TODO: Make this configurable, may intro a manager for analyzer
-        self._avaliable_analyzers = [DBAnalyzer]
+        self._avaliable_analyzers = AnalyzerManager(config).init()
 
         self._analyzers: Dict[str, Analyzer] = {
-            analyzer.config_scope: self._init_analyzer(analyzer)
-            for analyzer in self._avaliable_analyzers
+            analyzer.config_scope: analyzer for analyzer in self._avaliable_analyzers
         }
 
     def _init_analyzer(self, analyzer: type):

duetector/service/query/routes.py

@@ -1,4 +1,7 @@
+from asyncio import sleep
+
 from fastapi import APIRouter, Body, Depends
+from fastapi.concurrency import run_in_threadpool
 
 from duetector.service.base import get_controller
 from duetector.service.query.controller import AnalyzerController
@@ -35,7 +38,7 @@ async def query(
     Query data from analyzer
     """
     analyzer = controller.get_analyzer(analyzer_name)
-    trackings = analyzer.query(**query_param.model_dump())
+    trackings = await run_in_threadpool(analyzer.query, **query_param.model_dump())
 
     return QueryResult(
         trackings=trackings,
@@ -50,8 +53,8 @@ async def query_brief(
 ):
     # type is not serializable, so we need to get analyzer without inspect type
     analyzer = controller.get_analyzer(analyzer_name)
-
+    brief = await run_in_threadpool(analyzer.brief, inspect_type=False)
     return BriefResult(
-        brief=analyzer.brief(inspect_type=False),
+        brief=brief,
         analyzer_name=analyzer_name,
     )

duetector/static/config.toml

@@ -77,6 +77,19 @@ maxlen = 1024
 [collector.dequecollector.backend_args]
 max_workers = 10
 
+[analyzer]
+disabled = false
+include_extension = true
+
+[analyzer.dbanalyzer]
+disabled = false
+
+[analyzer.dbanalyzer.db]
+table_prefix = "duetector_tracking"
+
+[analyzer.dbanalyzer.db.engine]
+url = "sqlite:///duetector-dbcollector.sqlite3"
+
 [monitor.bcc]
 disabled = false
 auto_init = true
@@ -99,11 +112,5 @@ max_workers = 10
 [monitor.sh.poller]
 interval_ms = 500
 
-[db_analyzer.db]
-table_prefix = "duetector_tracking"
-
-[db_analyzer.db.engine]
-url = "sqlite:///duetector-dbcollector.sqlite3"
-
 [server]
 token = ""