From 8c966dc2b376a79992670088b4a32869349bb8b7 Mon Sep 17 00:00:00 2001 From: wunder957 Date: Sat, 16 Sep 2023 21:32:49 +0800 Subject: [PATCH] Intro AnalyzerManager (#63) * Intro AnalyzerManager * Solve dependence --- docs/source/managers/analyzer.rst | 13 ++ docs/source/managers/collector.rst | 2 +- docs/source/managers/filter.rst | 2 +- docs/source/managers/index.rst | 3 +- docs/source/managers/tracer.rst | 2 +- duetector/analyzer/base.py | 23 ++- duetector/analyzer/db.py | 11 +- duetector/analyzer/register.py | 4 + duetector/cli/main.py | 5 +- duetector/extension/analyzer/__init__.py | 4 + duetector/managers/__init__.py | 6 - duetector/managers/analyzer.py | 70 ++++++++ duetector/managers/collector.py | 2 +- duetector/managers/filter.py | 2 +- duetector/managers/tracer.py | 2 +- duetector/monitors/bcc_monitor.py | 4 +- duetector/monitors/sh_monitor.py | 4 +- duetector/service/query/controller.py | 7 +- duetector/static/config.toml | 19 ++- duetector/tools/config_generator.py | 15 +- .../extension/analyzer-extension/README.md | 156 ++++++++++++++++++ .../duetector_emptyanalyzer/__init__.py | 1 + .../duetector_emptyanalyzer/empty.py | 100 +++++++++++ .../analyzer-extension/pyproject.toml | 17 ++ tests/config.toml | 11 +- tests/conftest.py | 2 +- tests/service/test_query.py | 21 ++- tests/test_bcc_monitor.py | 4 +- tests/test_collector.py | 2 +- tests/test_db_analyzer.py | 10 +- tests/test_filter.py | 2 +- 31 files changed, 465 insertions(+), 61 deletions(-) create mode 100644 docs/source/managers/analyzer.rst create mode 100644 duetector/analyzer/register.py create mode 100644 duetector/extension/analyzer/__init__.py create mode 100644 duetector/managers/analyzer.py create mode 100644 examples/extension/analyzer-extension/README.md create mode 100644 examples/extension/analyzer-extension/duetector_emptyanalyzer/__init__.py create mode 100644 examples/extension/analyzer-extension/duetector_emptyanalyzer/empty.py create mode 100644 examples/extension/analyzer-extension/pyproject.toml diff --git a/docs/source/managers/analyzer.rst b/docs/source/managers/analyzer.rst new file mode 100644 index 0000000..f79d859 --- /dev/null +++ b/docs/source/managers/analyzer.rst @@ -0,0 +1,13 @@ +AnalyzerManager +=========================================== + + +.. autodata:: duetector.managers.analyzer.PROJECT_NAME +.. autofunction:: duetector.managers.analyzer.init_analyzer + + +.. autoclass:: duetector.managers.analyzer.AnalyzerManager + :members: + :undoc-members: + :inherited-members: + :show-inheritance: diff --git a/docs/source/managers/collector.rst b/docs/source/managers/collector.rst index 8c5fc6b..103d82a 100644 --- a/docs/source/managers/collector.rst +++ b/docs/source/managers/collector.rst @@ -6,7 +6,7 @@ CollectorManager .. autofunction:: duetector.managers.collector.init_collector -.. autoclass:: duetector.managers.CollectorManager +.. autoclass:: duetector.managers.collector.CollectorManager :members: :undoc-members: :inherited-members: diff --git a/docs/source/managers/filter.rst b/docs/source/managers/filter.rst index 548ebe4..0e98bee 100644 --- a/docs/source/managers/filter.rst +++ b/docs/source/managers/filter.rst @@ -6,7 +6,7 @@ FilterManager .. autofunction:: duetector.managers.filter.init_filter -.. autoclass:: duetector.managers.FilterManager +.. autoclass:: duetector.managers.filter.FilterManager :members: :undoc-members: :inherited-members: diff --git a/docs/source/managers/index.rst b/docs/source/managers/index.rst index 7defda4..3d422e8 100644 --- a/docs/source/managers/index.rst +++ b/docs/source/managers/index.rst @@ -4,7 +4,7 @@ Managers Managers provides a way to get instances of both built-in implementations and extensions. -.. autoclass:: duetector.managers.Manager +.. autoclass:: duetector.managers.base.Manager :members: :undoc-members: :inherited-members: @@ -20,3 +20,4 @@ Avaliable Manager Collector Manager Filter Manager Tracer Manager + Analyzer Manager diff --git a/docs/source/managers/tracer.rst b/docs/source/managers/tracer.rst index 89e09fc..3c53fc6 100644 --- a/docs/source/managers/tracer.rst +++ b/docs/source/managers/tracer.rst @@ -6,7 +6,7 @@ TracerManager .. autofunction:: duetector.managers.tracer.init_tracer -.. autoclass:: duetector.managers.TracerManager +.. autoclass:: duetector.managers.tracer.TracerManager :members: :undoc-members: :inherited-members: diff --git a/duetector/analyzer/base.py b/duetector/analyzer/base.py index 8c5c8b9..8f49b1a 100644 --- a/duetector/analyzer/base.py +++ b/duetector/analyzer/base.py @@ -10,17 +10,28 @@ class Analyzer(Configuable): A base class for all analyzers. """ - default_config = {} + default_config = { + "disabled": False, + } """ Default config for ``Analyzer``. """ - config_scope = "analyzer" - """ - Config scope for this analyzer. + @property + def disabled(self) -> bool: + """ + Weather this analyzer is disabled. + """ + return self.config.disabled - Subclasses cloud override this. - """ + @property + def config_scope(self): + """ + Config scope for this analyzer. + + Subclasses cloud override this. + """ + return self.__class__.__name__.lower() def get_all_tracers(self) -> List[str]: """ diff --git a/duetector/analyzer/db.py b/duetector/analyzer/db.py index df9fdee..83aa2f5 100644 --- a/duetector/analyzer/db.py +++ b/duetector/analyzer/db.py @@ -6,6 +6,7 @@ from duetector.analyzer.base import Analyzer from duetector.analyzer.models import AnalyzerBrief, Brief, Tracking from duetector.db import SessionManager +from duetector.extension.analyzer import hookimpl from duetector.log import logger @@ -68,11 +69,6 @@ class DBAnalyzer(Analyzer): Default config for ``DBAnalyzer``. """ - config_scope = "db_analyzer" - """ - Config scope for this analyzer. - """ - def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) # Init as a submodel @@ -299,3 +295,8 @@ def brief( collector_ids=set([brief.collector_id for brief in briefs]), briefs={f"{brief.tracer}@{brief.collector_id}": brief for brief in briefs}, ) + + +@hookimpl +def init_analyzer(config): + return DBAnalyzer(config) diff --git a/duetector/analyzer/register.py b/duetector/analyzer/register.py new file mode 100644 index 0000000..2692569 --- /dev/null +++ b/duetector/analyzer/register.py @@ -0,0 +1,4 @@ +# Expose for plugin system +from . import db + +registers = [db] diff --git a/duetector/cli/main.py b/duetector/cli/main.py index 2654f6f..1979d45 100644 --- a/duetector/cli/main.py +++ b/duetector/cli/main.py @@ -10,6 +10,7 @@ from duetector.analyzer.db import DBAnalyzer from duetector.config import CONFIG_PATH, ConfigLoader from duetector.log import logger +from duetector.managers.analyzer import AnalyzerManager from duetector.monitors import BccMonitor, ShMonitor from duetector.monitors.base import Monitor from duetector.tools.config_generator import ConfigGenerator @@ -170,7 +171,9 @@ def _shutdown(sig=None, frame=None): if brief: try: logger.info("Generating brief...") - logger.info(str(DBAnalyzer(c).brief(inspect_type=False))) + analyzers = AnalyzerManager(c).init() + for a in analyzers: + logger.info(str(a.brief(inspect_type=False))) except Exception as e: logger.error("Exception when generating brief") logger.exception(e) diff --git a/duetector/extension/analyzer/__init__.py b/duetector/extension/analyzer/__init__.py new file mode 100644 index 0000000..4004a0e --- /dev/null +++ b/duetector/extension/analyzer/__init__.py @@ -0,0 +1,4 @@ +import pluggy + +project_name = "duetector.analyzer" +hookimpl = pluggy.HookimplMarker(project_name) diff --git a/duetector/managers/__init__.py b/duetector/managers/__init__.py index c89c7fd..e69de29 100644 --- a/duetector/managers/__init__.py +++ b/duetector/managers/__init__.py @@ -1,6 +0,0 @@ -from .base import Manager -from .collector import CollectorManager -from .filter import FilterManager -from .tracer import TracerManager - -__all__ = ["Manager", "CollectorManager", "FilterManager", "TracerManager"] diff --git a/duetector/managers/analyzer.py b/duetector/managers/analyzer.py new file mode 100644 index 0000000..2d58208 --- /dev/null +++ b/duetector/managers/analyzer.py @@ -0,0 +1,70 @@ +import sys +from typing import Any, Dict, List, Optional + +import pluggy + +import duetector.analyzer.register +from duetector.analyzer.base import Analyzer +from duetector.extension.analyzer import project_name +from duetector.log import logger +from duetector.managers.base import Manager + +PROJECT_NAME = project_name #: Default project name for pluggy +hookspec = pluggy.HookspecMarker(PROJECT_NAME) + + +@hookspec +def init_analyzer(config) -> Optional[Analyzer]: + """ + Initialize analyzer from config + None means the analyzer is not available + Also the analyzer can be disabled by config, Manager will discard disabled analyzer + """ + + +class AnalyzerManager(Manager): + """ + Manager for all analyzers. + + Analyzers are initialized from config, and can be ``disabled`` by config. + """ + + config_scope = "analyzer" + """ + Config scope for ``AnalyzerManager``. + """ + + def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): + super().__init__(config, *args, **kwargs) + + self.pm = pluggy.PluginManager(PROJECT_NAME) + self.pm.add_hookspecs(sys.modules[__name__]) + self.pm.load_setuptools_entrypoints(PROJECT_NAME) + self.register(duetector.analyzer.register) + + def init(self, analyzer_type=Analyzer, ignore_disabled=True) -> List[Analyzer]: + """ + Initialize all analyzers from config. + + Args: + analyzer_type: Only return analyzers of this type + ignore_disabled: Ignore disabled analyzers + """ + if self.disabled: + logger.info("AnalyzerManager disabled.") + return [] + + objs = [] + for f in self.pm.hook.init_analyzer(config=self.config._config_dict): + if not f or (f.disabled and ignore_disabled): + logger.debug(f"Analyzer {f.__class__.__name__} is not available (None or Disabled)") + continue + if not isinstance(f, analyzer_type): + logger.debug( + f"Skip Analyzer {f.__class__.__name__} (Not a instance of {analyzer_type})" + ) + continue + + objs.append(f) + + return objs diff --git a/duetector/managers/collector.py b/duetector/managers/collector.py index 5ad7bf3..d35f1fc 100644 --- a/duetector/managers/collector.py +++ b/duetector/managers/collector.py @@ -7,7 +7,7 @@ from duetector.collectors.base import Collector from duetector.extension.collector import project_name from duetector.log import logger -from duetector.managers import Manager +from duetector.managers.base import Manager PROJECT_NAME = project_name #: Default project name for pluggy hookspec = pluggy.HookspecMarker(PROJECT_NAME) diff --git a/duetector/managers/filter.py b/duetector/managers/filter.py index d9a8bd3..0865c4a 100644 --- a/duetector/managers/filter.py +++ b/duetector/managers/filter.py @@ -7,7 +7,7 @@ from duetector.extension.filter import project_name from duetector.filters.base import Filter from duetector.log import logger -from duetector.managers import Manager +from duetector.managers.base import Manager PROJECT_NAME = project_name #: Default project name for pluggy hookspec = pluggy.HookspecMarker(PROJECT_NAME) diff --git a/duetector/managers/tracer.py b/duetector/managers/tracer.py index 0afedf6..dfb1ca5 100644 --- a/duetector/managers/tracer.py +++ b/duetector/managers/tracer.py @@ -6,7 +6,7 @@ import duetector.tracers.register from duetector.extension.tracer import project_name from duetector.log import logger -from duetector.managers import Manager +from duetector.managers.base import Manager from duetector.tracers.base import Tracer PROJECT_NAME = project_name #: Default project name for pluggy diff --git a/duetector/monitors/bcc_monitor.py b/duetector/monitors/bcc_monitor.py index 8494cda..6bc98df 100644 --- a/duetector/monitors/bcc_monitor.py +++ b/duetector/monitors/bcc_monitor.py @@ -3,7 +3,9 @@ from duetector.collectors.base import Collector from duetector.log import logger -from duetector.managers import CollectorManager, FilterManager, TracerManager +from duetector.managers.collector import CollectorManager +from duetector.managers.filter import FilterManager +from duetector.managers.tracer import TracerManager from duetector.monitors.base import Monitor from duetector.tracers import BccTracer diff --git a/duetector/monitors/sh_monitor.py b/duetector/monitors/sh_monitor.py index b9044c1..f2fce3c 100644 --- a/duetector/monitors/sh_monitor.py +++ b/duetector/monitors/sh_monitor.py @@ -3,7 +3,9 @@ from duetector.collectors.base import Collector from duetector.log import logger -from duetector.managers import CollectorManager, FilterManager, TracerManager +from duetector.managers.collector import CollectorManager +from duetector.managers.filter import FilterManager +from duetector.managers.tracer import TracerManager from duetector.monitors.base import Monitor from duetector.tracers.base import ShellTracer diff --git a/duetector/service/query/controller.py b/duetector/service/query/controller.py index fd5a29a..1736508 100644 --- a/duetector/service/query/controller.py +++ b/duetector/service/query/controller.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional from duetector.analyzer.base import Analyzer -from duetector.analyzer.db import DBAnalyzer +from duetector.managers.analyzer import AnalyzerManager from duetector.service.base import Controller from duetector.service.exceptions import NotFoundError @@ -11,11 +11,10 @@ def __init__(self, config: Optional[Dict[str, Any]] = None, *args, **kwargs): super().__init__(config, *args, **kwargs) # TODO: Make this configurable, may intro a manager for analyzer - self._avaliable_analyzers = [DBAnalyzer] + self._avaliable_analyzers = AnalyzerManager(config).init() self._analyzers: Dict[str, Analyzer] = { - analyzer.config_scope: self._init_analyzer(analyzer) - for analyzer in self._avaliable_analyzers + analyzer.config_scope: analyzer for analyzer in self._avaliable_analyzers } def _init_analyzer(self, analyzer: type): diff --git a/duetector/static/config.toml b/duetector/static/config.toml index 6f5d8d1..71c339b 100644 --- a/duetector/static/config.toml +++ b/duetector/static/config.toml @@ -77,6 +77,19 @@ maxlen = 1024 [collector.dequecollector.backend_args] max_workers = 10 +[analyzer] +disabled = false +include_extension = true + +[analyzer.dbanalyzer] +disabled = false + +[analyzer.dbanalyzer.db] +table_prefix = "duetector_tracking" + +[analyzer.dbanalyzer.db.engine] +url = "sqlite:///duetector-dbcollector.sqlite3" + [monitor.bcc] disabled = false auto_init = true @@ -99,11 +112,5 @@ max_workers = 10 [monitor.sh.poller] interval_ms = 500 -[db_analyzer.db] -table_prefix = "duetector_tracking" - -[db_analyzer.db.engine] -url = "sqlite:///duetector-dbcollector.sqlite3" - [server] token = "" diff --git a/duetector/tools/config_generator.py b/duetector/tools/config_generator.py index 05962e1..7313bd1 100644 --- a/duetector/tools/config_generator.py +++ b/duetector/tools/config_generator.py @@ -7,7 +7,10 @@ from duetector.analyzer.db import DBAnalyzer from duetector.config import ConfigLoader from duetector.log import logger -from duetector.managers import CollectorManager, FilterManager, TracerManager +from duetector.managers.analyzer import AnalyzerManager +from duetector.managers.collector import CollectorManager +from duetector.managers.filter import FilterManager +from duetector.managers.tracer import TracerManager from duetector.monitors import BccMonitor, ShMonitor from duetector.service.config import ServerConfig @@ -48,7 +51,7 @@ class ConfigGenerator: """ - managers = [FilterManager, TracerManager, CollectorManager] + managers = [FilterManager, TracerManager, CollectorManager, AnalyzerManager] """ All managers to inspect. """ @@ -58,11 +61,6 @@ class ConfigGenerator: All monitors to inspect. """ - analyzer = [DBAnalyzer] - """ - All analyzers to inspect. - """ - others = [ServerConfig] def __init__( @@ -88,9 +86,6 @@ def __init__( for m in self.monitors: _recursive_load(m.config_scope, self.dynamic_config, m.default_config) - for a in self.analyzer: - _recursive_load(a.config_scope, self.dynamic_config, a.default_config) - for o in self.others: _recursive_load(o.config_scope, self.dynamic_config, o.default_config) # This will generate default config file if not exists diff --git a/examples/extension/analyzer-extension/README.md b/examples/extension/analyzer-extension/README.md new file mode 100644 index 0000000..8b796a0 --- /dev/null +++ b/examples/extension/analyzer-extension/README.md @@ -0,0 +1,156 @@ +In this example, we will write a empty Analyzer. The Analyzer returns nothing and does nothing. + +Full example and more details can be found in [duetector_emptyanalyzer](./duetector_emptyanalyzer/) + +## 1 Write a Analyzer + +Here we write the echo Analyzer and set it to not `disabled` by defaylt. + +```python +class EmptyAnalyzer(Analyzer): + def get_all_tracers(self) -> List[str]: + """ + Get all tracers from storage. + + Returns: + List[str]: List of tracer's name. + """ + return [] + + def get_all_collector_ids(self) -> List[str]: + """ + Get all collector id from storage. + + Returns: + List[str]: List of collector id. + """ + return [] + + def query( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + start: int = 0, + limit: int = 0, + columns: Optional[List[str]] = None, + where: Optional[Dict[str, Any]] = None, + distinct: bool = False, + order_by_asc: Optional[List[str]] = None, + order_by_desc: Optional[List[str]] = None, + ) -> List[Tracking]: + """ + Query all tracking records from database. + + Args: + tracers (Optional[List[str]], optional): Tracer's name. Defaults to None, all tracers will be queried. + collector_ids (Optional[List[str]], optional): Collector id. Defaults to None, all collector id will be queried. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + start (int, optional): Start index. Defaults to 0. + limit (int, optional): Limit of records. Defaults to 20. ``0`` means no limit. + columns (Optional[List[str]], optional): Columns to query. Defaults to None, all columns will be queried. + where (Optional[Dict[str, Any]], optional): Where clause. Defaults to None. + distinct (bool, optional): Distinct. Defaults to False. + order_by_asc (Optional[List[str]], optional): Order by asc. Defaults to None. + order_by_desc (Optional[List[str]], optional): Order by desc. Defaults to None. + Returns: + List[duetector.analyzer.models.Tracking]: List of tracking records. + """ + return [] + + def brief( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + with_details: bool = True, + distinct: bool = False, + inspect_type: bool = False, + ) -> AnalyzerBrief: + """ + Get a brief of this analyzer. + + Args: + tracers (Optional[List[str]], optional): + Tracers. Defaults to None, all tracers will be queried. + If a specific tracer is not found, it will be ignored. + collector_ids (Optional[List[str]], optional): + Collector ids. Defaults to None, all collector ids will be queried. + If a specific collector id is not found, it will be ignored. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + with_details (bool, optional): With details. Defaults to True. + distinct (bool, optional): Distinct. Defaults to False. + inspect_type (bool, optional): Weather fileds's value is type or type name. Defaults to False, type name. + + Returns: + AnalyzerBrief: A brief of this analyzer. + """ + return AnalyzerBrief(tracers=set(), collector_ids=set(), briefs={}) + + def analyze(self): + # TODO: Not design yet. + pass + + +``` + +## 2 Register the Analyzer + +```python +from duetector.extension.analyzer import hookimpl + +@hookimpl +def init_analyzer(config): + return EmptyAnalyzer(config) + +``` + +## 3 Turning the Analyzer into a package + +Here we use `pyproject.toml` as the configuration file. + +```toml +# Build with hatch, you can use any build tool you like. +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "duetector-emptyanalyzer" + +dependencies = ["duetector"] +dynamic = ["version"] + +# This is the entry point for the AnalyzerManager to find the Analyzer. +[project.entry-points."duetector.analyzer"] +emptyanalyzer= "duetector_emptyanalyzer.empty" + +[tool.hatch.version] +path = "duetector_emptyanalyzer/__init__.py" + +``` + +## 4 Verification + +Simply run the following code to verify the Analyzer is registered. + +```python +from duetector.manager import AnalyzerManager +from duetector_emptyanalyzer.empty import EmptyAnalyzer + + +assert EmptyAnalyzer in (type(a) for a in AnalyzerManager().init()) +``` + +## 5 Configuration + +In `config.toml` you can set the Analyzer to be disabled. + +```toml +[analyzer.EmptyAnalyzer] +disabled = true +``` diff --git a/examples/extension/analyzer-extension/duetector_emptyanalyzer/__init__.py b/examples/extension/analyzer-extension/duetector_emptyanalyzer/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/examples/extension/analyzer-extension/duetector_emptyanalyzer/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/examples/extension/analyzer-extension/duetector_emptyanalyzer/empty.py b/examples/extension/analyzer-extension/duetector_emptyanalyzer/empty.py new file mode 100644 index 0000000..ef34477 --- /dev/null +++ b/examples/extension/analyzer-extension/duetector_emptyanalyzer/empty.py @@ -0,0 +1,100 @@ +from datetime import datetime +from typing import Any, Dict, List, Optional + +from duetector.analyzer.base import Analyzer +from duetector.analyzer.models import AnalyzerBrief, Tracking +from duetector.extension.analyzer import hookimpl + + +class EmptyAnalyzer(Analyzer): + def get_all_tracers(self) -> List[str]: + """ + Get all tracers from storage. + + Returns: + List[str]: List of tracer's name. + """ + return [] + + def get_all_collector_ids(self) -> List[str]: + """ + Get all collector id from storage. + + Returns: + List[str]: List of collector id. + """ + return [] + + def query( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + start: int = 0, + limit: int = 0, + columns: Optional[List[str]] = None, + where: Optional[Dict[str, Any]] = None, + distinct: bool = False, + order_by_asc: Optional[List[str]] = None, + order_by_desc: Optional[List[str]] = None, + ) -> List[Tracking]: + """ + Query all tracking records from database. + + Args: + tracers (Optional[List[str]], optional): Tracer's name. Defaults to None, all tracers will be queried. + collector_ids (Optional[List[str]], optional): Collector id. Defaults to None, all collector id will be queried. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + start (int, optional): Start index. Defaults to 0. + limit (int, optional): Limit of records. Defaults to 20. ``0`` means no limit. + columns (Optional[List[str]], optional): Columns to query. Defaults to None, all columns will be queried. + where (Optional[Dict[str, Any]], optional): Where clause. Defaults to None. + distinct (bool, optional): Distinct. Defaults to False. + order_by_asc (Optional[List[str]], optional): Order by asc. Defaults to None. + order_by_desc (Optional[List[str]], optional): Order by desc. Defaults to None. + Returns: + List[duetector.analyzer.models.Tracking]: List of tracking records. + """ + return [] + + def brief( + self, + tracers: Optional[List[str]] = None, + collector_ids: Optional[List[str]] = None, + start_datetime: Optional[datetime] = None, + end_datetime: Optional[datetime] = None, + with_details: bool = True, + distinct: bool = False, + inspect_type: bool = False, + ) -> AnalyzerBrief: + """ + Get a brief of this analyzer. + + Args: + tracers (Optional[List[str]], optional): + Tracers. Defaults to None, all tracers will be queried. + If a specific tracer is not found, it will be ignored. + collector_ids (Optional[List[str]], optional): + Collector ids. Defaults to None, all collector ids will be queried. + If a specific collector id is not found, it will be ignored. + start_datetime (Optional[datetime], optional): Start time. Defaults to None. + end_datetime (Optional[datetime], optional): End time. Defaults to None. + with_details (bool, optional): With details. Defaults to True. + distinct (bool, optional): Distinct. Defaults to False. + inspect_type (bool, optional): Weather fileds's value is type or type name. Defaults to False, type name. + + Returns: + AnalyzerBrief: A brief of this analyzer. + """ + return AnalyzerBrief(tracers=set(), collector_ids=set(), briefs={}) + + def analyze(self): + # TODO: Not design yet. + pass + + +@hookimpl +def init_analyzer(config): + return EmptyAnalyzer(config) diff --git a/examples/extension/analyzer-extension/pyproject.toml b/examples/extension/analyzer-extension/pyproject.toml new file mode 100644 index 0000000..76ab0ad --- /dev/null +++ b/examples/extension/analyzer-extension/pyproject.toml @@ -0,0 +1,17 @@ +# Build with hatch, you can use any build tool you like. +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "duetector-emptyanalyzer" + +dependencies = ["duetector"] +dynamic = ["version"] + +# This is the entry point for the AnalyzerManager to find the Analyzer. +[project.entry-points."duetector.analyzer"] +emptyanalyzer= "duetector_emptyanalyzer.empty" + +[tool.hatch.version] +path = "duetector_emptyanalyzer/__init__.py" diff --git a/tests/config.toml b/tests/config.toml index 72129ac..036696f 100644 --- a/tests/config.toml +++ b/tests/config.toml @@ -98,10 +98,17 @@ max_workers = 10 [monitor.sh.poller] interval_ms = 500 -[db_analyzer.db] +[analyzer] +disabled = false +include_extension = true + +[analyzer.dbanalyzer] +disabled = false + +[analyzer.dbanalyzer.db] table_prefix = "duetector_tracking" -[db_analyzer.db.engine] +[analyzer.dbanalyzer.db.engine] url = "sqlite:///:memory:" [server] diff --git a/tests/conftest.py b/tests/conftest.py index 4cdb124..f1692a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,4 +23,4 @@ def full_config_file(tmpdir): @pytest.fixture def full_config(full_config_file): - yield ConfigLoader(full_config_file).load_config() + yield ConfigLoader(full_config_file, load_env=False, dump_when_load=False).load_config() diff --git a/tests/service/test_query.py b/tests/service/test_query.py index 48378a0..11ce7d3 100644 --- a/tests/service/test_query.py +++ b/tests/service/test_query.py @@ -2,6 +2,7 @@ from fastapi.testclient import TestClient from duetector.analyzer.db import DBAnalyzer +from duetector.managers.analyzer import AnalyzerManager from duetector.service.app import app from duetector.service.config import get_config @@ -12,26 +13,34 @@ def configed_app(full_config): return app +@pytest.fixture +def db_analyzer(full_config): + config = AnalyzerManager(full_config).config._config_dict + + yield DBAnalyzer(config) + + @pytest.fixture def client(configed_app): with TestClient(configed_app) as client: yield client -def test_query(client: TestClient): +def test_query(client: TestClient, db_analyzer): response = client.get(f"/query/") assert response.status_code == 200 - assert response.json() == {"analyzers": [DBAnalyzer.config_scope]} + assert "analyzers" in response.json() + assert db_analyzer.config_scope in response.json()["analyzers"] -def test_query_brief(client: TestClient): - response = client.get(f"/query/{DBAnalyzer.config_scope}/brief") +def test_query_brief(client: TestClient, db_analyzer): + response = client.get(f"/query/{db_analyzer.config_scope}/brief") assert response.status_code == 200 assert response.json() -def test_query_analyzer(client: TestClient): - response = client.post(f"/query/{DBAnalyzer.config_scope}") +def test_query_analyzer(client: TestClient, db_analyzer): + response = client.post(f"/query/{db_analyzer.config_scope}") assert response.status_code == 200 assert response.json() diff --git a/tests/test_bcc_monitor.py b/tests/test_bcc_monitor.py index 81d779f..6afa61a 100644 --- a/tests/test_bcc_monitor.py +++ b/tests/test_bcc_monitor.py @@ -4,7 +4,9 @@ import pytest from duetector.collectors.models import Tracking -from duetector.managers import CollectorManager, FilterManager, TracerManager +from duetector.managers.collector import CollectorManager +from duetector.managers.filter import FilterManager +from duetector.managers.tracer import TracerManager from duetector.monitors.bcc_monitor import BccMonitor, Monitor from duetector.tracers.base import BccTracer, Tracer from duetector.utils import get_boot_time_duration_ns diff --git a/tests/test_collector.py b/tests/test_collector.py index 0006a36..ef654e2 100644 --- a/tests/test_collector.py +++ b/tests/test_collector.py @@ -4,7 +4,7 @@ from duetector.collectors.db import DBCollector from duetector.collectors.models import Tracking -from duetector.managers import CollectorManager +from duetector.managers.collector import CollectorManager from duetector.utils import get_boot_time_duration_ns timestamp = 13205215231927 diff --git a/tests/test_db_analyzer.py b/tests/test_db_analyzer.py index ba4a05f..12b0823 100644 --- a/tests/test_db_analyzer.py +++ b/tests/test_db_analyzer.py @@ -5,6 +5,7 @@ from duetector.analyzer.db import DBAnalyzer from duetector.analyzer.models import Tracking as AT from duetector.collectors.models import Tracking as CT +from duetector.managers.analyzer import AnalyzerManager now = datetime.now() @@ -42,8 +43,13 @@ def a_tracking(): @pytest.fixture -def db_analyzer(full_config, c_tracking, collector_id): - db_analyzer = DBAnalyzer(full_config) +def config(full_config): + yield AnalyzerManager(full_config).config._config_dict + + +@pytest.fixture +def db_analyzer(config, c_tracking, collector_id): + db_analyzer = DBAnalyzer(config) sessionmanager = db_analyzer.sm m = sessionmanager.get_tracking_model(c_tracking.tracer, collector_id) diff --git a/tests/test_filter.py b/tests/test_filter.py index 731ffd4..7e480d6 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -4,7 +4,7 @@ from duetector.config import ConfigLoader from duetector.filters.pattern import PatternFilter -from duetector.managers import FilterManager +from duetector.managers.filter import FilterManager data_t = namedtuple( "Tracking", ["pid", "uid", "gid", "comm", "fname", "timestamp", "custom", "gcustom"]