From cffd50873de9c8cbde8ff221137e008d58eb091d Mon Sep 17 00:00:00 2001 From: wunder957 Date: Mon, 27 Nov 2023 17:06:18 +0800 Subject: [PATCH] Adding empty injector --- duetector/extension/injector/__init__.py | 4 ++ duetector/filters/base.py | 11 ++-- duetector/injectors/__init__.py | 0 duetector/injectors/base.py | 58 ++++++++++++++++++++ duetector/injectors/docker.py | 11 ++++ duetector/injectors/k8s.py | 11 ++++ duetector/injectors/register.py | 4 ++ duetector/managers/injector.py | 68 ++++++++++++++++++++++++ duetector/monitors/base.py | 39 +++++++++++--- duetector/monitors/bcc_monitor.py | 8 ++- duetector/monitors/sh_monitor.py | 10 ++-- duetector/monitors/subprocess_monitor.py | 7 +-- duetector/static/config.toml | 10 ++++ duetector/tools/config_generator.py | 3 +- pyproject.toml | 3 ++ 15 files changed, 217 insertions(+), 30 deletions(-) create mode 100644 duetector/extension/injector/__init__.py create mode 100644 duetector/injectors/__init__.py create mode 100644 duetector/injectors/base.py create mode 100644 duetector/injectors/docker.py create mode 100644 duetector/injectors/k8s.py create mode 100644 duetector/injectors/register.py create mode 100644 duetector/managers/injector.py diff --git a/duetector/extension/injector/__init__.py b/duetector/extension/injector/__init__.py new file mode 100644 index 0000000..b35c28c --- /dev/null +++ b/duetector/extension/injector/__init__.py @@ -0,0 +1,4 @@ +import pluggy + +project_name = "duetector.injector" +hookimpl = pluggy.HookimplMarker(project_name) diff --git a/duetector/filters/base.py b/duetector/filters/base.py index 371324d..b82d062 100644 --- a/duetector/filters/base.py +++ b/duetector/filters/base.py @@ -1,10 +1,8 @@ from __future__ import annotations -import os from collections import namedtuple from duetector.config import Configuable -from duetector.extension.filter import hookimpl class Filter(Configuable): @@ -22,17 +20,18 @@ class Filter(Configuable): .. code-block:: python from duetector.filters import Filter - from duetector.collectors.models import Tracking + from collections import namedtuple class MyFilter(Filter): - def filter(self, data: Tracking) -> Optional[Tracking]: + def filter(self, data: namedtuple) -> namedtuple | None: if data.fname == "/etc/passwd": return None return data f = MyFilter() - f(Tracking(fname="/etc/passwd")) # None - f(Tracking(fname="/etc/shadow")) # Tracking(fname="/etc/shadow") + data_t = namedtuple("Tracking", "fname") + f(data_t(fname="/etc/passwd")) # None + f(data_t(fname="/etc/shadow")) # Tracking(fname="/etc/shadow") """ default_config = { diff --git a/duetector/injectors/__init__.py b/duetector/injectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/duetector/injectors/base.py b/duetector/injectors/base.py new file mode 100644 index 0000000..6a72192 --- /dev/null +++ b/duetector/injectors/base.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import os +from collections import namedtuple + +from duetector.config import Configuable + + +class NamespaceMixin: + pass + + +class CgroupMixin: + pass + + +class Injector(Configuable): + """ + A base class for all Injectors. + + Default config scope is ``Injector.{class_name}``. + + subclass should override ``inject`` method and ``super`` it. + + User should call Injector() directly to Injector data, + """ + + default_config = { + "disabled": False, + } + """ + Default config for ``Injector``. + """ + + @property + def config_scope(self): + """ + Config scope for current Injector. + """ + return self.__class__.__name__ + + @property + def disabled(self): + """ + If current Injector is disabled. + """ + return self.config.disabled + + def inject(self, data: namedtuple) -> namedtuple: + """ + Implement this method to patch ``data`` + """ + return data + + def __call__(self, data: namedtuple) -> namedtuple | None: + if self.disabled: + return data + return self.inject(data) diff --git a/duetector/injectors/docker.py b/duetector/injectors/docker.py new file mode 100644 index 0000000..faa61ea --- /dev/null +++ b/duetector/injectors/docker.py @@ -0,0 +1,11 @@ +from duetector.extension.injector import hookimpl +from duetector.injectors.base import Injector + + +class DockerInjector(Injector): + pass + + +@hookimpl +def init_injector(config=None): + return DockerInjector(config=config) diff --git a/duetector/injectors/k8s.py b/duetector/injectors/k8s.py new file mode 100644 index 0000000..d645b43 --- /dev/null +++ b/duetector/injectors/k8s.py @@ -0,0 +1,11 @@ +from duetector.extension.injector import hookimpl +from duetector.injectors.base import Injector + + +class K8SInjector(Injector): + pass + + +@hookimpl +def init_injector(config=None): + return K8SInjector(config=config) diff --git a/duetector/injectors/register.py b/duetector/injectors/register.py new file mode 100644 index 0000000..c60385f --- /dev/null +++ b/duetector/injectors/register.py @@ -0,0 +1,4 @@ +# Expose for plugin system +from . import docker, k8s + +registers = [docker, k8s] diff --git a/duetector/managers/injector.py b/duetector/managers/injector.py new file mode 100644 index 0000000..1b6c88c --- /dev/null +++ b/duetector/managers/injector.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import sys +from typing import Any + +import pluggy + +import duetector.injectors.register +from duetector.extension.injector import project_name +from duetector.injectors.base import Injector +from duetector.log import logger +from duetector.managers.base import Manager + +PROJECT_NAME = project_name #: Default project name for pluggy +hookspec = pluggy.HookspecMarker(PROJECT_NAME) + + +@hookspec +def init_injector(config) -> Injector | None: + """ + Initialize Injector from config + None means the Injector is not available + Also the Injector can be disabled by config, Manager will discard disabled Injector + """ + + +class InjectorManager(Manager): + """ + Manager for all Injectors. + + Injectors are initialized from config, and can be ``disabled`` by config. + """ + + config_scope = "injector" + """ + Config scope for ``InjectorManager``. + """ + + def __init__(self, config: dict[str, Any] | None = None, *args, **kwargs): + super().__init__(config, *args, **kwargs) + + self.pm = pluggy.PluginManager(PROJECT_NAME) + self.pm.add_hookspecs(sys.modules[__name__]) + if self.include_extension: + self.pm.load_setuptools_entrypoints(PROJECT_NAME) + self.register(duetector.injectors.register) + + def init(self, ignore_disabled=True, *args, **kwargs) -> list[Injector]: + """ + Initialize all Injectors from config. + + Args: + ignore_disabled: Ignore disabled Injectors + """ + if self.disabled: + logger.info("InjectorManager disabled.") + return [] + objs = [] + for f in self.pm.hook.init_injector(config=self.config._config_dict): + if not f: + continue + if f.disabled and ignore_disabled: + logger.info(f"Injector {f.__class__.__name__} is disabled") + continue + + objs.append(f) + + return objs diff --git a/duetector/monitors/base.py b/duetector/monitors/base.py index fd32cc0..8b4c33a 100644 --- a/duetector/monitors/base.py +++ b/duetector/monitors/base.py @@ -2,6 +2,11 @@ from collections import namedtuple +from duetector.injectors.base import Injector +from duetector.managers.collector import CollectorManager +from duetector.managers.filter import FilterManager +from duetector.managers.injector import InjectorManager + try: from functools import cache except ImportError: @@ -30,7 +35,6 @@ class Monitor(Configuable): """ A list of tracers, should be initialized by ``TracerManager`` """ - filters: list[Filter] """ A list of filters, should be initialized by ``FilterManager`` @@ -39,6 +43,10 @@ class Monitor(Configuable): """ A list of collectors, should be initialized by ``CollectorManager`` """ + injectors: list[Injector] + """ + A list of collectors, should be initialized by ``InjectorManager`` + """ config_scope = "monitor" """ @@ -72,6 +80,16 @@ def __init__(self, config: dict[str, Any] | None = None, *args, **kwargs): self._backend = self._backend_imp(**self.backend_args._config_dict) self.poller = Poller(self.config._config_dict) + if self.disabled: + self.tracers = [] + self.filters = [] + self.collectors = [] + self.injectors = [] + return + self.filters: list[Filter] = FilterManager(config).init() + self.collectors: list[Collector] = CollectorManager(config).init() + self.injectors: list[Injector] = InjectorManager(config).init() + @property def disabled(self): """ @@ -126,18 +144,23 @@ def shutdown(self): c.shutdown() def _inject_extra_info(self, data: namedtuple) -> namedtuple: + for injector in self.injectors: + data = injector(data) return data @cache def _get_callback_fn(self, tracer) -> Callable[[namedtuple], None]: def _(data): - for filter in self.filters: - data = filter(data) - if not data: - return - data = self._inject_extra_info(data) - for collector in self.collectors: - collector.emit(tracer, data) + try: + for filter in self.filters: + data = filter(data) + if not data: + return + data = self._inject_extra_info(data) + for collector in self.collectors: + collector.emit(tracer, data) + except Exception as e: + logger.exception(e) return _ diff --git a/duetector/monitors/bcc_monitor.py b/duetector/monitors/bcc_monitor.py index fed7a9f..10f0220 100644 --- a/duetector/monitors/bcc_monitor.py +++ b/duetector/monitors/bcc_monitor.py @@ -3,9 +3,12 @@ from typing import Any, Callable from duetector.collectors.base import Collector +from duetector.filters.base import Filter +from duetector.injectors.base import Injector from duetector.log import logger from duetector.managers.collector import CollectorManager from duetector.managers.filter import FilterManager +from duetector.managers.injector import InjectorManager from duetector.managers.tracer import TracerManager from duetector.monitors.base import Monitor from duetector.tracers import BccTracer @@ -45,14 +48,9 @@ def __init__(self, config: dict[str, Any] | None = None, *args, **kwargs): super().__init__(config=config) if self.disabled: logger.info("BccMonitor disabled") - self.tracers = [] - self.filters = [] - self.collectors = [] return self.tracers: list[BccTracer] = TracerManager(config).init(tracer_type=BccTracer) # type: ignore - self.filters: list[Callable] = FilterManager(config).init() - self.collectors: list[Collector] = CollectorManager(config).init() self.bpf_tracers: dict[Any, BccTracer] = {} if self.auto_init: diff --git a/duetector/monitors/sh_monitor.py b/duetector/monitors/sh_monitor.py index be6debe..aecd2b7 100644 --- a/duetector/monitors/sh_monitor.py +++ b/duetector/monitors/sh_monitor.py @@ -5,6 +5,10 @@ from datetime import datetime from typing import Any, Callable +from duetector.filters.base import Filter +from duetector.injectors.base import Injector +from duetector.managers.injector import InjectorManager + try: from functools import cache except ImportError: @@ -124,14 +128,10 @@ def __init__(self, config: dict[str, Any] | None = None, *args, **kwargs): super().__init__(config=config, *args, **kwargs) if self.disabled: logger.info("ShMonitor disabled") - self.tracers = [] - self.filters = [] - self.collectors = [] return self.tracers: list[ShellTracer] = TracerManager(config).init(tracer_type=ShellTracer) # type: ignore - self.filters: list[Callable] = FilterManager(config).init() - self.collectors: list[Collector] = CollectorManager(config).init() + self.host = ShTracerHost(self._backend, self.timeout) if self.auto_init: self.init() diff --git a/duetector/monitors/subprocess_monitor.py b/duetector/monitors/subprocess_monitor.py index a4fe63e..0d5feba 100644 --- a/duetector/monitors/subprocess_monitor.py +++ b/duetector/monitors/subprocess_monitor.py @@ -10,9 +10,11 @@ import psutil from duetector.collectors.base import Collector +from duetector.injectors.base import Injector from duetector.log import logger from duetector.managers.collector import CollectorManager from duetector.managers.filter import FilterManager +from duetector.managers.injector import InjectorManager from duetector.managers.tracer import TracerManager from duetector.monitors.base import Monitor from duetector.proto.subprocess import ( @@ -243,14 +245,9 @@ def __init__(self, config: dict[str, Any] | None = None, *args, **kwargs): super().__init__(config=config, *args, **kwargs) if self.disabled: logger.info("SubprocessMonitor disabled") - self.tracers = [] - self.filters = [] - self.collectors = [] return self.tracers: list[SubprocessTracer] = TracerManager(config).init(tracer_type=SubprocessTracer) # type: ignore - self.filters: list[Callable] = FilterManager(config).init() - self.collectors: list[Collector] = CollectorManager(config).init() self.host = SubprocessHost( timeout=self.timeout, diff --git a/duetector/static/config.toml b/duetector/static/config.toml index 179c0ec..9a1bf74 100644 --- a/duetector/static/config.toml +++ b/duetector/static/config.toml @@ -124,6 +124,16 @@ table_prefix = "duetector_tracking" [analyzer.dbanalyzer.db.engine] url = "sqlite:///duetector-dbcollector.sqlite3" +[injector] +disabled = false +include_extension = true + +[injector.k8sinjector] +disabled = false + +[injector.dockerinjector] +disabled = false + [monitor.bcc] disabled = false auto_init = true diff --git a/duetector/tools/config_generator.py b/duetector/tools/config_generator.py index ce61f6a..93c4437 100644 --- a/duetector/tools/config_generator.py +++ b/duetector/tools/config_generator.py @@ -10,6 +10,7 @@ from duetector.managers.analyzer import AnalyzerManager from duetector.managers.collector import CollectorManager from duetector.managers.filter import FilterManager +from duetector.managers.injector import InjectorManager from duetector.managers.tracer import TracerManager from duetector.monitors import BccMonitor, ShMonitor, SubprocessMonitor from duetector.service.config import ServerConfig @@ -51,7 +52,7 @@ class ConfigGenerator: """ - managers = [FilterManager, TracerManager, CollectorManager, AnalyzerManager] + managers = [FilterManager, TracerManager, CollectorManager, AnalyzerManager, InjectorManager] """ All managers to inspect. """ diff --git a/pyproject.toml b/pyproject.toml index 38aa363..ed75fae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ dependencies = [ "SQLAlchemy>=2", "click", "psutil", + # Injector + "docker", + "kubernetes", # OTel "opentelemetry-sdk", "opentelemetry-api",