Feat/parser dp

docs/parsers.md

@@ -108,5 +108,6 @@ def test_my_parser_parse():
 - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates.
 - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure.
 - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data.
+- [Drain parser](parsers/drain_parser.md): Parser inspired by [drain publication](https://ieeexplore.ieee.org/document/8029742).
 
 Go back to [Index](index.md)

docs/parsers/drain_parser.md

@@ -0,0 +1,109 @@
+# Drain parser
+
+The parsed is based in the official [Drain publication](https://ieeexplore.ieee.org/document/8029742).
+
+This parser wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. Prefer  use the performance implementation when parsing many log lines in non-stream (batch) mode.
+
+|            | Schema                     | Description        |
+|------------|----------------------------|--------------------|
+| **Input**  | [LogSchema](../schemas.md) | Unstructured log   |
+| **Output** | [ParserSchema](../schemas.md) | Structured log   |
+
+WARNING: This parser is not yet in a stable release and may behave differently across platforms or hardware.
+
+## Configuration
+
+Drain parser arguments:
+
+- `method_type` (string): parser type identifier (for example `"tree_matcher"`).
+- `depth` (int): Number of word layers.
+- `max_childs` (int): max number of childs allow in the length layer.
+- `sim_thres` (float): similarity threshold.
+- `reset_in_post_train` (bool): if true remove the logs in the train buffer when the templates are generated. Otherwise, it safe them for the next train.
+- `auto_config` (bool): whether to attempt an optional auto-configuration phase (not required).
+
+Example YAML fragment:
+```yaml
+parsers:
+  DrainParser:
+    method_type: drain_parser
+    auto_config: False
+    params:
+      depth: 2
+```
+
+## Usage example
+
+Simple usage (Reset = False):
+
+```python
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+```
+
+Simple usage (Reset = True):
+
+```python
+from detectmatelibrary.parsers.drain import DrainParser
+from detectmatelibrary import schemas
+
+# instantiate parser (config can be a dict or a config object)
+config_dict = {
+    "parsers": {
+        "DrainParser": {
+            "method_type": "drain_parser",
+            "data_use_training": 2,
+            "reset_in_post_train": False,
+        }
+    }
+}
+
+parser = DrainParser(config=config_dict)
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"}))
+print(parsed["template"])  # "templates not yet generated"
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "hello there <*> kenobi"
+
+parser.update_state("keep_training")
+parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"}))
+parser.update_state("stop_training")
+
+parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"}))
+print(parsed["template"])  # "template not found"
+```
+
+Go back to [Index](../index.md)

mkdocs.yml

@@ -22,6 +22,7 @@ nav:
     - Template Tree Matcher: parsers/template_tree_matcher.md
     - Json Parser: parsers/json_parser.md
     - LogBatcher Parser: parsers/logbatcher_parser.md
+    - Drain Parser: parsers/drain_parser.md
   - Detectors Methods:
     - Random Detector: detectors/random_detector.md
     - New Value: detectors/new_value.md

pyproject.toml

@@ -13,7 +13,8 @@ dependencies = [
     "pyyaml>=6.0.3",
     "regex>=2025.11.3",
     "numpy>=2.3.2",
-    "detectmateperformance>=0.1.0",
+    #"detectmateperformance>=0.1.0",
+    "detectmateperformance @ git+https://github.com/ait-detectmate/DetectMatePerformance",
     "msgpack>=1.0.0",
     "fsspec>=2024.1.0",
     "pyarrow>=24.0.0",

src/detectmatelibrary/parsers/drain.py

@@ -0,0 +1,114 @@
+from detectmatelibrary.common.parser import CoreParser, CoreParserConfig
+from detectmatelibrary import schemas
+
+from detectmateperformance.match_tree import TreeMatcher
+from detectmateperformance.drain import Drain
+
+from detectmatelibrary.utils.finetune import Combinations
+
+from typing import Any
+
+
+class DrainConfig(CoreParserConfig):
+    method_type: str = "drain_parser"
+
+    depth: int = 2
+    max_childs: int = 10
+    sim_thres: float = 0.2
+
+    reset_in_post_train: bool = False
+
+    finetune: list[list[str | list[Any]]] = [
+        ["depth", [1, 2, 3, 4]],
+        ["max_childs", [10, 40]],
+        ["sim_thres", [0.2, 0.4, 0.6, 0.8]]
+    ]
+
+
+def _init_drain(config: DrainConfig) -> Drain:
+    return Drain(
+        depth=config.depth, max_child=config.max_childs, sim=config.sim_thres,
+    )
+
+
+def _found_ratio(logs: list[str], tree_matcher: TreeMatcher) -> float:
+    results = tree_matcher.match_batch(logs).get_all_templates()
+    print(results)
+
+    score = 0.0
+    for template in results:
+        if "template not found" == template:
+            score += 1.
+
+    return score / len(results)
+
+
+def _get_best_config(logs: list[str], config: DrainConfig) -> DrainConfig:
+
+    found_ratio: list[float] = []
+    length: list[int] = []
+
+    for config in (comb := Combinations(config))():  # type: ignore
+        drain = _init_drain(config)
+        for input_ in logs:
+            drain.add(input_)
+        tree_matcher = drain.generate()
+
+        found_ratio.append(_found_ratio(logs, tree_matcher))
+        length.append(len(tree_matcher))
+
+    n = max(length)
+    for le, sc in zip(length, found_ratio):
+        comb.add_value((float(le) / n) + sc)
+
+    new_config: DrainConfig = comb.get_best()  # type: ignore
+    return new_config
+
+
+class DrainParser(CoreParser):
+    def __init__(
+        self,
+        name: str = "DrainParser",
+        config: DrainConfig | dict[str, Any] = DrainConfig()
+    ) -> None:
+
+        if isinstance(config, dict):
+            config = DrainConfig.from_dict(config, name)
+        super().__init__(name=name, config=config)
+
+        self.config: DrainConfig
+        self.drain_gen = _init_drain(config=config)
+        self.tree_match: TreeMatcher | None = None
+
+        self.config_buffer: list[str] = []
+
+    def configure(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.config_buffer.append(input_["log"])
+
+    def set_configuration(self) -> None:
+        self.config = _get_best_config(self.config_buffer, config=self.config)
+        self.config_buffer = []
+
+    def train(self, input_: schemas.LogSchema) -> None:  # type: ignore
+        self.drain_gen.add(input_["log"])
+
+    def post_train(self) -> None:
+        self.tree_match = self.drain_gen.generate()
+        if self.config.reset_in_post_train:
+            self.drain_gen.reset()
+
+    def parse(
+        self,
+        input_: schemas.LogSchema,
+        output_: schemas.ParserSchema
+    ) -> None:
+
+        if self.tree_match is None:
+            output_["EventID"] = -1
+            output_["template"] = "templates not yet generated"
+        else:
+            parsed = self.tree_match.match_log(input_["log"], get_var=True)[0]
+
+            output_["EventID"] = parsed["EventID"]
+            output_["variables"].extend(parsed["ParamList"])
+            output_["template"] = parsed["Template"]

src/detectmatelibrary/utils/finetune.py

@@ -0,0 +1,51 @@
+from detectmatelibrary.common.core import CoreConfig
+
+from typing import Any
+import numpy as np
+
+import itertools
+import warnings
+import typing
+import copy
+
+
+class Combinations:
+    def __init__(self, config: CoreConfig) -> None:
+        self.config = config
+
+        self.paths: list[str] = []
+        self.combs: list[tuple[Any, ...]] = []
+        if "finetune" not in dir(config):
+            warnings.warn("No finetune options found")
+        else:
+            self.paths = [path[0] for path in getattr(config, "finetune")]
+            self.combs = list(itertools.product(
+                *[path[-1] for path in getattr(config, "finetune")]
+            ))
+        self.values: list[float] = []
+
+    def add_value(self, value: float) -> None:
+        self.values.append(value)
+
+    def get_best(self) -> CoreConfig:
+        if self.values == []:
+            return self.config
+
+        idx = np.argmin(self.values)
+        for i, combo in enumerate(self.combs):
+            if i == idx:
+                config = copy.deepcopy(self.config)
+                for path, value in zip(self.paths, combo):
+                    if path in dir(self.config):
+                        setattr(config, path, value)
+                return config
+        return self.config
+
+    def __call__(self) -> typing.Iterable[CoreConfig]:
+        for combo in self.combs:
+            config = copy.deepcopy(self.config)
+            for path, value in zip(self.paths, combo):
+                if path in dir(config):
+                    setattr(config, path, value)
+
+            yield config