diff --git a/docs/parsers.md b/docs/parsers.md index a862c87..b6e94cb 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -108,5 +108,6 @@ def test_my_parser_parse(): - [Template Matcher](parsers/template_matcher.md): matches logs against a predefined set of `<*>` templates. - [Template Tree Matcher](parsers/template_tree_matcher.md): matches logs against a predefined set of `<*>` templates using a tree structure. - [LogBatcher Parser](parsers/logbatcher_parser.md): LLM-based parser that infers templates from raw logs with no training data. +- [Drain parser](parsers/drain_parser.md): Parser inspired by [drain publication](https://ieeexplore.ieee.org/document/8029742). Go back to [Index](index.md) diff --git a/docs/parsers/drain_parser.md b/docs/parsers/drain_parser.md new file mode 100644 index 0000000..9201471 --- /dev/null +++ b/docs/parsers/drain_parser.md @@ -0,0 +1,109 @@ +# Drain parser + +The parsed is based in the official [Drain publication](https://ieeexplore.ieee.org/document/8029742). + +This parser wraps functionality from the DetectMatePerformance project: https://github.com/ait-detectmate/DetectMatePerformance. Prefer use the performance implementation when parsing many log lines in non-stream (batch) mode. + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [LogSchema](../schemas.md) | Unstructured log | +| **Output** | [ParserSchema](../schemas.md) | Structured log | + +WARNING: This parser is not yet in a stable release and may behave differently across platforms or hardware. + +## Configuration + +Drain parser arguments: + +- `method_type` (string): parser type identifier (for example `"tree_matcher"`). +- `depth` (int): Number of word layers. +- `max_childs` (int): max number of childs allow in the length layer. +- `sim_thres` (float): similarity threshold. +- `reset_in_post_train` (bool): if true remove the logs in the train buffer when the templates are generated. Otherwise, it safe them for the next train. +- `auto_config` (bool): whether to attempt an optional auto-configuration phase (not required). + +Example YAML fragment: +```yaml +parsers: + DrainParser: + method_type: drain_parser + auto_config: False + params: + depth: 2 +``` + +## Usage example + +Simple usage (Reset = False): + +```python +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" +``` + +Simple usage (Reset = True): + +```python +from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary import schemas + +# instantiate parser (config can be a dict or a config object) +config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "data_use_training": 2, + "reset_in_post_train": False, + } + } +} + +parser = DrainParser(config=config_dict) + +parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) +print(parsed["template"]) # "templates not yet generated" + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "hello there <*> kenobi" + +parser.update_state("keep_training") +parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) +parser.update_state("stop_training") + +parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) +print(parsed["template"]) # "template not found" +``` + +Go back to [Index](../index.md) diff --git a/mkdocs.yml b/mkdocs.yml index 49b0ece..623c5fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Template Tree Matcher: parsers/template_tree_matcher.md - Json Parser: parsers/json_parser.md - LogBatcher Parser: parsers/logbatcher_parser.md + - Drain Parser: parsers/drain_parser.md - Detectors Methods: - Random Detector: detectors/random_detector.md - New Value: detectors/new_value.md diff --git a/pyproject.toml b/pyproject.toml index 904edf5..ef97e9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ dependencies = [ "pyyaml>=6.0.3", "regex>=2025.11.3", "numpy>=2.3.2", - "detectmateperformance>=0.1.0", + #"detectmateperformance>=0.1.0", + "detectmateperformance @ git+https://github.com/ait-detectmate/DetectMatePerformance", "msgpack>=1.0.0", "fsspec>=2024.1.0", "pyarrow>=24.0.0", diff --git a/src/detectmatelibrary/parsers/drain.py b/src/detectmatelibrary/parsers/drain.py index e69de29..c10e589 100644 --- a/src/detectmatelibrary/parsers/drain.py +++ b/src/detectmatelibrary/parsers/drain.py @@ -0,0 +1,114 @@ +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig +from detectmatelibrary import schemas + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.drain import Drain + +from detectmatelibrary.utils.finetune import Combinations + +from typing import Any + + +class DrainConfig(CoreParserConfig): + method_type: str = "drain_parser" + + depth: int = 2 + max_childs: int = 10 + sim_thres: float = 0.2 + + reset_in_post_train: bool = False + + finetune: list[list[str | list[Any]]] = [ + ["depth", [1, 2, 3, 4]], + ["max_childs", [10, 40]], + ["sim_thres", [0.2, 0.4, 0.6, 0.8]] + ] + + +def _init_drain(config: DrainConfig) -> Drain: + return Drain( + depth=config.depth, max_child=config.max_childs, sim=config.sim_thres, + ) + + +def _found_ratio(logs: list[str], tree_matcher: TreeMatcher) -> float: + results = tree_matcher.match_batch(logs).get_all_templates() + print(results) + + score = 0.0 + for template in results: + if "template not found" == template: + score += 1. + + return score / len(results) + + +def _get_best_config(logs: list[str], config: DrainConfig) -> DrainConfig: + + found_ratio: list[float] = [] + length: list[int] = [] + + for config in (comb := Combinations(config))(): # type: ignore + drain = _init_drain(config) + for input_ in logs: + drain.add(input_) + tree_matcher = drain.generate() + + found_ratio.append(_found_ratio(logs, tree_matcher)) + length.append(len(tree_matcher)) + + n = max(length) + for le, sc in zip(length, found_ratio): + comb.add_value((float(le) / n) + sc) + + new_config: DrainConfig = comb.get_best() # type: ignore + return new_config + + +class DrainParser(CoreParser): + def __init__( + self, + name: str = "DrainParser", + config: DrainConfig | dict[str, Any] = DrainConfig() + ) -> None: + + if isinstance(config, dict): + config = DrainConfig.from_dict(config, name) + super().__init__(name=name, config=config) + + self.config: DrainConfig + self.drain_gen = _init_drain(config=config) + self.tree_match: TreeMatcher | None = None + + self.config_buffer: list[str] = [] + + def configure(self, input_: schemas.LogSchema) -> None: # type: ignore + self.config_buffer.append(input_["log"]) + + def set_configuration(self) -> None: + self.config = _get_best_config(self.config_buffer, config=self.config) + self.config_buffer = [] + + def train(self, input_: schemas.LogSchema) -> None: # type: ignore + self.drain_gen.add(input_["log"]) + + def post_train(self) -> None: + self.tree_match = self.drain_gen.generate() + if self.config.reset_in_post_train: + self.drain_gen.reset() + + def parse( + self, + input_: schemas.LogSchema, + output_: schemas.ParserSchema + ) -> None: + + if self.tree_match is None: + output_["EventID"] = -1 + output_["template"] = "templates not yet generated" + else: + parsed = self.tree_match.match_log(input_["log"], get_var=True)[0] + + output_["EventID"] = parsed["EventID"] + output_["variables"].extend(parsed["ParamList"]) + output_["template"] = parsed["Template"] diff --git a/src/detectmatelibrary/utils/finetune.py b/src/detectmatelibrary/utils/finetune.py new file mode 100644 index 0000000..c29c6d4 --- /dev/null +++ b/src/detectmatelibrary/utils/finetune.py @@ -0,0 +1,51 @@ +from detectmatelibrary.common.core import CoreConfig + +from typing import Any +import numpy as np + +import itertools +import warnings +import typing +import copy + + +class Combinations: + def __init__(self, config: CoreConfig) -> None: + self.config = config + + self.paths: list[str] = [] + self.combs: list[tuple[Any, ...]] = [] + if "finetune" not in dir(config): + warnings.warn("No finetune options found") + else: + self.paths = [path[0] for path in getattr(config, "finetune")] + self.combs = list(itertools.product( + *[path[-1] for path in getattr(config, "finetune")] + )) + self.values: list[float] = [] + + def add_value(self, value: float) -> None: + self.values.append(value) + + def get_best(self) -> CoreConfig: + if self.values == []: + return self.config + + idx = np.argmin(self.values) + for i, combo in enumerate(self.combs): + if i == idx: + config = copy.deepcopy(self.config) + for path, value in zip(self.paths, combo): + if path in dir(self.config): + setattr(config, path, value) + return config + return self.config + + def __call__(self) -> typing.Iterable[CoreConfig]: + for combo in self.combs: + config = copy.deepcopy(self.config) + for path, value in zip(self.paths, combo): + if path in dir(config): + setattr(config, path, value) + + yield config diff --git a/tests/test_parsers/test_drain.py b/tests/test_parsers/test_drain.py new file mode 100644 index 0000000..07aebb9 --- /dev/null +++ b/tests/test_parsers/test_drain.py @@ -0,0 +1,140 @@ +"""Most of the functionality is test it in DetectMatePerformance.""" +from detectmatelibrary.parsers.drain import DrainParser, _found_ratio + +from detectmateperformance.match_tree import TreeMatcher +from detectmateperformance.types_ import LogTemplates + +from detectmatelibrary import schemas + + +class TestDrainParser: + def test_train_process(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general R2D2!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_reset_after_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": True, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "template not found" + + def test_not_reset_train(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "data_use_training": 2, + "reset_in_post_train": False, + } + } + } + parser = DrainParser(config=config_dict) + + parsed = parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + assert parsed["EventID"] == -1 + assert parsed["template"] == "templates not yet generated" + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["EventID"] == 0 + assert parsed["template"] == "hello there <*> kenobi" + + parser.update_state("keep_training") + parsed = parser.process(schemas.LogSchema({"log": "bella ciao bella ciao"})) + parser.update_state("stop_training") + + parsed = parser.process(schemas.LogSchema({"log": "hello there, sargent kenobi!"})) + assert parsed["template"] == "hello there <*> kenobi" + + def test_not_ration_found(self): + tree_matcher = TreeMatcher(LogTemplates(["hello there <*> kenobi"])) + + logs = ["hello there general kenobi", "akuna matata"] + assert 0.5 == _found_ratio(logs, tree_matcher) + + logs = ["hello there general kenobi"] + assert 0. == _found_ratio(logs, tree_matcher) + + def test_no_auto_config_but_no_initialization(self): + config_dict = { + "parsers": { + "DrainParser": { + "method_type": "drain_parser", + "depth": 2, + "max_childs": 10, + "sim_thres": 0.2, + "auto_config": True, + "data_use_configure": 2, + "data_use_training": 2, + } + } + } + parser = DrainParser(config=config_dict) + parser.process(schemas.LogSchema({"log": "hello there, general kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + parser.process(schemas.LogSchema({"log": "hello there, captain kenobi!"})) + + assert parser.config.depth == 1 + assert parser.config.max_childs == 10 + assert parser.config.sim_thres == 0.2 diff --git a/tests/test_utils/test_finetune.py b/tests/test_utils/test_finetune.py new file mode 100644 index 0000000..feb9c0a --- /dev/null +++ b/tests/test_utils/test_finetune.py @@ -0,0 +1,82 @@ + +from detectmatelibrary.utils.finetune import Combinations + +from detectmatelibrary.common.core import CoreConfig + +from typing import Any +import pytest + + +class DummyConfig(CoreConfig): + a: int = 2 + b: int = 10 + c: float = 0.2 + + finetune: list[tuple[str, list[Any]]] = [ + ["a", [1, 2, 3, 4]], + ["b", [10, 40]], + ["c", [0.2, 0.4, 0.6, 0.8]] + ] + + +class DummyConfig2(CoreConfig): + a: int = 2 + b: int = 10 + c: float = 0.4 + + +class DummyConfig3(CoreConfig): + a: int = 2 + b: int = 10 + + finetune: list[tuple[str, list[Any]]] = [ + ["a", [1, 2, 3, 4]], + ["b", [10, 40]], + ["c", [0.2, 0.4, 0.6, 0.8]] + ] + + +class TestCombinations: + def test_combination_no_overwrite(self): + comb = Combinations(config := DummyConfig()) + assert config != next(comb()) + + def test_call_format(self): + comb = Combinations(DummyConfig()) + config = next(comb()) + + assert config.a == 1 + assert config.b == 10 + assert config.c == 0.2 + + def test_get_best(self): + comb = Combinations(DummyConfig()) + for j, _ in enumerate(comb()): + comb.add_value(j) + + best_config = comb.get_best() + assert best_config.a == 1 + assert best_config.b == 10 + assert best_config.c == 0.2 + + def test_finetune_not_found(self): + with pytest.warns(UserWarning): + comb = Combinations(DummyConfig2()) + + for _ in comb(): + assert False + config = comb.get_best() + + assert config.a == 2 + assert config.b == 10 + assert config.c == 0.4 + + def test_argument_missing(self): + comb = Combinations(DummyConfig3()) + + for j, _ in enumerate(comb()): + comb.add_value(j) + config = comb.get_best() + + assert config.a == 1 + assert config.b == 10 diff --git a/uv.lock b/uv.lock index d2ccd2b..6c7df30 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -247,7 +247,7 @@ requires-dist = [ { name = "detectmatelibrary", extras = ["dataframes"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["llm"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["polars-rtcompat"], marker = "extra == 'full'" }, - { name = "detectmateperformance", specifier = ">=0.1.0" }, + { name = "detectmateperformance", git = "https://github.com/ait-detectmate/DetectMatePerformance" }, { name = "fsspec", specifier = ">=2024.1.0" }, { name = "msgpack", specifier = ">=1.0.0" }, { name = "numpy", specifier = ">=2.3.2" }, @@ -280,7 +280,7 @@ dev = [ [[package]] name = "detectmateperformance" version = "0.1.0" -source = { registry = "https://pypi.org/simple" } +source = { git = "https://github.com/ait-detectmate/DetectMatePerformance#a5bb075b0bd15e406430e77ccbfcf6fe94a50ab0" } dependencies = [ { name = "levenshtein" }, { name = "numpy" }, @@ -289,10 +289,6 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e8/2b/7ff298303c5ba3ba4899d1f3854b214f63a4fac4eff191012f5608f5149e/detectmateperformance-0.1.0.tar.gz", hash = "sha256:2630d509e7e2bbe6b5bcc857e8f25dfae7d99e9e2cc52f5981da9171600531dc", size = 552409, upload-time = "2026-06-12T10:42:13.326Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/dd/3db227cd60d170203ab82dd2e670bcb50c6111769c39ac8dd64935fa4c1d/detectmateperformance-0.1.0-py3-none-any.whl", hash = "sha256:db70f239e53aa17a983f99948c9239c143744c03044eeae454f546fedbaf69a5", size = 557939, upload-time = "2026-06-12T10:42:11.526Z" }, -] [[package]] name = "distro"