From 275bacf985ca059a8c10308eb8c5c9a3c675029e Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Wed, 9 Sep 2026 17:19:31 -0300 Subject: [PATCH] chore(plugins): remove obsolete feeds egress plugin and its Dockerfile reference --- event_processor.Dockerfile | 1 - plugins/feeds/README.md | 18 - plugins/feeds/config/config.go | 43 -- plugins/feeds/config/const.go | 27 - plugins/feeds/go.mod | 61 -- plugins/feeds/go.sum | 216 ------- .../association/association_builder.go | 144 ----- .../association/association_context.go | 40 -- .../internal/association/association_rules.go | 524 ----------------- .../feeds/internal/client/backend_client.go | 232 -------- plugins/feeds/internal/client/cm_client.go | 79 --- plugins/feeds/internal/client/dependencies.go | 31 - .../internal/client/opensearch_client.go | 103 ---- .../internal/client/threadwinds_client.go | 170 ------ .../internal/client/threadwinds_setup.go | 58 -- .../internal/extractor/field_extractor.go | 535 ------------------ plugins/feeds/internal/initializer/app.go | 72 --- plugins/feeds/internal/initializer/clients.go | 33 -- .../feeds/internal/initializer/pipeline.go | 41 -- plugins/feeds/internal/initializer/setup.go | 25 - .../feeds/internal/mapper/entity_mapper.go | 140 ----- plugins/feeds/internal/models/alert.go | 8 - plugins/feeds/internal/models/event.go | 112 ---- plugins/feeds/internal/models/incident.go | 21 - .../internal/scheduler/ingestion_scheduler.go | 121 ---- .../feeds/internal/service/alert_processor.go | 58 -- .../feeds/internal/service/entity_builder.go | 105 ---- .../internal/service/incident_processor.go | 69 --- plugins/feeds/main.go | 69 --- plugins/feeds/utils/aes.go | 11 - plugins/feeds/utils/env.go | 23 - plugins/feeds/utils/files.go | 40 -- plugins/feeds/utils/retry.go | 156 ----- 33 files changed, 3386 deletions(-) delete mode 100644 plugins/feeds/README.md delete mode 100644 plugins/feeds/config/config.go delete mode 100644 plugins/feeds/config/const.go delete mode 100644 plugins/feeds/go.mod delete mode 100644 plugins/feeds/go.sum delete mode 100644 plugins/feeds/internal/association/association_builder.go delete mode 100644 plugins/feeds/internal/association/association_context.go delete mode 100644 plugins/feeds/internal/association/association_rules.go delete mode 100644 plugins/feeds/internal/client/backend_client.go delete mode 100644 plugins/feeds/internal/client/cm_client.go delete mode 100644 plugins/feeds/internal/client/dependencies.go delete mode 100644 plugins/feeds/internal/client/opensearch_client.go delete mode 100644 plugins/feeds/internal/client/threadwinds_client.go delete mode 100644 plugins/feeds/internal/client/threadwinds_setup.go delete mode 100644 plugins/feeds/internal/extractor/field_extractor.go delete mode 100644 plugins/feeds/internal/initializer/app.go delete mode 100644 plugins/feeds/internal/initializer/clients.go delete mode 100644 plugins/feeds/internal/initializer/pipeline.go delete mode 100644 plugins/feeds/internal/initializer/setup.go delete mode 100644 plugins/feeds/internal/mapper/entity_mapper.go delete mode 100644 plugins/feeds/internal/models/alert.go delete mode 100644 plugins/feeds/internal/models/event.go delete mode 100644 plugins/feeds/internal/models/incident.go delete mode 100644 plugins/feeds/internal/scheduler/ingestion_scheduler.go delete mode 100644 plugins/feeds/internal/service/alert_processor.go delete mode 100644 plugins/feeds/internal/service/entity_builder.go delete mode 100644 plugins/feeds/internal/service/incident_processor.go delete mode 100644 plugins/feeds/main.go delete mode 100644 plugins/feeds/utils/aes.go delete mode 100644 plugins/feeds/utils/env.go delete mode 100644 plugins/feeds/utils/files.go delete mode 100644 plugins/feeds/utils/retry.go diff --git a/event_processor.Dockerfile b/event_processor.Dockerfile index b0dac424c..2375d2699 100644 --- a/event_processor.Dockerfile +++ b/event_processor.Dockerfile @@ -18,5 +18,4 @@ COPY ./plugins/stats/com.utmstack.stats.plugin /workdir/plugins/utmstack/ COPY ./plugins/soc-ai/com.utmstack.soc-ai.plugin /workdir/plugins/utmstack/ COPY ./plugins/modules-config/com.utmstack.modules-config.plugin /workdir/plugins/utmstack/ COPY ./plugins/crowdstrike/com.utmstack.crowdstrike.plugin /workdir/plugins/utmstack/ -COPY ./plugins/feeds/com.utmstack.feeds.plugin /workdir/plugins/utmstack/ COPY ./plugins/rule-flood-guard/com.utmstack.rule-flood-guard.plugin /workdir/plugins/utmstack/ \ No newline at end of file diff --git a/plugins/feeds/README.md b/plugins/feeds/README.md deleted file mode 100644 index b8ddfc1cf..000000000 --- a/plugins/feeds/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# UTMStack Plugin for ThreadWinds Ingestion - - -## Description - -UTMStack Plugin for ThreadWinds Ingestion is a connector developed in Golang that extracts security entities from `UTMStack incidents and alerts` and sends them to the `ThreadWinds` threat intelligence platform. - -This plugin processes incidents from UTMStack, extracts entities (IPs, domains, hashes, emails, etc.) from their associated alerts and events, and ingests them into ThreadWinds for global threat intelligence correlation and enrichment. - -The connector automatically registers with ThreadWinds services using the admin email from the UTMStack system. It periodically polls for recent incidents, extracts all relevant entities (network indicators, file hashes, user identities, etc.), builds associations between entities, and sends them to ThreadWinds for analysis. - -### Requirements -**ThreadWinds Credentials:** - -- API Key -- API Secret - -Please note that the connector automatically registers with ThreadWinds using the admin email if credentials are not already configured. The connector requires a valid admin email to run. diff --git a/plugins/feeds/config/config.go b/plugins/feeds/config/config.go deleted file mode 100644 index 75ad76097..000000000 --- a/plugins/feeds/config/config.go +++ /dev/null @@ -1,43 +0,0 @@ -package config - -import ( - "github.com/threatwinds/go-sdk/plugins" -) - -type TWConfig struct { - InternalKey string - BackendURL string - ThreadWindsURL string - OpenSearchHost string - OpenSearchPort string - OpenSearchUser string - OpenSearchPassword string - DBHost string - DBPort string - DBUser string - DBPassword string - DBName string -} - -func GetTWConfig() (*TWConfig, error) { - utmCfg := plugins.PluginCfg("com.utmstack") - osCfg := plugins.PluginCfg("org.opensearch").Get("opensearch") - pgCfg := utmCfg.Get("postgresql") - - cfg := &TWConfig{ - InternalKey: utmCfg.Get("internalKey").String(), - BackendURL: utmCfg.Get("backend").String(), - ThreadWindsURL: GetThreadWindsURL(), - OpenSearchHost: osCfg.Get("host").String(), - OpenSearchPort: osCfg.Get("port").String(), - OpenSearchUser: osCfg.Get("user").String(), - OpenSearchPassword: osCfg.Get("password").String(), - DBHost: pgCfg.Get("server").String(), - DBPort: pgCfg.Get("port").String(), - DBUser: pgCfg.Get("user").String(), - DBPassword: pgCfg.Get("password").String(), - DBName: pgCfg.Get("database").String(), - } - - return cfg, nil -} diff --git a/plugins/feeds/config/const.go b/plugins/feeds/config/const.go deleted file mode 100644 index 16c0fbacc..000000000 --- a/plugins/feeds/config/const.go +++ /dev/null @@ -1,27 +0,0 @@ -package config - -import ( - "strings" - - "github.com/threatwinds/go-sdk/plugins" -) - -func GetThreadWindsURL() string { - if isDevEnvironment() { - return "https://apis.dev.threatwinds.com" - } - return "https://apis.threatwinds.com" -} - -func isDevEnvironment() bool { - env := plugins.PluginCfg("com.utmstack").Get("env").String() - if env != "" { - if strings.Contains(env, "dev") || - strings.Contains(env, "qa") || - strings.Contains(env, "rc") { - return true - } - } - - return false -} diff --git a/plugins/feeds/go.mod b/plugins/feeds/go.mod deleted file mode 100644 index 813f51a8b..000000000 --- a/plugins/feeds/go.mod +++ /dev/null @@ -1,61 +0,0 @@ -module github.com/utmstack/UTMStack/plugins/feeds - -go 1.26.0 - -require ( - github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 - github.com/opensearch-project/opensearch-go/v2 v2.3.0 - github.com/threatwinds/go-sdk v1.1.31 - golang.org/x/sync v0.23.0 - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - cel.dev/expr v0.25.2 // indirect - github.com/antlr4-go/antlr/v4 v4.13.1 // indirect - github.com/bytedance/gopkg v0.1.4 // indirect - github.com/bytedance/sonic v1.15.2 // indirect - github.com/bytedance/sonic/loader v0.5.1 // indirect - github.com/cloudwego/base64x v0.1.7 // indirect - github.com/gabriel-vasile/mimetype v1.4.13 // indirect - github.com/gin-contrib/sse v1.1.1 // indirect - github.com/gin-gonic/gin v1.12.0 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.30.3 // indirect - github.com/goccy/go-json v0.10.6 // indirect - github.com/goccy/go-yaml v1.19.2 // indirect - github.com/google/cel-go v0.28.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/leodido/go-urn v1.4.0 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/opensearch-project/opensearch-go/v4 v4.6.0 // indirect - github.com/pelletier/go-toml/v2 v2.3.1 // indirect - github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.59.1 // indirect - github.com/tidwall/gjson v1.19.0 // indirect - github.com/tidwall/match v1.2.0 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.3.1 // indirect - go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.27.0 // indirect - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.81.1 // indirect - google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect -) diff --git a/plugins/feeds/go.sum b/plugins/feeds/go.sum deleted file mode 100644 index 310cbc08c..000000000 --- a/plugins/feeds/go.sum +++ /dev/null @@ -1,216 +0,0 @@ -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0 h1:TBiBl9KCa4i4epY0/q9WSC4ugavL6+6JUkOXWDnMM6I= -github.com/AtlasInsideCorp/AtlasInsideAES v1.0.0/go.mod h1:cRhQ3TS/VEfu/z+qaciyuDZdtxgaXgaX8+G6Wa5NzBk= -github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= -github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= -github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM= -github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4= -github.com/bytedance/sonic v1.15.2 h1:90H+rcF/FwLXwfB1cudOLq/je83n683Utf4Cbp0xHCo= -github.com/bytedance/sonic v1.15.2/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= -github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= -github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI= -github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= -github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= -github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko= -github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s= -github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= -github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= -github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= -github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= -github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= -github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= -github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= -github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= -github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= -github.com/opensearch-project/opensearch-go/v4 v4.6.0 h1:Ac8aLtDSmLEyOmv0r1qhQLw3b4vcUhE42NE9k+Z4cRc= -github.com/opensearch-project/opensearch-go/v4 v4.6.0/go.mod h1:3iZtb4SNt3IzaxavKq0dURh1AmtVgYW71E4XqmYnIiQ= -github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= -github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= -github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= -github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/threatwinds/go-sdk v1.1.31 h1:H8QNjm+q2zwvOxp+htM5XX/4QDyiZn4Np62MhhUQu7A= -github.com/threatwinds/go-sdk v1.1.31/go.mod h1:aN6Oe3zJop9ngS83oZcKFXDLKWzrny2XhkYm7uoyDbQ= -github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= -github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= -github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ= -github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/5qi8= -go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= -go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= -golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= -golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= -golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/plugins/feeds/internal/association/association_builder.go b/plugins/feeds/internal/association/association_builder.go deleted file mode 100644 index 389718622..000000000 --- a/plugins/feeds/internal/association/association_builder.go +++ /dev/null @@ -1,144 +0,0 @@ -package association - -import ( - "sync" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" -) - -type AssociationBuilder struct { - rules []*AssociationRule - entityRegistry *sync.Map - seenHashes *sync.Map - mu sync.RWMutex -} - -func NewAssociationBuilder() *AssociationBuilder { - builder := &AssociationBuilder{ - rules: GetEnabledRules(), - entityRegistry: &sync.Map{}, - seenHashes: &sync.Map{}, - } - catcher.Info("association builder initialized", map[string]any{ - "total_rules": len(builder.rules), - }) - return builder -} - -func (b *AssociationBuilder) RegisterEntity(entity *entities.Entity, entityID, sourcePath string, ctx AssociationContext) { - ref := &EntityReference{ - Entity: entity, - EntityID: entityID, - EntityType: entity.Type, - SourcePath: sourcePath, - Context: ctx, - } - b.entityRegistry.Store(entityID, ref) -} - -func (b *AssociationBuilder) BuildAssociations() []*entities.Entity { - contextGroups := b.groupByContext() - for _, refs := range contextGroups { - b.detectAssociationsInContext(refs) - } - result := make([]*entities.Entity, 0, 256) - b.entityRegistry.Range(func(key, value any) bool { - if ref, ok := value.(*EntityReference); ok { - if entity, ok := ref.Entity.(*entities.Entity); ok { - result = append(result, entity) - } - } - return true - }) - return result -} - -func (b *AssociationBuilder) groupByContext() map[string][]*EntityReference { - groups := make(map[string][]*EntityReference) - b.entityRegistry.Range(func(key, value any) bool { - if ref, ok := value.(*EntityReference); ok { - contextKey := ref.Context.AlertID - if contextKey != "" { - groups[contextKey] = append(groups[contextKey], ref) - } - } - return true - }) - return groups -} - -func (b *AssociationBuilder) detectAssociationsInContext(refs []*EntityReference) { - for _, rule := range b.rules { - if !rule.Enabled { - continue - } - for i, sourceRef := range refs { - if sourceRef.EntityType != rule.SourceType { - continue - } - for j, targetRef := range refs { - if i == j { - continue - } - if targetRef.EntityType != rule.TargetType { - continue - } - if b.shouldCreateAssociation(sourceRef, targetRef) { - b.createAssociation(sourceRef, targetRef, rule) - } - } - } - } -} - -func (b *AssociationBuilder) shouldCreateAssociation(source, target *EntityReference) bool { - if source.Context.SameEvent(target.Context) { - return true - } - - if source.Context.IsOriginToTarget(target.Context) { - return true - } - - if source.Context.CrossEventAssociation(target.Context) { - return true - } - - return false -} - -func (b *AssociationBuilder) createAssociation(source, target *EntityReference, rule *AssociationRule) { - sourceEntity, ok := source.Entity.(*entities.Entity) - if !ok { - return - } - targetEntity, ok := target.Entity.(*entities.Entity) - if !ok { - return - } - associatedEntity := entities.EntityAssociation{ - Mode: string(rule.Mode), - Entity: entities.Entity{ - Type: targetEntity.Type, - Attributes: targetEntity.Attributes, - }, - } - if sourceEntity.Associations == nil { - sourceEntity.Associations = make([]entities.EntityAssociation, 0, 50) - } - sourceEntity.Associations = append(sourceEntity.Associations, associatedEntity) -} - -func (b *AssociationBuilder) CountAssociations(entities []*entities.Entity) int { - count := 0 - for _, entity := range entities { - count += len(entity.Associations) - } - return count -} - -func (b *AssociationBuilder) ClearRegistry() { - b.entityRegistry = &sync.Map{} - b.seenHashes = &sync.Map{} -} diff --git a/plugins/feeds/internal/association/association_context.go b/plugins/feeds/internal/association/association_context.go deleted file mode 100644 index 8121f8c38..000000000 --- a/plugins/feeds/internal/association/association_context.go +++ /dev/null @@ -1,40 +0,0 @@ -package association - -type AssociationContext struct { - AlertID string - EventID string - IncidentID string - SourceField string -} - -type EntityReference struct { - Entity any - EntityID string - EntityType string - SourcePath string - Context AssociationContext -} - -func (ctx *AssociationContext) IsOrigin() bool { - return ctx.SourceField == "origin" -} - -func (ctx *AssociationContext) IsTarget() bool { - return ctx.SourceField == "target" -} - -func (ctx *AssociationContext) SameAlert(other AssociationContext) bool { - return ctx.AlertID != "" && ctx.AlertID == other.AlertID -} - -func (ctx *AssociationContext) SameEvent(other AssociationContext) bool { - return ctx.EventID != "" && ctx.EventID == other.EventID -} - -func (ctx *AssociationContext) IsOriginToTarget(other AssociationContext) bool { - return ctx.SameEvent(other) && ctx.IsOrigin() && other.IsTarget() -} - -func (ctx *AssociationContext) CrossEventAssociation(other AssociationContext) bool { - return ctx.SameAlert(other) && ctx.EventID != "" && other.EventID != "" && ctx.EventID != other.EventID -} diff --git a/plugins/feeds/internal/association/association_rules.go b/plugins/feeds/internal/association/association_rules.go deleted file mode 100644 index 0a657c6ed..000000000 --- a/plugins/feeds/internal/association/association_rules.go +++ /dev/null @@ -1,524 +0,0 @@ -package association - -type AssociationMode string - -const ( - Association AssociationMode = "association" - Aggregation AssociationMode = "aggregation" -) - -type AssociationRule struct { - Name string - SourceType string - TargetType string - Mode AssociationMode - Enabled bool -} - -var DefaultRules = []*AssociationRule{ - // ==================== Network Associations ==================== - // Bidirectional IP-Domain relationships - { - Name: "ip-to-domain", - SourceType: "ip", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "domain-to-ip", - SourceType: "domain", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - - // Bidirectional IP-Hostname relationships - { - Name: "hostname-to-ip", - SourceType: "hostname", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-hostname", - SourceType: "ip", - TargetType: "hostname", - Mode: Association, - Enabled: true, - }, - - // IP Network Infrastructure - { - Name: "ip-to-port", - SourceType: "ip", - TargetType: "port", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "ip-to-mac", - SourceType: "ip", - TargetType: "mac-address", - Mode: Association, - Enabled: true, - }, - { - Name: "mac-to-ip", - SourceType: "mac-address", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-asn", - SourceType: "ip", - TargetType: "asn", - Mode: Association, - Enabled: true, - }, - - // URL relationships - { - Name: "url-to-domain", - SourceType: "url", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "url-to-ip", - SourceType: "url", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - - // ==================== Email Associations ==================== - { - Name: "email-to-emailaddress", - SourceType: "email", - TargetType: "email-address", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "email-to-domain", - SourceType: "email", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "email-to-ip", - SourceType: "email", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "emailaddress-to-domain", - SourceType: "email-address", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "domain-to-emailaddress", - SourceType: "domain", - TargetType: "email-address", - Mode: Association, - Enabled: true, - }, - { - Name: "dkim-to-domain", - SourceType: "dkim", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "dkim-signature-to-email", - SourceType: "dkim-signature", - TargetType: "email", - Mode: Aggregation, - Enabled: true, - }, - - // ==================== File Associations ==================== - { - Name: "file-to-path", - SourceType: "file", - TargetType: "path", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-mimetype", - SourceType: "file", - TargetType: "mime-type", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-ip", - SourceType: "file", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-url", - SourceType: "file", - TargetType: "url", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-domain", - SourceType: "file", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - - // File Hash Associations - { - Name: "file-to-md5", - SourceType: "file", - TargetType: "md5", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "file-to-sha1", - SourceType: "file", - TargetType: "sha1", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "file-to-sha256", - SourceType: "file", - TargetType: "sha256", - Mode: Aggregation, - Enabled: true, - }, - - // ==================== Malware Associations ==================== - { - Name: "malware-to-domain", - SourceType: "malware", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-ip", - SourceType: "malware", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-file", - SourceType: "malware", - TargetType: "file", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "file-to-malware", - SourceType: "file", - TargetType: "malware", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-url", - SourceType: "malware", - TargetType: "url", - Mode: Association, - Enabled: true, - }, - { - Name: "malware-to-process", - SourceType: "malware", - TargetType: "process", - Mode: Association, - Enabled: true, - }, - { - Name: "process-to-malware", - SourceType: "process", - TargetType: "malware", - Mode: Association, - Enabled: true, - }, - - // ==================== Certificate and Fingerprint Associations ==================== - { - Name: "certificate-to-domain", - SourceType: "certificate-fingerprint", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "domain-to-certificate", - SourceType: "domain", - TargetType: "certificate-fingerprint", - Mode: Association, - Enabled: true, - }, - { - Name: "certificate-to-ip", - SourceType: "certificate-fingerprint", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ja3-to-ip", - SourceType: "ja3-fingerprint", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "jarm-to-ip", - SourceType: "jarm-fingerprint", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "jarm-to-domain", - SourceType: "jarm-fingerprint", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "ssh-fingerprint-to-ip", - SourceType: "ssh-fingerprint", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ssh-banner-to-ip", - SourceType: "ssh-banner", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - - // ==================== System and Process Associations ==================== - { - Name: "process-to-ip", - SourceType: "process", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-process", - SourceType: "ip", - TargetType: "process", - Mode: Association, - Enabled: true, - }, - { - Name: "process-to-file", - SourceType: "process", - TargetType: "file", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-process", - SourceType: "file", - TargetType: "process", - Mode: Association, - Enabled: true, - }, - { - Name: "process-to-command", - SourceType: "process", - TargetType: "command", - Mode: Aggregation, - Enabled: true, - }, - { - Name: "process-to-user", - SourceType: "process", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-process", - SourceType: "username", - TargetType: "process", - Mode: Association, - Enabled: true, - }, - { - Name: "command-to-user", - SourceType: "command", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-command", - SourceType: "username", - TargetType: "command", - Mode: Association, - Enabled: true, - }, - { - Name: "windows-task-to-user", - SourceType: "windows-scheduled-task", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - { - Name: "windows-service-to-file", - SourceType: "windows-service-name", - TargetType: "file", - Mode: Association, - Enabled: true, - }, - { - Name: "hostname-to-user", - SourceType: "hostname", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - - // ==================== Vulnerability Associations ==================== - { - Name: "cve-to-cpe", - SourceType: "cve", - TargetType: "cpe", - Mode: Association, - Enabled: true, - }, - { - Name: "cpe-to-cve", - SourceType: "cpe", - TargetType: "cve", - Mode: Association, - Enabled: true, - }, - { - Name: "cve-to-file", - SourceType: "cve", - TargetType: "file", - Mode: Association, - Enabled: true, - }, - { - Name: "file-to-cve", - SourceType: "file", - TargetType: "cve", - Mode: Association, - Enabled: true, - }, - { - Name: "cve-to-ip", - SourceType: "cve", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-cve", - SourceType: "ip", - TargetType: "cve", - Mode: Association, - Enabled: true, - }, - - // ==================== Web Associations ==================== - { - Name: "cookie-to-domain", - SourceType: "cookie", - TargetType: "domain", - Mode: Association, - Enabled: true, - }, - { - Name: "domain-to-cookie", - SourceType: "domain", - TargetType: "cookie", - Mode: Association, - Enabled: true, - }, - - // ==================== Identity Associations ==================== - { - Name: "user-to-ip", - SourceType: "username", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, - { - Name: "ip-to-user", - SourceType: "ip", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-group", - SourceType: "username", - TargetType: "group", - Mode: Association, - Enabled: true, - }, - { - Name: "group-to-user", - SourceType: "group", - TargetType: "username", - Mode: Association, - Enabled: true, - }, - { - Name: "user-to-hostname", - SourceType: "username", - TargetType: "hostname", - Mode: Association, - Enabled: true, - }, - { - Name: "hostname-to-group", - SourceType: "hostname", - TargetType: "group", - Mode: Association, - Enabled: true, - }, - { - Name: "jabber-to-ip", - SourceType: "jabber-id", - TargetType: "ip", - Mode: Association, - Enabled: true, - }, -} - -func GetEnabledRules() []*AssociationRule { - rules := make([]*AssociationRule, 0, len(DefaultRules)) - for _, rule := range DefaultRules { - if rule.Enabled { - rules = append(rules, rule) - } - } - return rules -} diff --git a/plugins/feeds/internal/client/backend_client.go b/plugins/feeds/internal/client/backend_client.go deleted file mode 100644 index 543c22845..000000000 --- a/plugins/feeds/internal/client/backend_client.go +++ /dev/null @@ -1,232 +0,0 @@ -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" - "github.com/utmstack/UTMStack/plugins/feeds/utils" -) - -const ( - threadwindsSectionID = 10 -) - -type BackendClient struct { - baseURL string - internalKey string - httpClient *http.Client -} - -func NewBackendClient(cfg *config.TWConfig) *BackendClient { - return &BackendClient{ - baseURL: cfg.BackendURL, - internalKey: cfg.InternalKey, - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - } -} - -func (c *BackendClient) GetRecentIncidents(ctx context.Context) ([]*models.Incident, error) { - url := fmt.Sprintf("%s/api/utm-incidents?incidentStatus.in=OPEN,IN_REVIEW&sort=incidentCreatedDate,desc&size=100", c.baseURL) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, catcher.Error("failed to create request", err, nil) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, catcher.Error("request failed", err, nil) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, catcher.Error("unexpected status from backend", nil, map[string]any{ - "status": resp.StatusCode, - "body": string(body), - }) - } - - var incidents []*models.Incident - if err := json.NewDecoder(resp.Body).Decode(&incidents); err != nil { - return nil, catcher.Error("failed to decode response", err, nil) - } - - return incidents, nil -} - -func (c *BackendClient) GetIncidentAlerts(ctx context.Context, incidentID int64) ([]*models.IncidentAlert, error) { - url := fmt.Sprintf("%s/api/utm-incident-alerts?incidentId.equals=%d", c.baseURL, incidentID) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, catcher.Error("failed to create request", err, nil) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, catcher.Error("request failed", err, nil) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, catcher.Error("unexpected status from backend", nil, map[string]any{ - "status": resp.StatusCode, - "body": string(body), - }) - } - - var alerts []*models.IncidentAlert - if err := json.NewDecoder(resp.Body).Decode(&alerts); err != nil { - return nil, catcher.Error("failed to decode response", err, nil) - } - - return alerts, nil -} - -type ThreadWindsConfig struct { - APIKey string - APISecret string - Enabled string - KeyID int64 - SecretID int64 -} - -type ConfigParameter struct { - ID int64 `json:"id"` - SectionID int64 `json:"sectionId"` - ConfParamShort string `json:"confParamShort"` - ConfParamLarge string `json:"confParamLarge,omitempty"` - ConfParamDescription string `json:"confParamDescription,omitempty"` - ConfParamValue string `json:"confParamValue"` - ConfParamRequired bool `json:"confParamRequired,omitempty"` - ConfParamDatatype string `json:"confParamDatatype,omitempty"` -} - -func (c *BackendClient) GetThreadWindsConfig(ctx context.Context) (*ThreadWindsConfig, error) { - url := fmt.Sprintf("%s/api/utm-configuration-parameters?sectionId.equals=%d&size=100", c.baseURL, threadwindsSectionID) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, catcher.Error("failed to create request", err, nil) - } - - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, catcher.Error("request failed", err, nil) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, catcher.Error("unexpected status from backend", nil, map[string]any{ - "status": resp.StatusCode, - "body": string(body), - }) - } - - var params []ConfigParameter - if err := json.NewDecoder(resp.Body).Decode(¶ms); err != nil { - return nil, catcher.Error("failed to decode response", err, nil) - } - - config := &ThreadWindsConfig{} - - for _, param := range params { - switch param.ConfParamShort { - case "utmstack.tw.enabled": - config.Enabled = param.ConfParamValue - case "utmstack.tw.apiKey": - config.APIKey = param.ConfParamValue - config.KeyID = param.ID - case "utmstack.tw.apiSecret": - if param.ConfParamDatatype == "password" && param.ConfParamValue != "" { - decrypted, err := utils.DecryptValue(param.ConfParamValue) - if err != nil { - return nil, catcher.Error("failed to decrypt API Secret", err, nil) - } - config.APISecret = decrypted - } else { - config.APISecret = param.ConfParamValue - } - config.SecretID = param.ID - } - } - - return config, nil -} - -func (c *BackendClient) SaveThreadWindsCredentials(ctx context.Context, apiKey, apiSecret string, keyID, secretID int64) error { - const threadwindsSectionID = 6 - url := fmt.Sprintf("%s/api/utm-configuration-parameters", c.baseURL) - - params := []ConfigParameter{ - { - ID: keyID, - SectionID: threadwindsSectionID, - ConfParamShort: "utmstack.tw.apiKey", - ConfParamLarge: "ThreatWinds API Key", - ConfParamDescription: "API Key for ThreatWinds integration.", - ConfParamValue: apiKey, - ConfParamRequired: true, - ConfParamDatatype: "text", - }, - { - ID: secretID, - SectionID: threadwindsSectionID, - ConfParamShort: "utmstack.tw.apiSecret", - ConfParamLarge: "ThreatWinds API Secret", - ConfParamDescription: "API Secret for ThreatWinds integration.", - ConfParamValue: apiSecret, - ConfParamRequired: true, - ConfParamDatatype: "password", - }, - } - - payload, err := json.Marshal(params) - if err != nil { - return catcher.Error("failed to marshal parameters", err, nil) - } - - req, err := http.NewRequestWithContext(ctx, "PUT", url, bytes.NewReader(payload)) - if err != nil { - return catcher.Error("failed to create request", err, nil) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Utm-Internal-Key", c.internalKey) - - resp, err := c.httpClient.Do(req) - if err != nil { - return catcher.Error("request failed", err, nil) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return catcher.Error("unexpected status from backend", nil, map[string]any{ - "status": resp.StatusCode, - "body": string(body), - }) - } - - catcher.Info("ThreadWinds credentials saved successfully", nil) - return nil -} diff --git a/plugins/feeds/internal/client/cm_client.go b/plugins/feeds/internal/client/cm_client.go deleted file mode 100644 index 60b929487..000000000 --- a/plugins/feeds/internal/client/cm_client.go +++ /dev/null @@ -1,79 +0,0 @@ -package client - -import ( - "fmt" - "net/http" - "time" - - "github.com/threatwinds/go-sdk/catcher" - - sdkutils "github.com/threatwinds/go-sdk/utils" - "github.com/utmstack/UTMStack/plugins/feeds/utils" -) - -const ( - instanceConfigPath = "/updates/instance-config.yml" -) - -type CustomersManagerClient struct { - Server string `yaml:"server"` - InstanceID string `yaml:"instance_id"` - InstanceKey string `yaml:"instance_key"` -} - -type RegistrationResponse struct { - APIKey string `json:"api_key"` - APISecret string `json:"api_secret"` -} - -func (c *CustomersManagerClient) LoadInstanceConfig() error { - time.Sleep(10 * time.Second) - - loadFunc := func() error { - if !utils.CheckIfPathExist(instanceConfigPath) { - return catcher.Error("config file not found", nil, nil) - } - - if err := utils.ReadYAML(instanceConfigPath, c); err != nil { - return catcher.Error("failed to read or parse YAML config", err, nil) - } - - if c.Server == "" || c.InstanceID == "" || c.InstanceKey == "" { - return catcher.Error("missing required fields in config", nil, nil) - } - - return nil - } - - return utils.Retry(loadFunc, "instance config loading", utils.DefaultRetryConfig()) -} - -func (c *CustomersManagerClient) RegisterUserReporter() (*RegistrationResponse, error) { - if c.Server == "" || c.InstanceID == "" || c.InstanceKey == "" { - return nil, catcher.Error("instance configuration not loaded", nil, nil) - } - - endpoint := fmt.Sprintf("%s/api/v1/intelligence/register", c.Server) - - headers := map[string]string{ - "accept": "application/json", - "id": c.InstanceID, - "Key": c.InstanceKey, - } - - credentials, _, err := sdkutils.DoReq[RegistrationResponse]( - endpoint, - nil, - http.MethodPost, - headers, - false, - ) - - if err != nil { - return nil, err - } - - catcher.Info("Successfully registered ThreadWinds intelligence reporter", nil) - - return &credentials, nil -} diff --git a/plugins/feeds/internal/client/dependencies.go b/plugins/feeds/internal/client/dependencies.go deleted file mode 100644 index 9f712ab97..000000000 --- a/plugins/feeds/internal/client/dependencies.go +++ /dev/null @@ -1,31 +0,0 @@ -package client - -import ( - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" -) - -type ClientDependencies struct { - Backend *BackendClient - CM *CustomersManagerClient - ThreadWinds *ThreadWindsClient - OpenSearch *OpenSearchClient -} - -func NewClientDependencies(cfg *config.TWConfig) (*ClientDependencies, error) { - catcher.Info("initializing client dependencies", nil) - - opensearch, err := NewOpenSearchClient(cfg) - if err != nil { - return nil, catcher.Error("failed to initialize opensearch client", err, nil) - } - - deps := &ClientDependencies{ - Backend: NewBackendClient(cfg), - CM: &CustomersManagerClient{}, - ThreadWinds: NewThreadWindsClient(cfg), - OpenSearch: opensearch, - } - - return deps, nil -} diff --git a/plugins/feeds/internal/client/opensearch_client.go b/plugins/feeds/internal/client/opensearch_client.go deleted file mode 100644 index b95131b90..000000000 --- a/plugins/feeds/internal/client/opensearch_client.go +++ /dev/null @@ -1,103 +0,0 @@ -package client - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "net/http" - - opensearch "github.com/opensearch-project/opensearch-go/v2" - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type OpenSearchClient struct { - client *opensearch.Client -} - -func NewOpenSearchClient(cfg *config.TWConfig) (*OpenSearchClient, error) { - osConfig := opensearch.Config{ - Addresses: []string{ - fmt.Sprintf("https://%s:%s", cfg.OpenSearchHost, cfg.OpenSearchPort), - }, - Username: cfg.OpenSearchUser, - Password: cfg.OpenSearchPassword, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - client, err := opensearch.NewClient(osConfig) - if err != nil { - return nil, catcher.Error("failed to create opensearch client", err, nil) - } - - info, err := client.Info() - if err != nil { - return nil, catcher.Error("opensearch connection failed", err, nil) - } - defer info.Body.Close() - - catcher.Info("opensearch client connected successfully", nil) - - return &OpenSearchClient{client: client}, nil -} - -func (c *OpenSearchClient) GetAlertByID(ctx context.Context, alertID string) (*models.Alert, error) { - query := map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "id.keyword": alertID, - }, - }, - } - - return c.searchSingleAlert(ctx, "v11-alert-*", query) -} - -func (c *OpenSearchClient) searchSingleAlert(ctx context.Context, index string, query map[string]any) (*models.Alert, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(query); err != nil { - return nil, catcher.Error("failed to encode query", err, nil) - } - - res, err := c.client.Search( - c.client.Search.WithContext(ctx), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(&buf), - ) - if err != nil { - return nil, catcher.Error("search failed", err, nil) - } - defer res.Body.Close() - - if res.IsError() { - body, _ := io.ReadAll(res.Body) - return nil, catcher.Error("search error", nil, map[string]any{ - "status": res.StatusCode, - "body": string(body), - }) - } - - var result struct { - Hits struct { - Hits []struct { - Source models.Alert `json:"_source"` - } `json:"hits"` - } `json:"hits"` - } - - if err := json.NewDecoder(res.Body).Decode(&result); err != nil { - return nil, catcher.Error("failed to decode response", err, nil) - } - - if len(result.Hits.Hits) == 0 { - return nil, catcher.Error("alert not found", nil, nil) - } - - return &result.Hits.Hits[0].Source, nil -} diff --git a/plugins/feeds/internal/client/threadwinds_client.go b/plugins/feeds/internal/client/threadwinds_client.go deleted file mode 100644 index 1fabe711f..000000000 --- a/plugins/feeds/internal/client/threadwinds_client.go +++ /dev/null @@ -1,170 +0,0 @@ -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "sync" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" - "github.com/utmstack/UTMStack/plugins/feeds/config" -) - -type ThreadWindsClient struct { - baseURL string - apiKey string - apiSecret string - httpClient *http.Client - mu sync.RWMutex -} - -func NewThreadWindsClient(cfg *config.TWConfig) *ThreadWindsClient { - return &ThreadWindsClient{ - baseURL: cfg.ThreadWindsURL, - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - } -} - -func (c *ThreadWindsClient) UpdateCredentials(apiKey, apiSecret string) { - c.mu.Lock() - defer c.mu.Unlock() - - c.apiKey = apiKey - c.apiSecret = apiSecret - - catcher.Info("ThreadWinds credentials updated", nil) -} - -func (c *ThreadWindsClient) ingestEntity(ctx context.Context, entity *entities.Entity) error { - url := fmt.Sprintf("%s/api/ingest/v1/entity", c.baseURL) - - payload, err := json.Marshal(entity) - if err != nil { - return catcher.Error("failed to marshal entity", err, map[string]any{ - "entity_type": entity.Type, - }) - } - - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) - if err != nil { - return catcher.Error("failed to create request", err, map[string]any{ - "entity_type": entity.Type, - }) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("api-key", c.apiKey) - req.Header.Set("api-secret", c.apiSecret) - - return c.executeWithRetry(req, entity.Type) -} - -func (c *ThreadWindsClient) executeWithRetry(req *http.Request, entityType string) error { - maxRetries := 3 - backoff := time.Second - - for attempt := 1; attempt <= maxRetries; attempt++ { - resp, err := c.httpClient.Do(req) - if err != nil { - catcher.Error("http request failed", err, map[string]any{ - "attempt": attempt, - "entity_type": entityType, - }) - if attempt < maxRetries { - time.Sleep(backoff) - backoff *= 2 - continue - } - return catcher.Error("failed after max attempts", err, map[string]any{ - "max_retries": maxRetries, - "entity_type": entityType, - }) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return catcher.Error("failed to read response body", err, map[string]any{ - "entity_type": entityType, - }) - } - resp.Body.Close() - - if resp.StatusCode == http.StatusAccepted { - return nil - } - - if resp.StatusCode >= 400 && resp.StatusCode < 500 { - return catcher.Error("client error from ThreadWinds", nil, map[string]any{ - "status": resp.StatusCode, - }) - } - - if resp.StatusCode >= 500 && attempt < maxRetries { - catcher.Error("server error, retrying", fmt.Errorf("server error %d", resp.StatusCode), map[string]any{ - "attempt": attempt, - "entity_type": entityType, - "response": string(body), - }) - time.Sleep(backoff) - backoff *= 2 - continue - } - - return catcher.Error("unexpected status code from ThreadWinds", nil, map[string]any{ - "status": resp.StatusCode, - }) - } - - return catcher.Error("max retries exceeded", nil, map[string]any{ - "entity_type": entityType, - }) -} - -func (c *ThreadWindsClient) IngestBatch(ctx context.Context, entityBatch []*entities.Entity) error { - successCount := 0 - errorCount := 0 - - for i, entity := range entityBatch { - select { - case <-ctx.Done(): - return catcher.Error("batch ingestion cancelled", ctx.Err(), map[string]any{ - "processed": successCount, - }) - default: - } - - err := c.ingestEntity(ctx, entity) - if err != nil { - errorCount++ - catcher.Error("failed to ingest entity", err, map[string]any{ - "entity_type": entity.Type, - "batch_index": i, - "success_count": successCount, - "error_count": errorCount, - }) - continue - } - successCount++ - - if i < len(entityBatch)-1 { - time.Sleep(100 * time.Millisecond) - } - } - - if errorCount > 0 { - return catcher.Error("batch completed with errors", nil, map[string]any{ - "error_count": errorCount, - "success_count": successCount, - "total_count": len(entityBatch), - }) - } - - return nil -} diff --git a/plugins/feeds/internal/client/threadwinds_setup.go b/plugins/feeds/internal/client/threadwinds_setup.go deleted file mode 100644 index ddbb44405..000000000 --- a/plugins/feeds/internal/client/threadwinds_setup.go +++ /dev/null @@ -1,58 +0,0 @@ -package client - -import ( - "context" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/utils" -) - -func ConfigureThreadWindsCredentials(ctx context.Context, deps *ClientDependencies, twConfig *ThreadWindsConfig) error { - if twConfig.APIKey == "" || twConfig.APISecret == "" { - catcher.Info("ThreadWinds not configured, will attempt registration with retry...", nil) - - if err := deps.CM.LoadInstanceConfig(); err != nil { - return catcher.Error("failed to load instance configuration", err, nil) - } - - regResp, err := registerWithRetry(deps.CM) - if err != nil { - return catcher.Error("failed to register after all retry attempts", err, nil) - } - - if err := deps.Backend.SaveThreadWindsCredentials(ctx, - regResp.APIKey, - regResp.APISecret, - twConfig.KeyID, - twConfig.SecretID); err != nil { - return catcher.Error("failed to save ThreadWinds credentials", err, nil) - } - - deps.ThreadWinds.UpdateCredentials(regResp.APIKey, regResp.APISecret) - catcher.Info("ThreadWinds configured successfully with new credentials", nil) - } else { - catcher.Info("ThreadWinds already configured", nil) - deps.ThreadWinds.UpdateCredentials(twConfig.APIKey, twConfig.APISecret) - } - - return nil -} - -func registerWithRetry(cm *CustomersManagerClient) (*RegistrationResponse, error) { - var regResp *RegistrationResponse - - registerFunc := func() error { - resp, err := cm.RegisterUserReporter() - if err != nil { - return err - } - regResp = resp - return nil - } - - if err := utils.Retry(registerFunc, "ThreadWinds registration", utils.DefaultRetryConfig()); err != nil { - return nil, err - } - - return regResp, nil -} diff --git a/plugins/feeds/internal/extractor/field_extractor.go b/plugins/feeds/internal/extractor/field_extractor.go deleted file mode 100644 index abfdef90b..000000000 --- a/plugins/feeds/internal/extractor/field_extractor.go +++ /dev/null @@ -1,535 +0,0 @@ -package extractor - -import ( - "fmt" - - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type FieldExtractor struct{} - -func NewFieldExtractor() *FieldExtractor { - return &FieldExtractor{} -} - -func (e *FieldExtractor) ExtractFromEvent(event *models.Event) []*models.FlattenedField { - fields := make([]*models.FlattenedField, 0, 50) - - if event.Origin != nil { - fields = append(fields, e.extractFromSide(event.Origin, fmt.Sprintf("event.%s.origin", event.ID))...) - } - - if event.Target != nil { - fields = append(fields, e.extractFromSide(event.Target, fmt.Sprintf("event.%s.target", event.ID))...) - } - - return fields -} - -func (e *FieldExtractor) extractFromSide(side *models.Side, prefix string) []*models.FlattenedField { - fields := make([]*models.FlattenedField, 0, 60) - - // ==================== Network Identification ==================== - if side.IP != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".ip", - Key: "ip", - Value: side.IP, - }) - } - - if side.Port != 0 { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".port", - Key: "port", - Value: side.Port, - }) - } - - if side.Host != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".host", - Key: "hostname", - Value: side.Host, - }) - } - - if side.Domain != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".domain", - Key: "domain", - Value: side.Domain, - }) - } - - if side.Mac != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".mac", - Key: "mac-address", - Value: side.Mac, - }) - } - - if side.URL != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".url", - Key: "url", - Value: side.URL, - }) - } - - if side.Cidr != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".cidr", - Key: "cidr", - Value: side.Cidr, - }) - } - - // ==================== Geolocation ==================== - if side.Geolocation != nil && side.Geolocation.ASN != 0 { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".geolocation.asn", - Key: "asn", - Value: side.Geolocation.ASN, - }) - } - - // ==================== Certificates & Fingerprints ==================== - if side.CertificateFingerprint != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".certificateFingerprint", - Key: "certificate-fingerprint", - Value: side.CertificateFingerprint, - }) - } - - if side.Ja3Fingerprint != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".ja3Fingerprint", - Key: "ja3-fingerprint", - Value: side.Ja3Fingerprint, - }) - } - - if side.JarmFingerprint != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".jarmFingerprint", - Key: "jarm-fingerprint", - Value: side.JarmFingerprint, - }) - } - - if side.SshBanner != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sshBanner", - Key: "ssh-banner", - Value: side.SshBanner, - }) - } - - if side.SshFingerprint != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sshFingerprint", - Key: "ssh-fingerprint", - Value: side.SshFingerprint, - }) - } - - // ==================== Web Attributes ==================== - if side.Cookie != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".cookie", - Key: "cookie", - Value: side.Cookie, - }) - } - - if side.JabberId != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".jabberId", - Key: "jabber-id", - Value: side.JabberId, - }) - } - - // ==================== Email Attributes ==================== - if side.EmailAddress != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailAddress", - Key: "email-address", - Value: side.EmailAddress, - }) - } - - if side.EmailBody != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailBody", - Key: "email-body", - Value: side.EmailBody, - }) - } - - if side.EmailDisplayName != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailDisplayName", - Key: "email-display-name", - Value: side.EmailDisplayName, - }) - } - - if side.EmailSubject != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailSubject", - Key: "email-subject", - Value: side.EmailSubject, - }) - } - - if side.EmailThreadIndex != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailThreadIndex", - Key: "email-thread-index", - Value: side.EmailThreadIndex, - }) - } - - if side.EmailXMailer != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".emailXMailer", - Key: "email-x-mailer", - Value: side.EmailXMailer, - }) - } - - if side.Dkim != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".dkim", - Key: "dkim", - Value: side.Dkim, - }) - } - - if side.DkimSignature != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".dkimSignature", - Key: "dkim-signature", - Value: side.DkimSignature, - }) - } - - // ==================== WHOIS Attributes ==================== - if side.WhoisRegistrant != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".whoisRegistrant", - Key: "whois-registrant", - Value: side.WhoisRegistrant, - }) - } - - if side.WhoisRegistrar != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".whoisRegistrar", - Key: "whois-registrar", - Value: side.WhoisRegistrar, - }) - } - - // ==================== Identity Attributes ==================== - if side.User != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".user", - Key: "username", - Value: side.User, - }) - } - - // ==================== Process Attributes ==================== - if side.Process != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".process", - Key: "process", - Value: side.Process, - }) - } - - if side.ProcessState != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".processState", - Key: "process-state", - Value: side.ProcessState, - }) - } - - if side.WindowsScheduledTask != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".windowsScheduledTask", - Key: "windows-scheduled-task", - Value: side.WindowsScheduledTask, - }) - } - - if side.WindowsServiceDisplayName != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".windowsServiceDisplayName", - Key: "windows-service-display-name", - Value: side.WindowsServiceDisplayName, - }) - } - - if side.WindowsServiceName != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".windowsServiceName", - Key: "windows-service-name", - Value: side.WindowsServiceName, - }) - } - - // ==================== File Attributes ==================== - if side.File != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".file", - Key: "file", - Value: side.File, - }) - } - - if side.Path != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".path", - Key: "path", - Value: side.Path, - }) - } - - if side.Filename != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".filename", - Key: "filename", - Value: side.Filename, - }) - } - - if side.SizeInBytes != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sizeInBytes", - Key: "size-in-bytes", - Value: side.SizeInBytes, - }) - } - - if side.MimeType != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".mimeType", - Key: "mime-type", - Value: side.MimeType, - }) - } - - // ==================== Hash Attributes ==================== - if side.Authentihash != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".authentihash", - Key: "authentihash", - Value: side.Authentihash, - }) - } - - if side.Cdhash != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".cdhash", - Key: "cdhash", - Value: side.Cdhash, - }) - } - - if side.MD5 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".md5", - Key: "md5", - Value: side.MD5, - }) - } - - if side.SHA1 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha1", - Key: "sha1", - Value: side.SHA1, - }) - } - - if side.SHA224 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha224", - Key: "sha224", - Value: side.SHA224, - }) - } - - if side.SHA256 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha256", - Key: "sha256", - Value: side.SHA256, - }) - } - - if side.SHA384 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha384", - Key: "sha384", - Value: side.SHA384, - }) - } - - if side.SHA3224 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha3224", - Key: "sha3-224", - Value: side.SHA3224, - }) - } - - if side.SHA3256 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha3256", - Key: "sha3-256", - Value: side.SHA3256, - }) - } - - if side.SHA3384 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha3384", - Key: "sha3-384", - Value: side.SHA3384, - }) - } - - if side.SHA3512 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha3512", - Key: "sha3-512", - Value: side.SHA3512, - }) - } - - if side.SHA512 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha512", - Key: "sha512", - Value: side.SHA512, - }) - } - - if side.SHA512224 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha512224", - Key: "sha512-224", - Value: side.SHA512224, - }) - } - - if side.SHA512256 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".sha512256", - Key: "sha512-256", - Value: side.SHA512256, - }) - } - - if side.Hex != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".hex", - Key: "hex", - Value: side.Hex, - }) - } - - if side.Base64 != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".base64", - Key: "base64", - Value: side.Base64, - }) - } - - // ==================== System Attributes ==================== - if side.ChromeExtension != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".chromeExtension", - Key: "chrome-extension-id", - Value: side.ChromeExtension, - }) - } - - if side.MobileAppId != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".mobileAppId", - Key: "mobile-app-id", - Value: side.MobileAppId, - }) - } - - // ==================== Vulnerability Attributes ==================== - if side.Cpe != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".cpe", - Key: "cpe", - Value: side.Cpe, - }) - } - - if side.Cve != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".cve", - Key: "cve", - Value: side.Cve, - }) - } - - // ==================== Malware Attributes ==================== - if side.Malware != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".malware", - Key: "malware", - Value: side.Malware, - }) - } - - if side.MalwareFamily != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".malwareFamily", - Key: "malware-family", - Value: side.MalwareFamily, - }) - } - - if side.MalwareType != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".malwareType", - Key: "malware-type", - Value: side.MalwareType, - }) - } - - // ==================== Key Attributes ==================== - if side.PgpPrivateKey != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".pgpPrivateKey", - Key: "pgp-private-key", - Value: side.PgpPrivateKey, - }) - } - - if side.PgpPublicKey != "" { - fields = append(fields, &models.FlattenedField{ - Path: prefix + ".pgpPublicKey", - Key: "pgp-public-key", - Value: side.PgpPublicKey, - }) - } - - return fields -} diff --git a/plugins/feeds/internal/initializer/app.go b/plugins/feeds/internal/initializer/app.go deleted file mode 100644 index 8fb9d553f..000000000 --- a/plugins/feeds/internal/initializer/app.go +++ /dev/null @@ -1,72 +0,0 @@ -package initializer - -import ( - "context" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" - "github.com/utmstack/UTMStack/plugins/feeds/internal/scheduler" - "golang.org/x/sync/errgroup" -) - -type App struct { - config *config.TWConfig - clients *client.ClientDependencies - scheduler *scheduler.IngestionScheduler - eg *errgroup.Group - egCtx context.Context -} - -func NewApp(ctx context.Context) (*App, error) { - app := &App{} - - if err := app.loadConfiguration(); err != nil { - return nil, err - } - - if err := app.initializeClients(); err != nil { - return nil, err - } - - if err := app.configureThreadWinds(ctx); err != nil { - return nil, err - } - - if err := app.buildProcessingPipeline(); err != nil { - return nil, err - } - - return app, nil -} - -func (a *App) Run(ctx context.Context) error { - a.eg, a.egCtx = errgroup.WithContext(ctx) - a.eg.Go(func() error { - a.scheduler.Start(a.egCtx) - return nil - }) - return nil -} - -func (a *App) Shutdown(ctx context.Context) error { - catcher.Info("shutting down application", nil) - - done := make(chan error, 1) - go func() { - done <- a.eg.Wait() - }() - - select { - case err := <-done: - if err != nil { - catcher.Error("scheduler stopped with error", err, nil) - return err - } - catcher.Info("scheduler stopped gracefully", nil) - return nil - case <-ctx.Done(): - catcher.Info("shutdown timeout exceeded, forcing shutdown", nil) - return ctx.Err() - } -} diff --git a/plugins/feeds/internal/initializer/clients.go b/plugins/feeds/internal/initializer/clients.go deleted file mode 100644 index 6ee7d56e5..000000000 --- a/plugins/feeds/internal/initializer/clients.go +++ /dev/null @@ -1,33 +0,0 @@ -package initializer - -import ( - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" -) - -func (a *App) loadConfiguration() error { - catcher.Info("loading configuration", nil) - - cfg, err := config.GetTWConfig() - if err != nil { - return catcher.Error("failed to load configuration", err, nil) - } - - a.config = cfg - return nil -} - -func (a *App) initializeClients() error { - catcher.Info("initializing clients", nil) - - clients, err := client.NewClientDependencies(a.config) - if err != nil { - return catcher.Error("failed to initialize clients", err, nil) - } - - a.clients = clients - - catcher.Info("all client dependencies initialized successfully", nil) - return nil -} diff --git a/plugins/feeds/internal/initializer/pipeline.go b/plugins/feeds/internal/initializer/pipeline.go deleted file mode 100644 index b03d564c5..000000000 --- a/plugins/feeds/internal/initializer/pipeline.go +++ /dev/null @@ -1,41 +0,0 @@ -package initializer - -import ( - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/internal/association" - "github.com/utmstack/UTMStack/plugins/feeds/internal/extractor" - "github.com/utmstack/UTMStack/plugins/feeds/internal/mapper" - "github.com/utmstack/UTMStack/plugins/feeds/internal/scheduler" - "github.com/utmstack/UTMStack/plugins/feeds/internal/service" -) - -func (a *App) buildProcessingPipeline() error { - catcher.Info("building processing pipeline", nil) - - fieldExtractor := extractor.NewFieldExtractor() - entityMapper := mapper.NewEntityMapper() - associationBuilder := association.NewAssociationBuilder() - - entityBuilder := service.NewEntityBuilder(entityMapper) - - alertProcessor := service.NewAlertProcessor( - a.clients.OpenSearch, - fieldExtractor, - entityBuilder, - ) - - incidentProcessor := service.NewIncidentProcessor( - a.clients, - alertProcessor, - associationBuilder, - ) - - a.scheduler = scheduler.NewIngestionScheduler( - a.config, - a.clients, - incidentProcessor, - ) - - catcher.Info("processing pipeline built successfully", nil) - return nil -} diff --git a/plugins/feeds/internal/initializer/setup.go b/plugins/feeds/internal/initializer/setup.go deleted file mode 100644 index b4e41c40a..000000000 --- a/plugins/feeds/internal/initializer/setup.go +++ /dev/null @@ -1,25 +0,0 @@ -package initializer - -import ( - "context" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" -) - -func (a *App) configureThreadWinds(ctx context.Context) error { - catcher.Info("configuring ThreadWinds credentials", nil) - - twConfig, err := a.clients.Backend.GetThreadWindsConfig(ctx) - if err != nil { - return catcher.Error("failed to check ThreadWinds configuration", err, nil) - } - - err = client.ConfigureThreadWindsCredentials(ctx, a.clients, twConfig) - if err != nil { - return catcher.Error("failed to configure ThreadWinds", err, nil) - } - - catcher.Info("ThreadWinds configured successfully", nil) - return nil -} diff --git a/plugins/feeds/internal/mapper/entity_mapper.go b/plugins/feeds/internal/mapper/entity_mapper.go deleted file mode 100644 index 3fdc96a1f..000000000 --- a/plugins/feeds/internal/mapper/entity_mapper.go +++ /dev/null @@ -1,140 +0,0 @@ -package mapper - -import ( - "fmt" - "strings" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/entities" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type EntityMapper struct { - entityTypes map[string]bool -} - -func NewEntityMapper() *EntityMapper { - mapper := &EntityMapper{ - entityTypes: make(map[string]bool), - } - - for _, def := range entities.Definitions { - mapper.entityTypes[def.Type] = true - } - - catcher.Info("entity mapper initialized", map[string]any{ - "total_entity_types": len(mapper.entityTypes), - }) - - return mapper -} - -func (m *EntityMapper) MapFieldToEntityType(field *models.FlattenedField) (string, bool) { - leafKey := normalizeKey(field.Key) - - if m.entityTypes[leafKey] { - return leafKey, true - } - - return "", false -} - -func normalizeKey(key string) string { - key = strings.ToLower(key) - key = strings.ReplaceAll(key, "_", "-") - return key -} - -func (m *EntityMapper) BuildEntity(entityType string, value any, context EntityEnrichmentContext) (*entities.Entity, string, error) { - validatedValue, hash, err := entities.ValidateValue(value, entityType) - if err != nil { - return nil, "", catcher.Error("validation failed for entity type", err, map[string]any{ - "entity_type": entityType, - }) - } - - attrs := entities.Attributes{} - if !attrs.SetAttribute(entityType, validatedValue) { - return nil, "", catcher.Error("failed to set attribute for entity type", nil, map[string]any{ - "entity_type": entityType, - }) - } - - if context.Country != "" { - attrs.Country = &context.Country - } - if context.City != "" { - attrs.City = &context.City - } - if context.ASO != "" { - attrs.Aso = &context.ASO - } - if context.Latitude != nil { - attrs.Latitude = context.Latitude - } - if context.Longitude != nil { - attrs.Longitude = context.Longitude - } - if context.AccuracyRadius != nil { - attrs.AccuracyRadius = context.AccuracyRadius - } - - reputation := calculateReputation(context.Severity) - - tags := buildEntityTags(context) - - entity := &entities.Entity{ - Type: entityType, - Attributes: attrs, - Reputation: reputation, - Tags: tags, - Associations: nil, - } - - entityID := fmt.Sprintf("%s-%s", entityType, hash) - - return entity, entityID, nil -} - -func buildEntityTags(context EntityEnrichmentContext) []string { - tags := []string{"utmstack"} - - if context.IncidentID != "" { - tags = append(tags, "incident-"+context.IncidentID) - } - - if context.DataType != "" { - tags = append(tags, "datasource-"+context.DataType) - } - - return tags -} - -type EntityEnrichmentContext struct { - IncidentID string - AlertID string - EventID string - - Severity int - DataType string - - SourceType string - - Country string - City string - Latitude *float64 - Longitude *float64 - ASO string - AccuracyRadius *float64 -} - -func calculateReputation(severity int) int { - switch { - case severity >= 7: - return -3 - case severity >= 4: - return -1 - default: - return 0 - } -} diff --git a/plugins/feeds/internal/models/alert.go b/plugins/feeds/internal/models/alert.go deleted file mode 100644 index fe732b4b0..000000000 --- a/plugins/feeds/internal/models/alert.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -type Alert struct { - ID string `json:"id"` - Name string `json:"name"` - DataType string `json:"dataType"` - Events []*Event `json:"events"` -} diff --git a/plugins/feeds/internal/models/event.go b/plugins/feeds/internal/models/event.go deleted file mode 100644 index 7e1465df7..000000000 --- a/plugins/feeds/internal/models/event.go +++ /dev/null @@ -1,112 +0,0 @@ -package models - -type Event struct { - ID string `json:"id"` - Origin *Side `json:"origin,omitempty"` - Target *Side `json:"target,omitempty"` -} - -type Side struct { - // Network identification attributes - IP string `json:"ip,omitempty"` - Port int `json:"port,omitempty"` - Host string `json:"host,omitempty"` - Domain string `json:"domain,omitempty"` - Mac string `json:"mac,omitempty"` - Geolocation *Geolocation `json:"geolocation,omitempty"` - URL string `json:"url,omitempty"` - Cidr string `json:"cidr,omitempty"` - - // Certificate and fingerprint attributes - CertificateFingerprint string `json:"certificateFingerprint,omitempty"` - Ja3Fingerprint string `json:"ja3Fingerprint,omitempty"` - JarmFingerprint string `json:"jarmFingerprint,omitempty"` - SshBanner string `json:"sshBanner,omitempty"` - SshFingerprint string `json:"sshFingerprint,omitempty"` - - // Web attributes - Cookie string `json:"cookie,omitempty"` - JabberId string `json:"jabberId,omitempty"` - - // Email attributes - EmailAddress string `json:"emailAddress,omitempty"` - EmailBody string `json:"emailBody,omitempty"` - EmailDisplayName string `json:"emailDisplayName,omitempty"` - EmailSubject string `json:"emailSubject,omitempty"` - EmailThreadIndex string `json:"emailThreadIndex,omitempty"` - EmailXMailer string `json:"emailXMailer,omitempty"` - Dkim string `json:"dkim,omitempty"` - DkimSignature string `json:"dkimSignature,omitempty"` - - // WHOIS attributes - WhoisRegistrant string `json:"whoisRegistrant,omitempty"` - WhoisRegistrar string `json:"whoisRegistrar,omitempty"` - - // Identity attributes - User string `json:"user,omitempty"` - - // Process-related attributes - Process string `json:"process,omitempty"` - ProcessState string `json:"processState,omitempty"` - WindowsScheduledTask string `json:"windowsScheduledTask,omitempty"` - WindowsServiceDisplayName string `json:"windowsServiceDisplayName,omitempty"` - WindowsServiceName string `json:"windowsServiceName,omitempty"` - - // File-related attributes - File string `json:"file,omitempty"` - Path string `json:"path,omitempty"` - Filename string `json:"filename,omitempty"` - SizeInBytes string `json:"sizeInBytes,omitempty"` - MimeType string `json:"mimeType,omitempty"` - - // Hash-related attributes - Authentihash string `json:"authentihash,omitempty"` - Cdhash string `json:"cdhash,omitempty"` - MD5 string `json:"md5,omitempty"` - SHA1 string `json:"sha1,omitempty"` - SHA224 string `json:"sha224,omitempty"` - SHA256 string `json:"sha256,omitempty"` - SHA384 string `json:"sha384,omitempty"` - SHA3224 string `json:"sha3224,omitempty"` - SHA3256 string `json:"sha3256,omitempty"` - SHA3384 string `json:"sha3384,omitempty"` - SHA3512 string `json:"sha3512,omitempty"` - SHA512 string `json:"sha512,omitempty"` - SHA512224 string `json:"sha512224,omitempty"` - SHA512256 string `json:"sha512256,omitempty"` - Hex string `json:"hex,omitempty"` - Base64 string `json:"base64,omitempty"` - - // System-related attributes - ChromeExtension string `json:"chromeExtension,omitempty"` - MobileAppId string `json:"mobileAppId,omitempty"` - - // Vulnerability-related attributes - Cpe string `json:"cpe,omitempty"` - Cve string `json:"cve,omitempty"` - - // Malware-related attributes - Malware string `json:"malware,omitempty"` - MalwareFamily string `json:"malwareFamily,omitempty"` - MalwareType string `json:"malwareType,omitempty"` - - // Key-related attributes - PgpPrivateKey string `json:"pgpPrivateKey,omitempty"` - PgpPublicKey string `json:"pgpPublicKey,omitempty"` -} - -type FlattenedField struct { - Path string - Key string - Value any -} - -type Geolocation struct { - Country string `json:"country,omitempty"` - City string `json:"city,omitempty"` - Latitude float64 `json:"latitude,omitempty"` - Longitude float64 `json:"longitude,omitempty"` - ASN int `json:"asn,omitempty"` - ASO string `json:"aso,omitempty"` - Accuracy int `json:"accuracy,omitempty"` -} diff --git a/plugins/feeds/internal/models/incident.go b/plugins/feeds/internal/models/incident.go deleted file mode 100644 index 3e513fd45..000000000 --- a/plugins/feeds/internal/models/incident.go +++ /dev/null @@ -1,21 +0,0 @@ -package models - -import "time" - -type Incident struct { - ID int64 `json:"id"` - Name string `json:"incidentName"` - Description string `json:"incidentDescription"` - Status string `json:"incidentStatus"` - Severity int `json:"incidentSeverity"` - CreatedDate time.Time `json:"incidentCreatedDate"` -} - -type IncidentAlert struct { - ID int64 `json:"id"` - IncidentID int64 `json:"incidentId"` - AlertID string `json:"alertId"` - AlertName string `json:"alertName"` - AlertStatus int `json:"alertStatus"` - AlertSeverity int `json:"alertSeverity"` -} diff --git a/plugins/feeds/internal/scheduler/ingestion_scheduler.go b/plugins/feeds/internal/scheduler/ingestion_scheduler.go deleted file mode 100644 index a8f0c323a..000000000 --- a/plugins/feeds/internal/scheduler/ingestion_scheduler.go +++ /dev/null @@ -1,121 +0,0 @@ -package scheduler - -import ( - "context" - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/config" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" - "github.com/utmstack/UTMStack/plugins/feeds/internal/service" -) - -const ( - pollInterval = 5 * time.Minute -) - -type IngestionScheduler struct { - cfg *config.TWConfig - backendClient *client.BackendClient - threadwindsClient *client.ThreadWindsClient - incidentProcessor *service.IncidentProcessor -} - -func NewIngestionScheduler( - cfg *config.TWConfig, - deps *client.ClientDependencies, - incidentProcessor *service.IncidentProcessor, -) *IngestionScheduler { - return &IngestionScheduler{ - cfg: cfg, - backendClient: deps.Backend, - threadwindsClient: deps.ThreadWinds, - incidentProcessor: incidentProcessor, - } -} - -func (s *IngestionScheduler) Start(ctx context.Context) { - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - - catcher.Info("ingestion scheduler started", map[string]any{ - "poll_interval": pollInterval, - }) - - s.runIngestionCycle(ctx) - - for { - select { - case <-ctx.Done(): - catcher.Info("scheduler received shutdown signal, stopping gracefully", nil) - return - case <-ticker.C: - s.runIngestionCycle(ctx) - } - } -} - -func (s *IngestionScheduler) runIngestionCycle(ctx context.Context) { - startTime := time.Now() - - cycleTimeout := time.Duration(float64(pollInterval) * 0.9) - cycleCtx, cancel := context.WithTimeout(ctx, cycleTimeout) - defer cancel() - - twConfig, err := s.backendClient.GetThreadWindsConfig(cycleCtx) - if err != nil { - catcher.Error("failed to get ThreadWinds configuration", err, nil) - return - } - - if twConfig.Enabled != "true" { - catcher.Info("ThreadWinds is disabled, skipping ingestion cycle", nil) - return - } - - if twConfig.APIKey != "" && twConfig.APISecret != "" { - s.threadwindsClient.UpdateCredentials(twConfig.APIKey, twConfig.APISecret) - } - - incidents, err := s.backendClient.GetRecentIncidents(cycleCtx) - if err != nil { - catcher.Error("failed to fetch incidents", err, nil) - return - } - - if len(incidents) == 0 { - catcher.Info("no recent incidents to process", nil) - return - } - - totalEntities := 0 - for i, incident := range incidents { - select { - case <-cycleCtx.Done(): - catcher.Info("cycle timeout or cancellation, stopping", map[string]any{ - "processed_incidents": i, - "total_incidents": len(incidents), - "reason": cycleCtx.Err().Error(), - }) - return - default: - } - - entitiesCount, err := s.incidentProcessor.ProcessIncident(cycleCtx, incident) - if err != nil { - catcher.Error("failed to process incident", err, map[string]any{ - "incident_id": incident.ID, - "incident_name": incident.Name, - }) - continue - } - totalEntities += entitiesCount - } - - duration := time.Since(startTime) - catcher.Info("ingestion cycle completed", map[string]any{ - "duration_seconds": duration.Seconds(), - "incidents_processed": len(incidents), - "total_entities": totalEntities, - }) -} diff --git a/plugins/feeds/internal/service/alert_processor.go b/plugins/feeds/internal/service/alert_processor.go deleted file mode 100644 index 996b83418..000000000 --- a/plugins/feeds/internal/service/alert_processor.go +++ /dev/null @@ -1,58 +0,0 @@ -package service - -import ( - "context" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/internal/association" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" - "github.com/utmstack/UTMStack/plugins/feeds/internal/extractor" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type AlertProcessor struct { - opensearchClient *client.OpenSearchClient - fieldExtractor *extractor.FieldExtractor - entityBuilder *EntityBuilder -} - -func NewAlertProcessor( - opensearchClient *client.OpenSearchClient, - fieldExtractor *extractor.FieldExtractor, - entityBuilder *EntityBuilder, -) *AlertProcessor { - return &AlertProcessor{ - opensearchClient: opensearchClient, - fieldExtractor: fieldExtractor, - entityBuilder: entityBuilder, - } -} - -func (p *AlertProcessor) ProcessAlertWithAssociations( - ctx context.Context, - incidentAlert *models.IncidentAlert, - incident *models.Incident, - associationBuilder *association.AssociationBuilder, -) error { - alert, err := p.opensearchClient.GetAlertByID(ctx, incidentAlert.AlertID) - if err != nil { - return catcher.Error("failed to get alert", err, map[string]any{ - "alert_id": incidentAlert.AlertID, - "incident_id": incident.ID, - }) - } - - for _, event := range alert.Events { - eventFields := p.fieldExtractor.ExtractFromEvent(event) - - p.entityBuilder.MapAndRegisterFieldsToEntities( - eventFields, - incident, - alert, - event, - associationBuilder, - ) - } - - return nil -} diff --git a/plugins/feeds/internal/service/entity_builder.go b/plugins/feeds/internal/service/entity_builder.go deleted file mode 100644 index f010f83a5..000000000 --- a/plugins/feeds/internal/service/entity_builder.go +++ /dev/null @@ -1,105 +0,0 @@ -package service - -import ( - "fmt" - "strings" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/internal/association" - "github.com/utmstack/UTMStack/plugins/feeds/internal/mapper" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type EntityBuilder struct { - entityMapper *mapper.EntityMapper -} - -func NewEntityBuilder(entityMapper *mapper.EntityMapper) *EntityBuilder { - return &EntityBuilder{ - entityMapper: entityMapper, - } -} - -func (b *EntityBuilder) MapAndRegisterFieldsToEntities( - fields []*models.FlattenedField, - incident *models.Incident, - alert *models.Alert, - event *models.Event, - associationBuilder *association.AssociationBuilder, -) { - for _, field := range fields { - entityType, matched := b.entityMapper.MapFieldToEntityType(field) - if !matched { - continue - } - - sourceField := "" - var sideContext *models.Side - var sourceType string - - if strings.Contains(field.Path, ".origin") { - sourceField = "origin" - sourceType = "event.origin" - sideContext = event.Origin - } else if strings.Contains(field.Path, ".target") { - sourceField = "target" - sourceType = "event.target" - sideContext = event.Target - } - - enrichmentCtx := b.buildEnrichmentContext(incident, alert, sideContext, event.ID, sourceType) - entity, entityID, err := b.entityMapper.BuildEntity(entityType, field.Value, enrichmentCtx) - if err != nil { - catcher.Error("failed to build entity", err, map[string]any{ - "entity_type": entityType, - "field_path": field.Path, - "event_id": event.ID, - }) - continue - } - - assocContext := association.AssociationContext{ - AlertID: alert.ID, - EventID: event.ID, - IncidentID: fmt.Sprintf("%d", incident.ID), - SourceField: sourceField, - } - associationBuilder.RegisterEntity(entity, entityID, field.Path, assocContext) - } -} - -func (b *EntityBuilder) buildEnrichmentContext( - incident *models.Incident, - alert *models.Alert, - side *models.Side, - eventID string, - sourceType string, -) mapper.EntityEnrichmentContext { - ctx := mapper.EntityEnrichmentContext{ - IncidentID: fmt.Sprintf("%d", incident.ID), - AlertID: alert.ID, - EventID: eventID, - Severity: incident.Severity, - DataType: alert.DataType, - SourceType: sourceType, - } - - if side != nil && side.Geolocation != nil { - geo := side.Geolocation - ctx.Country = geo.Country - ctx.City = geo.City - ctx.ASO = geo.ASO - - if geo.Latitude != 0.0 || geo.Longitude != 0.0 { - ctx.Latitude = &geo.Latitude - ctx.Longitude = &geo.Longitude - } - - if geo.Accuracy > 0 { - accuracy := float64(geo.Accuracy) - ctx.AccuracyRadius = &accuracy - } - } - - return ctx -} diff --git a/plugins/feeds/internal/service/incident_processor.go b/plugins/feeds/internal/service/incident_processor.go deleted file mode 100644 index 99e344f9d..000000000 --- a/plugins/feeds/internal/service/incident_processor.go +++ /dev/null @@ -1,69 +0,0 @@ -package service - -import ( - "context" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/utmstack/UTMStack/plugins/feeds/internal/association" - "github.com/utmstack/UTMStack/plugins/feeds/internal/client" - "github.com/utmstack/UTMStack/plugins/feeds/internal/models" -) - -type IncidentProcessor struct { - backendClient *client.BackendClient - opensearchClient *client.OpenSearchClient - threadwindsClient *client.ThreadWindsClient - alertProcessor *AlertProcessor - associationBuilder *association.AssociationBuilder -} - -func NewIncidentProcessor( - deps *client.ClientDependencies, - alertProcessor *AlertProcessor, - associationBuilder *association.AssociationBuilder, -) *IncidentProcessor { - return &IncidentProcessor{ - backendClient: deps.Backend, - opensearchClient: deps.OpenSearch, - threadwindsClient: deps.ThreadWinds, - alertProcessor: alertProcessor, - associationBuilder: associationBuilder, - } -} - -func (p *IncidentProcessor) ProcessIncident(ctx context.Context, incident *models.Incident) (int, error) { - p.associationBuilder.ClearRegistry() - - incidentAlerts, err := p.backendClient.GetIncidentAlerts(ctx, incident.ID) - if err != nil { - return 0, catcher.Error("failed to get incident alerts", err, nil) - } - - if len(incidentAlerts) == 0 { - return 0, nil - } - - for _, incidentAlert := range incidentAlerts { - err := p.alertProcessor.ProcessAlertWithAssociations(ctx, incidentAlert, incident, p.associationBuilder) - if err != nil { - catcher.Error("failed to process alert", err, map[string]any{ - "alert_id": incidentAlert.AlertID, - "incident_id": incident.ID, - }) - continue - } - } - - allEntities := p.associationBuilder.BuildAssociations() - - if len(allEntities) > 0 { - if err := p.threadwindsClient.IngestBatch(ctx, allEntities); err != nil { - return 0, catcher.Error("failed to ingest batch", err, map[string]any{ - "incident_id": incident.ID, - "entity_count": len(allEntities), - }) - } - } - - return len(allEntities), nil -} diff --git a/plugins/feeds/main.go b/plugins/feeds/main.go deleted file mode 100644 index bfef0d981..000000000 --- a/plugins/feeds/main.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "context" - "os" - "os/signal" - "syscall" - - "time" - - "github.com/threatwinds/go-sdk/catcher" - "github.com/threatwinds/go-sdk/plugins" - "github.com/utmstack/UTMStack/plugins/feeds/internal/initializer" - "github.com/utmstack/UTMStack/plugins/feeds/utils" -) - -const ( - urlCheckConnection = "https://apis.threatwinds.com" -) - -func main() { - mode := plugins.GetCfg("plugin_com.utmstack.feeds").Env.Mode - if mode != "manager" { - return - } - - catcher.Info("Starting ThreadWinds Ingestion Service", nil) - - for { - if err := utils.ConnectionChecker(urlCheckConnection); err != nil { - _ = catcher.Error("External connection failure detected", err, nil) - continue - } - break - } - - ctx := context.Background() - app, err := initializer.NewApp(ctx) - if err != nil { - _ = catcher.Error("failed to initialize application", err, nil) - time.Sleep(5 * time.Second) - os.Exit(1) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go app.Run(ctx) - - sig := <-sigChan - catcher.Info("received shutdown signal, initiating graceful shutdown", map[string]any{ - "signal": sig.String(), - }) - - cancel() - - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer shutdownCancel() - - if err := app.Shutdown(shutdownCtx); err != nil { - _ = catcher.Error("error during shutdown", err, nil) - time.Sleep(5 * time.Second) - } - - catcher.Info("ThreadWinds Ingestion Service stopped", nil) -} diff --git a/plugins/feeds/utils/aes.go b/plugins/feeds/utils/aes.go deleted file mode 100644 index ce2b0632d..000000000 --- a/plugins/feeds/utils/aes.go +++ /dev/null @@ -1,11 +0,0 @@ -package utils - -import ( - "github.com/AtlasInsideCorp/AtlasInsideAES" - "github.com/threatwinds/go-sdk/plugins" -) - -func DecryptValue(encryptedValue string) (string, error) { - passphrase := plugins.PluginCfg("com.utmstack").Get("internalKey").String() - return AtlasInsideAES.AESDecrypt(encryptedValue, []byte(passphrase)) -} diff --git a/plugins/feeds/utils/env.go b/plugins/feeds/utils/env.go deleted file mode 100644 index a1643bf4d..000000000 --- a/plugins/feeds/utils/env.go +++ /dev/null @@ -1,23 +0,0 @@ -package utils - -import ( - "os" - "time" - - "github.com/threatwinds/go-sdk/catcher" -) - -func Getenv(key string) string { - value, defined := os.LookupEnv(key) - if !defined { - _ = catcher.Error("Error loading environment variable, environment variable does not exist", nil, map[string]any{"key": key}) - time.Sleep(5 * time.Second) - os.Exit(1) - } - if (value == "") || (value == " ") { - _ = catcher.Error("Error loading environment variable, empty environment variable", nil, map[string]any{"key": key}) - time.Sleep(5 * time.Second) - os.Exit(1) - } - return value -} diff --git a/plugins/feeds/utils/files.go b/plugins/feeds/utils/files.go deleted file mode 100644 index 0bdb8a887..000000000 --- a/plugins/feeds/utils/files.go +++ /dev/null @@ -1,40 +0,0 @@ -package utils - -import ( - "os" - "reflect" - - "github.com/threatwinds/go-sdk/catcher" - "gopkg.in/yaml.v2" -) - -func ReadYAML(path string, result interface{}) error { - if result == nil { - return catcher.Error("result interface is nil", nil, nil) - } - - rv := reflect.ValueOf(result) - if rv.Kind() != reflect.Ptr || rv.IsNil() { - return catcher.Error("result must be a non-nil pointer", nil, nil) - } - - file, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = file.Close() }() - - d := yaml.NewDecoder(file) - if err := d.Decode(result); err != nil { - return err - } - - return nil -} - -func CheckIfPathExist(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { - return false - } - return true -} diff --git a/plugins/feeds/utils/retry.go b/plugins/feeds/utils/retry.go deleted file mode 100644 index dc523a7df..000000000 --- a/plugins/feeds/utils/retry.go +++ /dev/null @@ -1,156 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "net/http" - "strings" - "time" - - "github.com/threatwinds/go-sdk/catcher" -) - -type RetryConfig struct { - MaxRetries int // Maximum number of retry attempts (-1 for unlimited) - InitialBackoff time.Duration // Initial wait time before the first retry attempt - MaxBackoff time.Duration // Maximum wait time between retry attempts (upper limit for exponential backoff) - BackoffMultiplier float64 // Growth factor for exponential backoff (0 = fixed wait, >1 = exponential growth) - LogInterval int // Log every N attempts (0 = log only once, >0 = log periodically) - ErrorFilter []string // List of error message substrings to match for retry (nil = retry all errors) - StopOnMismatch bool // If true, return error when it doesn't match ErrorFilter; if false, continue without retrying -} - -func DefaultRetryConfig() RetryConfig { - return RetryConfig{ - MaxRetries: -1, - InitialBackoff: 5 * time.Second, - MaxBackoff: 2 * time.Minute, - BackoffMultiplier: 2.0, - LogInterval: 10, - ErrorFilter: nil, - StopOnMismatch: false, - } -} - -func ConnectionRetryConfig() RetryConfig { - return RetryConfig{ - MaxRetries: -1, - InitialBackoff: 3 * time.Second, - MaxBackoff: 3 * time.Second, - BackoffMultiplier: 0, - LogInterval: 0, - ErrorFilter: []string{"connection failed"}, - StopOnMismatch: true, - } -} - -func Retry(f func() error, operationName string, config RetryConfig) error { - attempt := 0 - currentBackoff := config.InitialBackoff - errorLogged := false - - retryType := "infinite retry" - if config.MaxRetries >= 0 { - retryType = fmt.Sprintf("max %d retries", config.MaxRetries) - } - - if config.LogInterval > 0 { - catcher.Info(fmt.Sprintf("Starting %s with %s", operationName, retryType), map[string]any{ - "initial_backoff": config.InitialBackoff.String(), - "max_backoff": config.MaxBackoff.String(), - }) - } - - for { - attempt++ - err := f() - - if err == nil { - return nil - } - - if len(config.ErrorFilter) > 0 && !matchesErrorFilter(err, config.ErrorFilter) { - if config.StopOnMismatch { - return err - } - continue - } - - if config.MaxRetries >= 0 && attempt > config.MaxRetries { - _ = catcher.Error(fmt.Sprintf("%s failed after %d attempts", operationName, attempt-1), err, map[string]any{ - "max_retries": config.MaxRetries, - }) - return err - } - - shouldLog := config.LogInterval == 0 && !errorLogged || - config.LogInterval > 0 && (attempt == 1 || attempt%config.LogInterval == 0) - - if shouldLog { - logMsg := fmt.Sprintf("%s failed, will retry...", operationName) - if config.MaxRetries < 0 { - logMsg = fmt.Sprintf("%s failed, will retry indefinitely...", operationName) - } - _ = catcher.Error(logMsg, err, map[string]any{ - "attempt": attempt, - "next_retry_in": currentBackoff.String(), - }) - errorLogged = true - } - - time.Sleep(currentBackoff) - - if config.BackoffMultiplier > 0 { - nextBackoff := time.Duration(float64(currentBackoff) * config.BackoffMultiplier) - currentBackoff = min(nextBackoff, config.MaxBackoff) - } - } -} - -func matchesErrorFilter(err error, filters []string) bool { - if err == nil { - return false - } - errMsg := err.Error() - for _, filter := range filters { - if strings.Contains(errMsg, filter) { - return true - } - } - return false -} - -func ConnectionChecker(url string) error { - checkConn := func() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := checkConnection(url, ctx); err != nil { - return fmt.Errorf("connection failed") - } - return nil - } - - return Retry(checkConn, "connection check", ConnectionRetryConfig()) -} - -func checkConnection(url string, ctx context.Context) error { - client := &http.Client{} - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { - if closeErr := resp.Body.Close(); closeErr != nil { - _ = catcher.Error("error closing response body", closeErr, nil) - } - }() - - return nil -}